From 8b9ccacad7244b6877b97a2b6aa6f4a0e53e05a6 Mon Sep 17 00:00:00 2001 From: gziv Date: Tue, 24 Mar 2026 09:16:54 +0200 Subject: [PATCH 1/2] adding tasks for skill evaluation (per skill) --- .../with_skills/PER_SKILL_REVIEW_REPORT.md | 556 ++++++ evaluation/with_skills/SKILL_PATH_FIXES.md | 180 ++ .../environment/.mcp.json | 20 + .../environment/Dockerfile | 70 + .../environment/docs/multi-cluster-auth.md | 248 +++ .../environment/mcp-servers/mock-ocp-mcp.py | 304 ++++ .../scripts/cluster-report/aggregate.py | 601 +++++++ .../scripts/cluster-report/assemble.py | 110 ++ .../cluster-report/build-kubeconfig.py | 446 +++++ .../cluster-report/cluster-reporter-rbac.yaml | 72 + .../scripts/cluster-report/test_aggregate.py | 863 +++++++++ .../scripts/cluster-report/test_assemble.py | 490 ++++++ .../skills/cluster-report/SKILL.md | 387 +++++ .../ocp-admin__cluster-report/instruction.md | 17 + .../solution/solve.sh | 30 + .../ocp-admin__cluster-report/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../ocp-admin__cluster-report/tests/test.sh | 85 + .../tests/test_outputs.py | 105 ++ .../environment/Dockerfile | 78 + .../environment/docs/common-issues.md | 84 + .../environment/docs/examples/model-deploy.md | 160 ++ .../environment/docs/examples/nim-setup.md | 115 ++ .../environment/docs/live-doc-lookup.md | 106 ++ .../docs/references/known-model-profiles.md | 83 + .../docs/references/supported-runtimes.md | 104 ++ .../environment/docs/skill-conventions.md | 85 + .../mcp-servers/mock-observability-mcp.py | 260 +++ .../mcp-servers/mock-openshift-mcp.py | 457 +++++ .../environment/mcp-servers/mock-rhoai-mcp.py | 780 +++++++++ .../skills/ai-observability/SKILL.md | 420 +++++ .../skills/references/common-issues.md | 84 + .../skills/references/live-doc-lookup.md | 106 ++ .../skills/references/skill-conventions.md | 85 + .../instruction.md | 13 + .../solution/solve.sh | 23 + .../task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 91 + .../environment/Dockerfile | 74 + .../environment/docs/common-issues.md | 84 + .../environment/docs/examples/model-deploy.md | 160 ++ .../environment/docs/examples/nim-setup.md | 115 ++ .../environment/docs/live-doc-lookup.md | 106 ++ .../docs/references/known-model-profiles.md | 83 + .../docs/references/supported-runtimes.md | 104 ++ .../environment/docs/skill-conventions.md | 85 + .../mcp-servers/mock-openshift-mcp.py | 457 +++++ .../environment/mcp-servers/mock-rhoai-mcp.py | 780 +++++++++ .../skills/debug-inference/SKILL.md | 348 ++++ .../skills/references/common-issues.md | 84 + .../skills/references/live-doc-lookup.md | 106 ++ .../skills/references/skill-conventions.md | 85 + .../instruction.md | 13 + .../solution/solve.sh | 36 + .../rh-ai-engineer__debug-inference/task.toml | 26 + .../tests/llm_judge.py | 114 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 98 ++ .../environment/Dockerfile | 74 + .../environment/docs/common-issues.md | 84 + .../environment/docs/examples/model-deploy.md | 160 ++ .../environment/docs/examples/nim-setup.md | 115 ++ .../environment/docs/live-doc-lookup.md | 106 ++ .../docs/references/known-model-profiles.md | 83 + .../docs/references/supported-runtimes.md | 104 ++ .../environment/docs/skill-conventions.md | 85 + .../mcp-servers/mock-openshift-mcp.py | 457 +++++ .../environment/mcp-servers/mock-rhoai-mcp.py | 796 +++++++++ .../skills/ds-project-setup/SKILL.md | 336 ++++ .../skills/references/common-issues.md | 84 + .../skills/references/live-doc-lookup.md | 106 ++ .../skills/references/skill-conventions.md | 85 + .../instruction.md | 20 + .../solution/solve.sh | 32 + .../task.toml | 26 + .../tests/llm_judge.py | 102 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 113 ++ .../environment/Dockerfile | 74 + .../environment/docs/common-issues.md | 84 + .../environment/docs/examples/model-deploy.md | 160 ++ .../environment/docs/examples/nim-setup.md | 115 ++ .../environment/docs/live-doc-lookup.md | 106 ++ .../docs/references/known-model-profiles.md | 83 + .../docs/references/supported-runtimes.md | 104 ++ .../environment/docs/skill-conventions.md | 85 + .../mcp-servers/mock-openshift-mcp.py | 457 +++++ .../environment/mcp-servers/mock-rhoai-mcp.py | 780 +++++++++ .../environment/skills/model-deploy/SKILL.md | 382 ++++ .../model-deploy-preflight-checklist.md | 64 + .../skills/references/common-issues.md | 84 + .../skills/references/live-doc-lookup.md | 106 ++ .../skills/references/skill-conventions.md | 85 + .../instruction.md | 15 + .../solution/solve.sh | 63 + .../rh-ai-engineer__model-deploy/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 94 + .../environment/Dockerfile | 74 + .../environment/docs/common-issues.md | 84 + .../environment/docs/examples/model-deploy.md | 160 ++ .../environment/docs/examples/nim-setup.md | 115 ++ .../environment/docs/live-doc-lookup.md | 106 ++ .../docs/references/known-model-profiles.md | 83 + .../docs/references/supported-runtimes.md | 104 ++ .../environment/docs/skill-conventions.md | 85 + .../mcp-servers/mock-openshift-mcp.py | 540 ++++++ .../environment/mcp-servers/mock-rhoai-mcp.py | 780 +++++++++ .../environment/skills/nim-setup/SKILL.md | 370 ++++ .../skills/references/common-issues.md | 84 + .../skills/references/live-doc-lookup.md | 106 ++ .../skills/references/skill-conventions.md | 85 + .../rh-ai-engineer__nim-setup/instruction.md | 17 + .../solution/solve.sh | 28 + .../rh-ai-engineer__nim-setup/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../rh-ai-engineer__nim-setup/tests/test.sh | 85 + .../tests/test_outputs.py | 89 + .../environment/Dockerfile | 74 + .../environment/docs/common-issues.md | 84 + .../environment/docs/examples/model-deploy.md | 160 ++ .../environment/docs/examples/nim-setup.md | 115 ++ .../environment/docs/live-doc-lookup.md | 106 ++ .../docs/references/known-model-profiles.md | 83 + .../docs/references/supported-runtimes.md | 104 ++ .../environment/docs/skill-conventions.md | 85 + .../mcp-servers/mock-openshift-mcp.py | 529 ++++++ .../environment/mcp-servers/mock-rhoai-mcp.py | 780 +++++++++ .../skills/references/common-issues.md | 84 + .../skills/references/live-doc-lookup.md | 106 ++ .../skills/references/skill-conventions.md | 85 + .../skills/serving-runtime-config/SKILL.md | 278 +++ .../instruction.md | 19 + .../solution/solve.sh | 31 + .../task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 97 ++ .../environment/Dockerfile | 74 + .../environment/docs/common-issues.md | 84 + .../environment/docs/examples/model-deploy.md | 160 ++ .../environment/docs/examples/nim-setup.md | 115 ++ .../environment/docs/live-doc-lookup.md | 106 ++ .../docs/references/known-model-profiles.md | 83 + .../docs/references/supported-runtimes.md | 104 ++ .../environment/docs/skill-conventions.md | 85 + .../mcp-servers/mock-openshift-mcp.py | 457 +++++ .../environment/mcp-servers/mock-rhoai-mcp.py | 866 ++++++++++ .../skills/references/common-issues.md | 84 + .../skills/references/live-doc-lookup.md | 106 ++ .../skills/references/skill-conventions.md | 85 + .../skills/workbench-manage/SKILL.md | 396 +++++ .../instruction.md | 13 + .../solution/solve.sh | 25 + .../task.toml | 26 + .../tests/llm_judge.py | 93 + .../tests/test.sh | 85 + .../tests/test_outputs.py | 73 + .../environment/Dockerfile | 70 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../skills/containerize-deploy/SKILL.md | 477 +++++ .../templates/buildconfig.yaml.template | 38 + .../templates/deployment.yaml.template | 61 + .../templates/helm/Chart.yaml.template | 13 + .../helm/templates/NOTES.txt.template | 32 + .../helm/templates/_helpers.tpl.template | 60 + .../helm/templates/deployment.yaml.template | 61 + .../helm/templates/route.yaml.template | 24 + .../helm/templates/service.yaml.template | 15 + .../templates/helm/values.yaml.template | 67 + .../templates/imagestream.yaml.template | 13 + .../environment/templates/route.yaml.template | 21 + .../templates/service.yaml.template | 20 + .../systemd/systemd-container-rootful.service | 27 + .../systemd-container-rootless.service | 27 + .../templates/systemd/systemd-native.service | 39 + .../instruction.md | 15 + .../solution/solve.sh | 23 + .../task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 110 ++ .../environment/Dockerfile | 70 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 755 ++++++++ .../environment/skills/debug-build/SKILL.md | 315 ++++ .../rh-developer__debug-build/instruction.md | 14 + .../solution/solve.sh | 21 + .../rh-developer__debug-build/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../rh-developer__debug-build/tests/test.sh | 85 + .../tests/test_outputs.py | 77 + .../environment/Dockerfile | 74 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../mcp-servers/mock-podman-mcp.py | 396 +++++ .../skills/debug-container/SKILL.md | 344 ++++ .../instruction.md | 16 + .../solution/solve.sh | 18 + .../rh-developer__debug-container/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 93 + .../environment/Dockerfile | 70 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../environment/skills/debug-network/SKILL.md | 331 ++++ .../instruction.md | 12 + .../solution/solve.sh | 19 + .../rh-developer__debug-network/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../rh-developer__debug-network/tests/test.sh | 85 + .../tests/test_outputs.py | 95 + .../environment/Dockerfile | 70 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../skills/debug-pipeline/SKILL.md | 306 ++++ .../instruction.md | 12 + .../solution/solve.sh | 21 + .../rh-developer__debug-pipeline/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 53 + .../environment/Dockerfile | 70 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../environment/skills/debug-pod/SKILL.md | 260 +++ .../rh-developer__debug-pod/instruction.md | 14 + .../rh-developer__debug-pod/solution/solve.sh | 39 + .../rh-developer__debug-pod/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../rh-developer__debug-pod/tests/test.sh | 85 + .../tests/test_outputs.py | 75 + .../environment/Dockerfile | 74 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../environment/mcp-servers/mock-rhel-mcp.py | 335 ++++ .../environment/skills/debug-rhel/SKILL.md | 455 +++++ .../rh-developer__debug-rhel/instruction.md | 12 + .../solution/solve.sh | 36 + .../rh-developer__debug-rhel/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../rh-developer__debug-rhel/tests/test.sh | 85 + .../tests/test_outputs.py | 97 ++ .../environment/Dockerfile | 70 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../environment/skills/deploy/SKILL.md | 277 +++ .../templates/buildconfig.yaml.template | 38 + .../templates/deployment.yaml.template | 61 + .../templates/helm/Chart.yaml.template | 13 + .../helm/templates/NOTES.txt.template | 32 + .../helm/templates/_helpers.tpl.template | 60 + .../helm/templates/deployment.yaml.template | 61 + .../helm/templates/route.yaml.template | 24 + .../helm/templates/service.yaml.template | 15 + .../templates/helm/values.yaml.template | 67 + .../templates/imagestream.yaml.template | 13 + .../environment/templates/route.yaml.template | 21 + .../templates/service.yaml.template | 20 + .../systemd/systemd-container-rootful.service | 27 + .../systemd-container-rootless.service | 27 + .../templates/systemd/systemd-native.service | 39 + .../rh-developer__deploy/instruction.md | 14 + .../rh-developer__deploy/solution/solve.sh | 61 + .../rh-developer__deploy/task.toml | 26 + .../rh-developer__deploy/tests/llm_judge.py | 108 ++ .../rh-developer__deploy/tests/test.sh | 85 + .../tests/test_outputs.py | 87 + .../environment/Dockerfile | 71 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../sample-project/.s2i/environment | 1 + .../environment/sample-project/Dockerfile | 9 + .../environment/sample-project/app.py | 12 + .../sample-project/requirements.txt | 3 + .../sample-project/tests/test_app.py | 9 + .../skills/detect-project/SKILL.md | 277 +++ .../templates/buildconfig.yaml.template | 38 + .../templates/deployment.yaml.template | 61 + .../templates/helm/Chart.yaml.template | 13 + .../helm/templates/NOTES.txt.template | 32 + .../helm/templates/_helpers.tpl.template | 60 + .../helm/templates/deployment.yaml.template | 61 + .../helm/templates/route.yaml.template | 24 + .../helm/templates/service.yaml.template | 15 + .../templates/helm/values.yaml.template | 67 + .../templates/imagestream.yaml.template | 13 + .../environment/templates/route.yaml.template | 21 + .../templates/service.yaml.template | 20 + .../systemd/systemd-container-rootful.service | 27 + .../systemd-container-rootless.service | 27 + .../templates/systemd/systemd-native.service | 39 + .../instruction.md | 13 + .../solution/solve.sh | 37 + .../rh-developer__detect-project/task.toml | 26 + .../tests/llm_judge.py | 102 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 79 + .../environment/Dockerfile | 74 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../environment/mcp-servers/mock-helm-mcp.py | 231 +++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../environment/skills/helm-deploy/SKILL.md | 356 ++++ .../templates/buildconfig.yaml.template | 38 + .../templates/deployment.yaml.template | 61 + .../templates/helm/Chart.yaml.template | 13 + .../helm/templates/NOTES.txt.template | 32 + .../helm/templates/_helpers.tpl.template | 60 + .../helm/templates/deployment.yaml.template | 61 + .../helm/templates/route.yaml.template | 24 + .../helm/templates/service.yaml.template | 15 + .../templates/helm/values.yaml.template | 67 + .../templates/imagestream.yaml.template | 13 + .../environment/templates/route.yaml.template | 21 + .../templates/service.yaml.template | 20 + .../systemd/systemd-container-rootful.service | 27 + .../systemd-container-rootless.service | 27 + .../templates/systemd/systemd-native.service | 39 + .../rh-developer__helm-deploy/instruction.md | 12 + .../solution/solve.sh | 31 + .../rh-developer__helm-deploy/task.toml | 26 + .../tests/llm_judge.py | 102 ++ .../rh-developer__helm-deploy/tests/test.sh | 85 + .../tests/test_outputs.py | 61 + .../environment/Dockerfile | 70 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../skills/recommend-image/SKILL.md | 282 +++ .../instruction.md | 13 + .../solution/solve.sh | 18 + .../rh-developer__recommend-image/task.toml | 26 + .../tests/llm_judge.py | 102 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 66 + .../environment/Dockerfile | 74 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../mcp-servers/mock-rhel-host-mcp.py | 230 +++ .../environment/skills/rhel-deploy/SKILL.md | 482 ++++++ .../templates/buildconfig.yaml.template | 38 + .../templates/deployment.yaml.template | 61 + .../templates/helm/Chart.yaml.template | 13 + .../helm/templates/NOTES.txt.template | 32 + .../helm/templates/_helpers.tpl.template | 60 + .../helm/templates/deployment.yaml.template | 61 + .../helm/templates/route.yaml.template | 24 + .../helm/templates/service.yaml.template | 15 + .../templates/helm/values.yaml.template | 67 + .../templates/imagestream.yaml.template | 13 + .../environment/templates/route.yaml.template | 21 + .../templates/service.yaml.template | 20 + .../systemd/systemd-container-rootful.service | 27 + .../systemd-container-rootless.service | 27 + .../templates/systemd/systemd-native.service | 39 + .../rh-developer__rhel-deploy/instruction.md | 12 + .../solution/solve.sh | 43 + .../rh-developer__rhel-deploy/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../rh-developer__rhel-deploy/tests/test.sh | 85 + .../tests/test_outputs.py | 98 ++ .../environment/Dockerfile | 70 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../environment/skills/s2i-build/SKILL.md | 391 +++++ .../rh-developer__s2i-build/instruction.md | 12 + .../rh-developer__s2i-build/solution/solve.sh | 60 + .../rh-developer__s2i-build/task.toml | 26 + .../tests/llm_judge.py | 114 ++ .../rh-developer__s2i-build/tests/test.sh | 85 + .../tests/test_outputs.py | 84 + .../environment/Dockerfile | 70 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../skills/validate-environment/SKILL.md | 232 +++ .../instruction.md | 13 + .../solution/solve.sh | 36 + .../task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 67 + .../rh-sre__cve-impact/environment/Dockerfile | 52 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../mcp-servers/mock-lightspeed-mcp.py | 695 ++++++++ .../environment/skills/cve-impact/SKILL.md | 409 +++++ .../cve-impact/flows/01-account-cves.md | 92 + .../cve-impact/flows/02-system-all-cves.md | 89 + .../flows/03-system-remediatable-cves.md | 96 + .../references/01-cve-response-parser.py | 225 +++ .../references/02-cve-parsing-guide.md | 147 ++ .../references/03-output-templates.md | 39 + .../cve-impact/references/04-examples.md | 37 + .../references/05-error-handling.md | 24 + .../skills/mcp-lightspeed-validator/SKILL.md | 61 + .../rh-sre__cve-impact/instruction.md | 14 + .../rh-sre__cve-impact/solution/solve.sh | 15 + .../with_skills/rh-sre__cve-impact/task.toml | 26 + .../rh-sre__cve-impact/tests/llm_judge.py | 94 + .../rh-sre__cve-impact/tests/test.sh | 85 + .../rh-sre__cve-impact/tests/test_outputs.py | 92 + .../environment/Dockerfile | 52 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../mcp-servers/mock-lightspeed-mcp.py | 695 ++++++++ .../skills/cve-validation/SKILL.md | 340 ++++ .../references/01-remediation-indicators.md | 66 + .../references/03-output-template.md | 36 + .../cve-validation/references/04-examples.md | 35 + .../references/05-error-handling.md | 37 + .../skills/mcp-lightspeed-validator/SKILL.md | 61 + .../rh-sre__cve-validation/instruction.md | 12 + .../rh-sre__cve-validation/solution/solve.sh | 14 + .../rh-sre__cve-validation/task.toml | 26 + .../rh-sre__cve-validation/tests/llm_judge.py | 93 + .../rh-sre__cve-validation/tests/test.sh | 85 + .../tests/test_outputs.py | 81 + .../environment/Dockerfile | 52 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../mcp-servers/mock-lightspeed-mcp.py | 695 ++++++++ .../skills/execution-summary/SKILL.md | 357 ++++ .../rh-sre__execution-summary/instruction.md | 15 + .../solution/solve.sh | 13 + .../rh-sre__execution-summary/task.toml | 26 + .../tests/llm_judge.py | 93 + .../rh-sre__execution-summary/tests/test.sh | 85 + .../tests/test_outputs.py | 55 + .../environment/Dockerfile | 52 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../mcp-servers/mock-lightspeed-mcp.py | 695 ++++++++ .../skills/fleet-inventory/SKILL.md | 254 +++ .../references/01-parameter-reference.md | 49 + .../references/03-output-templates.md | 80 + .../fleet-inventory/references/04-examples.md | 32 + .../references/05-error-handling.md | 45 + .../skills/mcp-lightspeed-validator/SKILL.md | 61 + .../rh-sre__fleet-inventory/instruction.md | 17 + .../rh-sre__fleet-inventory/solution/solve.sh | 25 + .../rh-sre__fleet-inventory/task.toml | 26 + .../tests/llm_judge.py | 92 + .../rh-sre__fleet-inventory/tests/test.sh | 85 + .../tests/test_outputs.py | 67 + .../environment/Dockerfile | 56 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../environment/mcp-servers/mock-aap-mcp.py | 1048 +++++++++++ .../mcp-servers/mock-lightspeed-mcp.py | 695 ++++++++ .../skills/job-template-creator/SKILL.md | 321 ++++ .../references/01-git-setup.md | 25 + .../references/02-web-ui-form.md | 24 + .../references/03-output-template.md | 20 + .../references/04-examples.md | 19 + .../skills/mcp-aap-validator/SKILL.md | 66 + .../skills/playbook-executor/SKILL.md | 499 ++++++ .../01-execution-report-templates.md | 168 ++ .../references/02-error-handling-guide.md | 108 ++ .../references/03-workflow-examples.md | 119 ++ .../04-dry-run-display-templates.md | 93 + .../references/05-git-flow-prompts.md | 97 ++ .../instruction.md | 17 + .../solution/solve.sh | 19 + .../rh-sre__job-template-creator/task.toml | 26 + .../tests/llm_judge.py | 93 + .../tests/test.sh | 85 + .../tests/test_outputs.py | 98 ++ .../environment/Dockerfile | 56 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../environment/mcp-servers/mock-aap-mcp.py | 1048 +++++++++++ .../mcp-servers/mock-lightspeed-mcp.py | 695 ++++++++ .../skills/job-template-creator/SKILL.md | 321 ++++ .../references/01-git-setup.md | 25 + .../references/02-web-ui-form.md | 24 + .../references/03-output-template.md | 20 + .../references/04-examples.md | 19 + .../SKILL.md | 414 +++++ .../skills/mcp-aap-validator/SKILL.md | 66 + .../skills/playbook-executor/SKILL.md | 499 ++++++ .../01-execution-report-templates.md | 168 ++ .../references/02-error-handling-guide.md | 108 ++ .../references/03-workflow-examples.md | 119 ++ .../04-dry-run-display-templates.md | 93 + .../references/05-git-flow-prompts.md | 97 ++ .../instruction.md | 18 + .../solution/solve.sh | 21 + .../task.toml | 26 + .../tests/llm_judge.py | 93 + .../tests/test.sh | 85 + .../tests/test_outputs.py | 63 + .../environment/Dockerfile | 56 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../environment/mcp-servers/mock-aap-mcp.py | 1048 +++++++++++ .../mcp-servers/mock-lightspeed-mcp.py | 695 ++++++++ .../skills/mcp-aap-validator/SKILL.md | 66 + .../rh-sre__mcp-aap-validator/instruction.md | 16 + .../solution/solve.sh | 25 + .../rh-sre__mcp-aap-validator/task.toml | 26 + .../tests/llm_judge.py | 93 + .../rh-sre__mcp-aap-validator/tests/test.sh | 85 + .../tests/test_outputs.py | 66 + .../environment/Dockerfile | 52 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../mcp-servers/mock-lightspeed-mcp.py | 695 ++++++++ .../skills/mcp-lightspeed-validator/SKILL.md | 61 + .../instruction.md | 16 + .../solution/solve.sh | 29 + .../task.toml | 26 + .../tests/llm_judge.py | 93 + .../tests/test.sh | 85 + .../tests/test_outputs.py | 64 + .../environment/Dockerfile | 56 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../environment/mcp-servers/mock-aap-mcp.py | 1048 +++++++++++ .../mcp-servers/mock-lightspeed-mcp.py | 695 ++++++++ .../skills/mcp-aap-validator/SKILL.md | 66 + .../skills/playbook-executor/SKILL.md | 499 ++++++ .../01-execution-report-templates.md | 168 ++ .../references/02-error-handling-guide.md | 108 ++ .../references/03-workflow-examples.md | 119 ++ .../04-dry-run-display-templates.md | 93 + .../references/05-git-flow-prompts.md | 97 ++ .../rh-sre__playbook-executor/instruction.md | 18 + .../solution/solve.sh | 21 + .../rh-sre__playbook-executor/task.toml | 26 + .../tests/llm_judge.py | 93 + .../rh-sre__playbook-executor/tests/test.sh | 85 + .../tests/test_outputs.py | 89 + .../environment/Dockerfile | 52 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../mcp-servers/mock-lightspeed-mcp.py | 722 ++++++++ .../skills/playbook-generator/SKILL.md | 377 ++++ .../rh-sre__playbook-generator/instruction.md | 17 + .../solution/solve.sh | 38 + .../rh-sre__playbook-generator/task.toml | 26 + .../tests/llm_judge.py | 93 + .../rh-sre__playbook-generator/tests/test.sh | 85 + .../tests/test_outputs.py | 74 + .../environment/Dockerfile | 52 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../mcp-servers/mock-lightspeed-mcp.py | 759 ++++++++ .../skills/remediation-verifier/SKILL.md | 399 +++++ .../instruction.md | 18 + .../solution/solve.sh | 20 + .../rh-sre__remediation-verifier/task.toml | 26 + .../tests/llm_judge.py | 93 + .../tests/test.sh | 85 + .../tests/test_outputs.py | 75 + .../environment/Dockerfile | 52 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../mcp-servers/mock-lightspeed-mcp.py | 722 ++++++++ .../skills/cve-validation/SKILL.md | 340 ++++ .../references/01-remediation-indicators.md | 66 + .../references/03-output-template.md | 36 + .../cve-validation/references/04-examples.md | 35 + .../references/05-error-handling.md | 37 + .../environment/skills/remediation/SKILL.md | 279 +++ .../01-remediation-plan-template.md | 85 + .../rh-sre__remediation/instruction.md | 19 + .../rh-sre__remediation/solution/solve.sh | 21 + .../with_skills/rh-sre__remediation/task.toml | 26 + .../rh-sre__remediation/tests/llm_judge.py | 93 + .../rh-sre__remediation/tests/test.sh | 85 + .../rh-sre__remediation/tests/test_outputs.py | 78 + .../environment/Dockerfile | 52 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../mcp-servers/mock-lightspeed-mcp.py | 759 ++++++++ .../skills/system-context/SKILL.md | 488 ++++++ .../rh-sre__system-context/instruction.md | 16 + .../rh-sre__system-context/solution/solve.sh | 19 + .../rh-sre__system-context/task.toml | 26 + .../rh-sre__system-context/tests/llm_judge.py | 93 + .../rh-sre__system-context/tests/test.sh | 85 + .../tests/test_outputs.py | 84 + .../rh-virt__vm-clone/environment/Dockerfile | 70 + .../.ai-index/semantic-index.json | 148 ++ .../environment/docs/troubleshooting/INDEX.md | 332 ++++ .../docs/troubleshooting/lifecycle-errors.md | 869 ++++++++++ .../docs/troubleshooting/network-errors.md | 429 +++++ .../docs/troubleshooting/runtime-errors.md | 616 +++++++ .../docs/troubleshooting/scheduling-errors.md | 417 +++++ .../docs/troubleshooting/storage-errors.md | 1011 +++++++++++ .../environment/mcp-servers/mock-virt-mcp.py | 1465 ++++++++++++++++ .../environment/skills/vm-clone/SKILL.md | 456 +++++ .../rh-virt__vm-clone/instruction.md | 13 + .../rh-virt__vm-clone/solution/solve.sh | 34 + .../with_skills/rh-virt__vm-clone/task.toml | 26 + .../rh-virt__vm-clone/tests/llm_judge.py | 93 + .../rh-virt__vm-clone/tests/test.sh | 85 + .../rh-virt__vm-clone/tests/test_outputs.py | 90 + .../rh-virt__vm-create/environment/Dockerfile | 70 + .../.ai-index/semantic-index.json | 148 ++ .../environment/docs/troubleshooting/INDEX.md | 332 ++++ .../docs/troubleshooting/lifecycle-errors.md | 869 ++++++++++ .../docs/troubleshooting/network-errors.md | 429 +++++ .../docs/troubleshooting/runtime-errors.md | 616 +++++++ .../docs/troubleshooting/scheduling-errors.md | 417 +++++ .../docs/troubleshooting/storage-errors.md | 1011 +++++++++++ .../environment/mcp-servers/mock-virt-mcp.py | 1518 ++++++++++++++++ .../environment/skills/vm-create/SKILL.md | 403 +++++ .../rh-virt__vm-create/instruction.md | 14 + .../rh-virt__vm-create/solution/solve.sh | 71 + .../with_skills/rh-virt__vm-create/task.toml | 26 + .../rh-virt__vm-create/tests/llm_judge.py | 92 + .../rh-virt__vm-create/tests/test.sh | 85 + .../rh-virt__vm-create/tests/test_outputs.py | 71 + .../rh-virt__vm-delete/environment/Dockerfile | 70 + .../.ai-index/semantic-index.json | 148 ++ .../environment/docs/troubleshooting/INDEX.md | 332 ++++ .../docs/troubleshooting/lifecycle-errors.md | 869 ++++++++++ .../docs/troubleshooting/network-errors.md | 429 +++++ .../docs/troubleshooting/runtime-errors.md | 616 +++++++ .../docs/troubleshooting/scheduling-errors.md | 417 +++++ .../docs/troubleshooting/storage-errors.md | 1011 +++++++++++ .../environment/mcp-servers/mock-virt-mcp.py | 1464 ++++++++++++++++ .../environment/skills/vm-delete/SKILL.md | 403 +++++ .../rh-virt__vm-delete/instruction.md | 12 + .../rh-virt__vm-delete/solution/solve.sh | 31 + .../with_skills/rh-virt__vm-delete/task.toml | 26 + .../rh-virt__vm-delete/tests/llm_judge.py | 93 + .../rh-virt__vm-delete/tests/test.sh | 85 + .../rh-virt__vm-delete/tests/test_outputs.py | 82 + .../environment/Dockerfile | 70 + .../.ai-index/semantic-index.json | 148 ++ .../environment/docs/troubleshooting/INDEX.md | 332 ++++ .../docs/troubleshooting/lifecycle-errors.md | 869 ++++++++++ .../docs/troubleshooting/network-errors.md | 429 +++++ .../docs/troubleshooting/runtime-errors.md | 616 +++++++ .../docs/troubleshooting/scheduling-errors.md | 417 +++++ .../docs/troubleshooting/storage-errors.md | 1011 +++++++++++ .../environment/mcp-servers/mock-virt-mcp.py | 1458 ++++++++++++++++ .../environment/skills/vm-inventory/SKILL.md | 390 +++++ .../rh-virt__vm-inventory/instruction.md | 14 + .../rh-virt__vm-inventory/solution/solve.sh | 32 + .../rh-virt__vm-inventory/task.toml | 26 + .../rh-virt__vm-inventory/tests/llm_judge.py | 92 + .../rh-virt__vm-inventory/tests/test.sh | 85 + .../tests/test_outputs.py | 67 + .../environment/Dockerfile | 70 + .../.ai-index/semantic-index.json | 148 ++ .../environment/docs/troubleshooting/INDEX.md | 332 ++++ .../docs/troubleshooting/lifecycle-errors.md | 869 ++++++++++ .../docs/troubleshooting/network-errors.md | 429 +++++ .../docs/troubleshooting/runtime-errors.md | 616 +++++++ .../docs/troubleshooting/scheduling-errors.md | 417 +++++ .../docs/troubleshooting/storage-errors.md | 1011 +++++++++++ .../environment/mcp-servers/mock-virt-mcp.py | 1467 ++++++++++++++++ .../skills/vm-lifecycle-manager/SKILL.md | 308 ++++ .../instruction.md | 12 + .../solution/solve.sh | 31 + .../rh-virt__vm-lifecycle-manager/task.toml | 26 + .../tests/llm_judge.py | 93 + .../tests/test.sh | 85 + .../tests/test_outputs.py | 75 + .../environment/Dockerfile | 70 + .../.ai-index/semantic-index.json | 148 ++ .../environment/docs/troubleshooting/INDEX.md | 332 ++++ .../docs/troubleshooting/lifecycle-errors.md | 869 ++++++++++ .../docs/troubleshooting/network-errors.md | 429 +++++ .../docs/troubleshooting/runtime-errors.md | 616 +++++++ .../docs/troubleshooting/scheduling-errors.md | 417 +++++ .../docs/troubleshooting/storage-errors.md | 1011 +++++++++++ .../environment/mcp-servers/mock-virt-mcp.py | 1458 ++++++++++++++++ .../vm-rebalance/REBALANCE_AUTOMATIC.md | 760 ++++++++ .../skills/vm-rebalance/REBALANCE_MANUAL.md | 848 +++++++++ .../environment/skills/vm-rebalance/SKILL.md | 391 +++++ .../vm-rebalance/references/anti-patterns.md | 869 ++++++++++ .../live-migration-best-practices.md | 794 +++++++++ .../references/performance-tuning.md | 719 ++++++++ .../references/production-considerations.md | 868 ++++++++++ .../rh-virt__vm-rebalance/instruction.md | 13 + .../rh-virt__vm-rebalance/solution/solve.sh | 41 + .../rh-virt__vm-rebalance/task.toml | 26 + .../rh-virt__vm-rebalance/tests/llm_judge.py | 92 + .../rh-virt__vm-rebalance/tests/test.sh | 85 + .../tests/test_outputs.py | 57 + .../environment/Dockerfile | 70 + .../.ai-index/semantic-index.json | 148 ++ .../environment/docs/troubleshooting/INDEX.md | 332 ++++ .../docs/troubleshooting/lifecycle-errors.md | 869 ++++++++++ .../docs/troubleshooting/network-errors.md | 429 +++++ .../docs/troubleshooting/runtime-errors.md | 616 +++++++ .../docs/troubleshooting/scheduling-errors.md | 417 +++++ .../docs/troubleshooting/storage-errors.md | 1011 +++++++++++ .../environment/mcp-servers/mock-virt-mcp.py | 1539 +++++++++++++++++ .../skills/vm-snapshot-create/SKILL.md | 423 +++++ .../instruction.md | 12 + .../solution/solve.sh | 39 + .../rh-virt__vm-snapshot-create/task.toml | 26 + .../tests/llm_judge.py | 93 + .../rh-virt__vm-snapshot-create/tests/test.sh | 85 + .../tests/test_outputs.py | 77 + .../environment/Dockerfile | 70 + .../.ai-index/semantic-index.json | 148 ++ .../environment/docs/troubleshooting/INDEX.md | 332 ++++ .../docs/troubleshooting/lifecycle-errors.md | 869 ++++++++++ .../docs/troubleshooting/network-errors.md | 429 +++++ .../docs/troubleshooting/runtime-errors.md | 616 +++++++ .../docs/troubleshooting/scheduling-errors.md | 417 +++++ .../docs/troubleshooting/storage-errors.md | 1011 +++++++++++ .../environment/mcp-servers/mock-virt-mcp.py | 1458 ++++++++++++++++ .../skills/vm-snapshot-delete/SKILL.md | 447 +++++ .../instruction.md | 12 + .../solution/solve.sh | 26 + .../rh-virt__vm-snapshot-delete/task.toml | 26 + .../tests/llm_judge.py | 93 + .../rh-virt__vm-snapshot-delete/tests/test.sh | 85 + .../tests/test_outputs.py | 71 + .../environment/Dockerfile | 70 + .../.ai-index/semantic-index.json | 148 ++ .../environment/docs/troubleshooting/INDEX.md | 332 ++++ .../docs/troubleshooting/lifecycle-errors.md | 869 ++++++++++ .../docs/troubleshooting/network-errors.md | 429 +++++ .../docs/troubleshooting/runtime-errors.md | 616 +++++++ .../docs/troubleshooting/scheduling-errors.md | 417 +++++ .../docs/troubleshooting/storage-errors.md | 1011 +++++++++++ .../environment/mcp-servers/mock-virt-mcp.py | 1500 ++++++++++++++++ .../skills/vm-snapshot-list/SKILL.md | 402 +++++ .../rh-virt__vm-snapshot-list/instruction.md | 12 + .../solution/solve.sh | 30 + .../rh-virt__vm-snapshot-list/task.toml | 26 + .../tests/llm_judge.py | 93 + .../rh-virt__vm-snapshot-list/tests/test.sh | 85 + .../tests/test_outputs.py | 62 + .../environment/Dockerfile | 70 + .../.ai-index/semantic-index.json | 148 ++ .../environment/docs/troubleshooting/INDEX.md | 332 ++++ .../docs/troubleshooting/lifecycle-errors.md | 869 ++++++++++ .../docs/troubleshooting/network-errors.md | 429 +++++ .../docs/troubleshooting/runtime-errors.md | 616 +++++++ .../docs/troubleshooting/scheduling-errors.md | 417 +++++ .../docs/troubleshooting/storage-errors.md | 1011 +++++++++++ .../environment/mcp-servers/mock-virt-mcp.py | 1458 ++++++++++++++++ .../skills/vm-snapshot-restore/SKILL.md | 495 ++++++ .../instruction.md | 12 + .../solution/solve.sh | 42 + .../rh-virt__vm-snapshot-restore/task.toml | 26 + .../tests/llm_judge.py | 93 + .../tests/test.sh | 85 + .../tests/test_outputs.py | 71 + .../environment/Dockerfile | 63 + .../environment/docs/multi-cluster-auth.md | 248 +++ .../environment/mcp-servers/mock-ocp-mcp.py | 304 ++++ .../ocp-admin__cluster-report/instruction.md | 17 + .../solution/solve.sh | 30 + .../ocp-admin__cluster-report/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../ocp-admin__cluster-report/tests/test.sh | 85 + .../tests/test_outputs.py | 105 ++ .../environment/Dockerfile | 71 + .../environment/docs/common-issues.md | 84 + .../environment/docs/examples/model-deploy.md | 160 ++ .../environment/docs/examples/nim-setup.md | 115 ++ .../environment/docs/live-doc-lookup.md | 106 ++ .../docs/references/known-model-profiles.md | 83 + .../docs/references/supported-runtimes.md | 104 ++ .../environment/docs/skill-conventions.md | 85 + .../mcp-servers/mock-observability-mcp.py | 260 +++ .../mcp-servers/mock-openshift-mcp.py | 457 +++++ .../environment/mcp-servers/mock-rhoai-mcp.py | 780 +++++++++ .../instruction.md | 13 + .../solution/solve.sh | 23 + .../task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 91 + .../environment/Dockerfile | 67 + .../environment/docs/common-issues.md | 84 + .../environment/docs/examples/model-deploy.md | 160 ++ .../environment/docs/examples/nim-setup.md | 115 ++ .../environment/docs/live-doc-lookup.md | 106 ++ .../docs/references/known-model-profiles.md | 83 + .../docs/references/supported-runtimes.md | 104 ++ .../environment/docs/skill-conventions.md | 85 + .../mcp-servers/mock-openshift-mcp.py | 457 +++++ .../environment/mcp-servers/mock-rhoai-mcp.py | 780 +++++++++ .../instruction.md | 13 + .../solution/solve.sh | 36 + .../rh-ai-engineer__debug-inference/task.toml | 26 + .../tests/llm_judge.py | 114 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 98 ++ .../environment/Dockerfile | 67 + .../environment/docs/common-issues.md | 84 + .../environment/docs/examples/model-deploy.md | 160 ++ .../environment/docs/examples/nim-setup.md | 115 ++ .../environment/docs/live-doc-lookup.md | 106 ++ .../docs/references/known-model-profiles.md | 83 + .../docs/references/supported-runtimes.md | 104 ++ .../environment/docs/skill-conventions.md | 85 + .../mcp-servers/mock-openshift-mcp.py | 457 +++++ .../environment/mcp-servers/mock-rhoai-mcp.py | 796 +++++++++ .../instruction.md | 20 + .../solution/solve.sh | 32 + .../task.toml | 26 + .../tests/llm_judge.py | 102 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 113 ++ .../environment/Dockerfile | 67 + .../environment/docs/common-issues.md | 84 + .../environment/docs/examples/model-deploy.md | 160 ++ .../environment/docs/examples/nim-setup.md | 115 ++ .../environment/docs/live-doc-lookup.md | 106 ++ .../docs/references/known-model-profiles.md | 83 + .../docs/references/supported-runtimes.md | 104 ++ .../environment/docs/skill-conventions.md | 85 + .../mcp-servers/mock-openshift-mcp.py | 457 +++++ .../environment/mcp-servers/mock-rhoai-mcp.py | 780 +++++++++ .../instruction.md | 15 + .../solution/solve.sh | 63 + .../rh-ai-engineer__model-deploy/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 94 + .../environment/Dockerfile | 67 + .../environment/docs/common-issues.md | 84 + .../environment/docs/examples/model-deploy.md | 160 ++ .../environment/docs/examples/nim-setup.md | 115 ++ .../environment/docs/live-doc-lookup.md | 106 ++ .../docs/references/known-model-profiles.md | 83 + .../docs/references/supported-runtimes.md | 104 ++ .../environment/docs/skill-conventions.md | 85 + .../mcp-servers/mock-openshift-mcp.py | 540 ++++++ .../environment/mcp-servers/mock-rhoai-mcp.py | 780 +++++++++ .../rh-ai-engineer__nim-setup/instruction.md | 17 + .../solution/solve.sh | 28 + .../rh-ai-engineer__nim-setup/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../rh-ai-engineer__nim-setup/tests/test.sh | 85 + .../tests/test_outputs.py | 89 + .../environment/Dockerfile | 67 + .../environment/docs/common-issues.md | 84 + .../environment/docs/examples/model-deploy.md | 160 ++ .../environment/docs/examples/nim-setup.md | 115 ++ .../environment/docs/live-doc-lookup.md | 106 ++ .../docs/references/known-model-profiles.md | 83 + .../docs/references/supported-runtimes.md | 104 ++ .../environment/docs/skill-conventions.md | 85 + .../mcp-servers/mock-openshift-mcp.py | 529 ++++++ .../environment/mcp-servers/mock-rhoai-mcp.py | 780 +++++++++ .../instruction.md | 19 + .../solution/solve.sh | 31 + .../task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 97 ++ .../environment/Dockerfile | 67 + .../environment/docs/common-issues.md | 84 + .../environment/docs/examples/model-deploy.md | 160 ++ .../environment/docs/examples/nim-setup.md | 115 ++ .../environment/docs/live-doc-lookup.md | 106 ++ .../docs/references/known-model-profiles.md | 83 + .../docs/references/supported-runtimes.md | 104 ++ .../environment/docs/skill-conventions.md | 85 + .../mcp-servers/mock-openshift-mcp.py | 457 +++++ .../environment/mcp-servers/mock-rhoai-mcp.py | 866 ++++++++++ .../instruction.md | 13 + .../solution/solve.sh | 25 + .../task.toml | 26 + .../tests/llm_judge.py | 93 + .../tests/test.sh | 85 + .../tests/test_outputs.py | 73 + .../environment/Dockerfile | 63 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../instruction.md | 15 + .../solution/solve.sh | 23 + .../task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 110 ++ .../environment/Dockerfile | 63 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 755 ++++++++ .../rh-developer__debug-build/instruction.md | 14 + .../solution/solve.sh | 21 + .../rh-developer__debug-build/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../rh-developer__debug-build/tests/test.sh | 85 + .../tests/test_outputs.py | 77 + .../environment/Dockerfile | 67 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../mcp-servers/mock-podman-mcp.py | 396 +++++ .../instruction.md | 16 + .../solution/solve.sh | 18 + .../rh-developer__debug-container/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 93 + .../environment/Dockerfile | 63 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../instruction.md | 12 + .../solution/solve.sh | 19 + .../rh-developer__debug-network/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../rh-developer__debug-network/tests/test.sh | 85 + .../tests/test_outputs.py | 95 + .../environment/Dockerfile | 63 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../instruction.md | 12 + .../solution/solve.sh | 21 + .../rh-developer__debug-pipeline/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 53 + .../environment/Dockerfile | 63 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../rh-developer__debug-pod/instruction.md | 14 + .../rh-developer__debug-pod/solution/solve.sh | 39 + .../rh-developer__debug-pod/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../rh-developer__debug-pod/tests/test.sh | 85 + .../tests/test_outputs.py | 75 + .../environment/Dockerfile | 67 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../environment/mcp-servers/mock-rhel-mcp.py | 335 ++++ .../rh-developer__debug-rhel/instruction.md | 12 + .../solution/solve.sh | 36 + .../rh-developer__debug-rhel/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../rh-developer__debug-rhel/tests/test.sh | 85 + .../tests/test_outputs.py | 97 ++ .../environment/Dockerfile | 63 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../rh-developer__deploy/instruction.md | 14 + .../rh-developer__deploy/solution/solve.sh | 61 + .../rh-developer__deploy/task.toml | 26 + .../rh-developer__deploy/tests/llm_judge.py | 108 ++ .../rh-developer__deploy/tests/test.sh | 85 + .../tests/test_outputs.py | 87 + .../environment/Dockerfile | 64 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../sample-project/.s2i/environment | 1 + .../environment/sample-project/Dockerfile | 9 + .../environment/sample-project/app.py | 12 + .../sample-project/requirements.txt | 3 + .../sample-project/tests/test_app.py | 9 + .../templates/buildconfig.yaml.template | 38 + .../templates/deployment.yaml.template | 61 + .../templates/helm/Chart.yaml.template | 13 + .../helm/templates/NOTES.txt.template | 32 + .../helm/templates/_helpers.tpl.template | 60 + .../helm/templates/deployment.yaml.template | 61 + .../helm/templates/route.yaml.template | 24 + .../helm/templates/service.yaml.template | 15 + .../templates/helm/values.yaml.template | 67 + .../templates/imagestream.yaml.template | 13 + .../environment/templates/route.yaml.template | 21 + .../templates/service.yaml.template | 20 + .../systemd/systemd-container-rootful.service | 27 + .../systemd-container-rootless.service | 27 + .../templates/systemd/systemd-native.service | 39 + .../instruction.md | 13 + .../solution/solve.sh | 37 + .../rh-developer__detect-project/task.toml | 26 + .../tests/llm_judge.py | 102 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 79 + .../environment/Dockerfile | 67 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../environment/mcp-servers/mock-helm-mcp.py | 231 +++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../templates/buildconfig.yaml.template | 38 + .../templates/deployment.yaml.template | 61 + .../templates/helm/Chart.yaml.template | 13 + .../helm/templates/NOTES.txt.template | 32 + .../helm/templates/_helpers.tpl.template | 60 + .../helm/templates/deployment.yaml.template | 61 + .../helm/templates/route.yaml.template | 24 + .../helm/templates/service.yaml.template | 15 + .../templates/helm/values.yaml.template | 67 + .../templates/imagestream.yaml.template | 13 + .../environment/templates/route.yaml.template | 21 + .../templates/service.yaml.template | 20 + .../systemd/systemd-container-rootful.service | 27 + .../systemd-container-rootless.service | 27 + .../templates/systemd/systemd-native.service | 39 + .../rh-developer__helm-deploy/instruction.md | 12 + .../solution/solve.sh | 31 + .../rh-developer__helm-deploy/task.toml | 26 + .../tests/llm_judge.py | 102 ++ .../rh-developer__helm-deploy/tests/test.sh | 85 + .../tests/test_outputs.py | 61 + .../environment/Dockerfile | 63 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../instruction.md | 13 + .../solution/solve.sh | 18 + .../rh-developer__recommend-image/task.toml | 26 + .../tests/llm_judge.py | 102 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 66 + .../environment/Dockerfile | 67 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../mcp-servers/mock-rhel-host-mcp.py | 230 +++ .../templates/buildconfig.yaml.template | 38 + .../templates/deployment.yaml.template | 61 + .../templates/helm/Chart.yaml.template | 13 + .../helm/templates/NOTES.txt.template | 32 + .../helm/templates/_helpers.tpl.template | 60 + .../helm/templates/deployment.yaml.template | 61 + .../helm/templates/route.yaml.template | 24 + .../helm/templates/service.yaml.template | 15 + .../templates/helm/values.yaml.template | 67 + .../templates/imagestream.yaml.template | 13 + .../environment/templates/route.yaml.template | 21 + .../templates/service.yaml.template | 20 + .../systemd/systemd-container-rootful.service | 27 + .../systemd-container-rootless.service | 27 + .../templates/systemd/systemd-native.service | 39 + .../rh-developer__rhel-deploy/instruction.md | 12 + .../solution/solve.sh | 43 + .../rh-developer__rhel-deploy/task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../rh-developer__rhel-deploy/tests/test.sh | 85 + .../tests/test_outputs.py | 98 ++ .../environment/Dockerfile | 63 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../rh-developer__s2i-build/instruction.md | 12 + .../rh-developer__s2i-build/solution/solve.sh | 60 + .../rh-developer__s2i-build/task.toml | 26 + .../tests/llm_judge.py | 114 ++ .../rh-developer__s2i-build/tests/test.sh | 85 + .../tests/test_outputs.py | 84 + .../environment/Dockerfile | 63 + .../environment/docs/builder-images.md | 308 ++++ .../environment/docs/debugging-patterns.md | 478 +++++ .../environment/docs/dynamic-validation.md | 259 +++ .../environment/docs/human-in-the-loop.md | 98 ++ .../docs/image-selection-criteria.md | 221 +++ .../environment/docs/prerequisites.md | 212 +++ .../docs/python-s2i-entrypoints.md | 70 + .../environment/docs/rhel-deployment.md | 580 +++++++ .../docs/selinux-troubleshooting.md | 387 +++++ .../mcp-servers/mock-openshift-mcp.py | 717 ++++++++ .../instruction.md | 13 + .../solution/solve.sh | 36 + .../task.toml | 26 + .../tests/llm_judge.py | 108 ++ .../tests/test.sh | 85 + .../tests/test_outputs.py | 67 + .../rh-sre__cve-impact/environment/Dockerfile | 43 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../mcp-servers/mock-lightspeed-mcp.py | 695 ++++++++ .../rh-sre__cve-impact/instruction.md | 14 + .../rh-sre__cve-impact/solution/solve.sh | 15 + .../rh-sre__cve-impact/task.toml | 26 + .../rh-sre__cve-impact/tests/llm_judge.py | 94 + .../rh-sre__cve-impact/tests/test.sh | 85 + .../rh-sre__cve-impact/tests/test_outputs.py | 92 + .../environment/Dockerfile | 43 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../mcp-servers/mock-lightspeed-mcp.py | 695 ++++++++ .../rh-sre__cve-validation/instruction.md | 12 + .../rh-sre__cve-validation/solution/solve.sh | 14 + .../rh-sre__cve-validation/task.toml | 26 + .../rh-sre__cve-validation/tests/llm_judge.py | 93 + .../rh-sre__cve-validation/tests/test.sh | 85 + .../tests/test_outputs.py | 81 + .../environment/Dockerfile | 43 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../mcp-servers/mock-lightspeed-mcp.py | 695 ++++++++ .../rh-sre__execution-summary/instruction.md | 15 + .../solution/solve.sh | 13 + .../rh-sre__execution-summary/task.toml | 26 + .../tests/llm_judge.py | 93 + .../rh-sre__execution-summary/tests/test.sh | 85 + .../tests/test_outputs.py | 55 + .../environment/Dockerfile | 43 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../mcp-servers/mock-lightspeed-mcp.py | 695 ++++++++ .../rh-sre__fleet-inventory/instruction.md | 17 + .../rh-sre__fleet-inventory/solution/solve.sh | 25 + .../rh-sre__fleet-inventory/task.toml | 26 + .../tests/llm_judge.py | 92 + .../rh-sre__fleet-inventory/tests/test.sh | 85 + .../tests/test_outputs.py | 67 + .../environment/Dockerfile | 47 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../environment/mcp-servers/mock-aap-mcp.py | 1048 +++++++++++ .../mcp-servers/mock-lightspeed-mcp.py | 695 ++++++++ .../instruction.md | 17 + .../solution/solve.sh | 19 + .../rh-sre__job-template-creator/task.toml | 26 + .../tests/llm_judge.py | 93 + .../tests/test.sh | 85 + .../tests/test_outputs.py | 98 ++ .../environment/Dockerfile | 47 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../environment/mcp-servers/mock-aap-mcp.py | 1048 +++++++++++ .../mcp-servers/mock-lightspeed-mcp.py | 695 ++++++++ .../instruction.md | 18 + .../solution/solve.sh | 21 + .../task.toml | 26 + .../tests/llm_judge.py | 93 + .../tests/test.sh | 85 + .../tests/test_outputs.py | 63 + .../environment/Dockerfile | 47 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../environment/mcp-servers/mock-aap-mcp.py | 1048 +++++++++++ .../mcp-servers/mock-lightspeed-mcp.py | 695 ++++++++ .../rh-sre__mcp-aap-validator/instruction.md | 16 + .../solution/solve.sh | 25 + .../rh-sre__mcp-aap-validator/task.toml | 26 + .../tests/llm_judge.py | 93 + .../rh-sre__mcp-aap-validator/tests/test.sh | 85 + .../tests/test_outputs.py | 66 + .../environment/Dockerfile | 43 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../mcp-servers/mock-lightspeed-mcp.py | 695 ++++++++ .../instruction.md | 16 + .../solution/solve.sh | 29 + .../task.toml | 26 + .../tests/llm_judge.py | 93 + .../tests/test.sh | 85 + .../tests/test_outputs.py | 64 + .../environment/Dockerfile | 47 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../environment/mcp-servers/mock-aap-mcp.py | 1048 +++++++++++ .../mcp-servers/mock-lightspeed-mcp.py | 695 ++++++++ .../rh-sre__playbook-executor/instruction.md | 18 + .../solution/solve.sh | 21 + .../rh-sre__playbook-executor/task.toml | 26 + .../tests/llm_judge.py | 93 + .../rh-sre__playbook-executor/tests/test.sh | 85 + .../tests/test_outputs.py | 89 + .../environment/Dockerfile | 43 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../mcp-servers/mock-lightspeed-mcp.py | 722 ++++++++ .../rh-sre__playbook-generator/instruction.md | 17 + .../solution/solve.sh | 38 + .../rh-sre__playbook-generator/task.toml | 26 + .../tests/llm_judge.py | 93 + .../rh-sre__playbook-generator/tests/test.sh | 85 + .../tests/test_outputs.py | 74 + .../environment/Dockerfile | 43 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../mcp-servers/mock-lightspeed-mcp.py | 759 ++++++++ .../instruction.md | 18 + .../solution/solve.sh | 20 + .../rh-sre__remediation-verifier/task.toml | 26 + .../tests/llm_judge.py | 93 + .../tests/test.sh | 85 + .../tests/test_outputs.py | 75 + .../environment/Dockerfile | 43 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../mcp-servers/mock-lightspeed-mcp.py | 722 ++++++++ .../rh-sre__remediation/instruction.md | 19 + .../rh-sre__remediation/solution/solve.sh | 21 + .../rh-sre__remediation/task.toml | 26 + .../rh-sre__remediation/tests/llm_judge.py | 93 + .../rh-sre__remediation/tests/test.sh | 85 + .../rh-sre__remediation/tests/test_outputs.py | 78 + .../environment/Dockerfile | 43 + .../docs/.ai-index/cross-reference-graph.json | 147 ++ .../docs/.ai-index/semantic-index.json | 297 ++++ .../docs/.ai-index/task-to-docs-mapping.json | 230 +++ .../environment/docs/INDEX.md | 389 +++++ .../environment/docs/SOURCES.md | 107 ++ .../environment/docs/ansible/README.md | 50 + .../docs/ansible/aap-job-execution.md | 532 ++++++ .../docs/ansible/cve-remediation-templates.md | 1500 ++++++++++++++++ .../docs/ansible/playbook-integration-aap.md | 667 +++++++ .../environment/docs/insights/README.md | 38 + .../docs/insights/vulnerability-logic.md | 568 ++++++ .../environment/docs/references/README.md | 39 + .../docs/references/cvss-scoring.md | 636 +++++++ .../references/lightspeed-mcp-parameters.md | 89 + .../lightspeed-mcp-tool-failures.md | 69 + .../docs/references/skill-invocation.md | 35 + .../environment/docs/rhel/README.md | 40 + .../docs/rhel/package-management.md | 738 ++++++++ .../testing/aap-integration-test-guide.md | 649 +++++++ .../mcp-servers/mock-lightspeed-mcp.py | 759 ++++++++ .../rh-sre__system-context/instruction.md | 16 + .../rh-sre__system-context/solution/solve.sh | 19 + .../rh-sre__system-context/task.toml | 26 + .../rh-sre__system-context/tests/llm_judge.py | 93 + .../rh-sre__system-context/tests/test.sh | 85 + .../tests/test_outputs.py | 84 + .../rh-virt__vm-clone/environment/Dockerfile | 50 + .../environment/mcp-servers/mock-virt-mcp.py | 1465 ++++++++++++++++ .../rh-virt__vm-clone/instruction.md | 13 + .../rh-virt__vm-clone/solution/solve.sh | 34 + .../rh-virt__vm-clone/task.toml | 26 + .../rh-virt__vm-clone/tests/llm_judge.py | 93 + .../rh-virt__vm-clone/tests/test.sh | 85 + .../rh-virt__vm-clone/tests/test_outputs.py | 90 + .../rh-virt__vm-create/environment/Dockerfile | 63 + .../.ai-index/semantic-index.json | 148 ++ .../environment/docs/troubleshooting/INDEX.md | 332 ++++ .../docs/troubleshooting/lifecycle-errors.md | 869 ++++++++++ .../docs/troubleshooting/network-errors.md | 429 +++++ .../docs/troubleshooting/runtime-errors.md | 616 +++++++ .../docs/troubleshooting/scheduling-errors.md | 417 +++++ .../docs/troubleshooting/storage-errors.md | 1011 +++++++++++ .../environment/mcp-servers/mock-virt-mcp.py | 1518 ++++++++++++++++ .../rh-virt__vm-create/instruction.md | 14 + .../rh-virt__vm-create/solution/solve.sh | 71 + .../rh-virt__vm-create/task.toml | 26 + .../rh-virt__vm-create/tests/llm_judge.py | 92 + .../rh-virt__vm-create/tests/test.sh | 85 + .../rh-virt__vm-create/tests/test_outputs.py | 71 + .../rh-virt__vm-delete/environment/Dockerfile | 50 + .../environment/mcp-servers/mock-virt-mcp.py | 1464 ++++++++++++++++ .../rh-virt__vm-delete/instruction.md | 12 + .../rh-virt__vm-delete/solution/solve.sh | 31 + .../rh-virt__vm-delete/task.toml | 26 + .../rh-virt__vm-delete/tests/llm_judge.py | 93 + .../rh-virt__vm-delete/tests/test.sh | 85 + .../rh-virt__vm-delete/tests/test_outputs.py | 82 + .../environment/Dockerfile | 50 + .../environment/mcp-servers/mock-virt-mcp.py | 1458 ++++++++++++++++ .../rh-virt__vm-inventory/instruction.md | 14 + .../rh-virt__vm-inventory/solution/solve.sh | 32 + .../rh-virt__vm-inventory/task.toml | 26 + .../rh-virt__vm-inventory/tests/llm_judge.py | 92 + .../rh-virt__vm-inventory/tests/test.sh | 85 + .../tests/test_outputs.py | 67 + .../environment/Dockerfile | 50 + .../environment/mcp-servers/mock-virt-mcp.py | 1467 ++++++++++++++++ .../instruction.md | 12 + .../solution/solve.sh | 29 + .../rh-virt__vm-lifecycle-manager/task.toml | 26 + .../tests/llm_judge.py | 93 + .../tests/test.sh | 85 + .../tests/test_outputs.py | 75 + .../environment/Dockerfile | 50 + .../environment/mcp-servers/mock-virt-mcp.py | 1458 ++++++++++++++++ .../rh-virt__vm-rebalance/instruction.md | 13 + .../rh-virt__vm-rebalance/solution/solve.sh | 41 + .../rh-virt__vm-rebalance/task.toml | 26 + .../rh-virt__vm-rebalance/tests/llm_judge.py | 92 + .../rh-virt__vm-rebalance/tests/test.sh | 85 + .../tests/test_outputs.py | 57 + .../environment/Dockerfile | 63 + .../.ai-index/semantic-index.json | 148 ++ .../environment/docs/troubleshooting/INDEX.md | 332 ++++ .../docs/troubleshooting/lifecycle-errors.md | 869 ++++++++++ .../docs/troubleshooting/network-errors.md | 429 +++++ .../docs/troubleshooting/runtime-errors.md | 616 +++++++ .../docs/troubleshooting/scheduling-errors.md | 417 +++++ .../docs/troubleshooting/storage-errors.md | 1011 +++++++++++ .../environment/mcp-servers/mock-virt-mcp.py | 1539 +++++++++++++++++ .../instruction.md | 12 + .../solution/solve.sh | 39 + .../rh-virt__vm-snapshot-create/task.toml | 26 + .../tests/llm_judge.py | 93 + .../rh-virt__vm-snapshot-create/tests/test.sh | 85 + .../tests/test_outputs.py | 77 + .../environment/Dockerfile | 50 + .../environment/mcp-servers/mock-virt-mcp.py | 1458 ++++++++++++++++ .../instruction.md | 12 + .../solution/solve.sh | 26 + .../rh-virt__vm-snapshot-delete/task.toml | 26 + .../tests/llm_judge.py | 93 + .../rh-virt__vm-snapshot-delete/tests/test.sh | 85 + .../tests/test_outputs.py | 71 + .../environment/Dockerfile | 50 + .../environment/mcp-servers/mock-virt-mcp.py | 1500 ++++++++++++++++ .../rh-virt__vm-snapshot-list/instruction.md | 12 + .../solution/solve.sh | 30 + .../rh-virt__vm-snapshot-list/task.toml | 26 + .../tests/llm_judge.py | 93 + .../rh-virt__vm-snapshot-list/tests/test.sh | 85 + .../tests/test_outputs.py | 62 + .../environment/Dockerfile | 50 + .../environment/mcp-servers/mock-virt-mcp.py | 1458 ++++++++++++++++ .../instruction.md | 12 + .../solution/solve.sh | 39 + .../rh-virt__vm-snapshot-restore/task.toml | 26 + .../tests/llm_judge.py | 93 + .../tests/test.sh | 85 + .../tests/test_outputs.py | 71 + 1948 files changed, 476656 insertions(+) create mode 100644 evaluation/with_skills/PER_SKILL_REVIEW_REPORT.md create mode 100644 evaluation/with_skills/SKILL_PATH_FIXES.md create mode 100644 evaluation/with_skills/ocp-admin__cluster-report/environment/.mcp.json create mode 100644 evaluation/with_skills/ocp-admin__cluster-report/environment/Dockerfile create mode 100644 evaluation/with_skills/ocp-admin__cluster-report/environment/docs/multi-cluster-auth.md create mode 100644 evaluation/with_skills/ocp-admin__cluster-report/environment/mcp-servers/mock-ocp-mcp.py create mode 100644 evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/aggregate.py create mode 100644 evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/assemble.py create mode 100644 evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/build-kubeconfig.py create mode 100644 evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/cluster-reporter-rbac.yaml create mode 100644 evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/test_aggregate.py create mode 100644 evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/test_assemble.py create mode 100644 evaluation/with_skills/ocp-admin__cluster-report/environment/skills/cluster-report/SKILL.md create mode 100644 evaluation/with_skills/ocp-admin__cluster-report/instruction.md create mode 100644 evaluation/with_skills/ocp-admin__cluster-report/solution/solve.sh create mode 100644 evaluation/with_skills/ocp-admin__cluster-report/task.toml create mode 100644 evaluation/with_skills/ocp-admin__cluster-report/tests/llm_judge.py create mode 100644 evaluation/with_skills/ocp-admin__cluster-report/tests/test.sh create mode 100644 evaluation/with_skills/ocp-admin__cluster-report/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/common-issues.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/examples/model-deploy.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/examples/nim-setup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/live-doc-lookup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/references/known-model-profiles.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/references/supported-runtimes.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/skill-conventions.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-observability-mcp.py create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-rhoai-mcp.py create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/ai-observability/SKILL.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/common-issues.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/live-doc-lookup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/skill-conventions.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/instruction.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/solution/solve.sh create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/task.toml create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/tests/test.sh create mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/common-issues.md create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/examples/model-deploy.md create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/examples/nim-setup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/live-doc-lookup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/references/known-model-profiles.md create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/references/supported-runtimes.md create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/skill-conventions.md create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/mcp-servers/mock-rhoai-mcp.py create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/debug-inference/SKILL.md create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/common-issues.md create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/live-doc-lookup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/skill-conventions.md create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/instruction.md create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/solution/solve.sh create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/task.toml create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/tests/test.sh create mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/common-issues.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/model-deploy.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/nim-setup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/live-doc-lookup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/known-model-profiles.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/supported-runtimes.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/skill-conventions.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/mcp-servers/mock-rhoai-mcp.py create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/ds-project-setup/SKILL.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/common-issues.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/live-doc-lookup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/skill-conventions.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/instruction.md create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/solution/solve.sh create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/task.toml create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/tests/test.sh create mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/common-issues.md create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/examples/model-deploy.md create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/examples/nim-setup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/live-doc-lookup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/references/known-model-profiles.md create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/references/supported-runtimes.md create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/skill-conventions.md create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/mcp-servers/mock-rhoai-mcp.py create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/model-deploy/SKILL.md create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/model-deploy/references/model-deploy-preflight-checklist.md create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/common-issues.md create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/live-doc-lookup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/skill-conventions.md create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/instruction.md create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/solution/solve.sh create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/task.toml create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/tests/test.sh create mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/common-issues.md create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/examples/model-deploy.md create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/examples/nim-setup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/live-doc-lookup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/references/known-model-profiles.md create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/references/supported-runtimes.md create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/skill-conventions.md create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/mcp-servers/mock-rhoai-mcp.py create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/nim-setup/SKILL.md create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/common-issues.md create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/live-doc-lookup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/skill-conventions.md create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/instruction.md create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/solution/solve.sh create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/task.toml create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/tests/test.sh create mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/common-issues.md create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/model-deploy.md create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/nim-setup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/live-doc-lookup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/known-model-profiles.md create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/supported-runtimes.md create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/skill-conventions.md create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/mcp-servers/mock-rhoai-mcp.py create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/common-issues.md create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/live-doc-lookup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/skill-conventions.md create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/serving-runtime-config/SKILL.md create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/instruction.md create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/solution/solve.sh create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/task.toml create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/tests/test.sh create mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/common-issues.md create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/model-deploy.md create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/nim-setup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/live-doc-lookup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/references/known-model-profiles.md create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/references/supported-runtimes.md create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/skill-conventions.md create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/mcp-servers/mock-rhoai-mcp.py create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/common-issues.md create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/live-doc-lookup.md create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/skill-conventions.md create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/workbench-manage/SKILL.md create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/instruction.md create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/solution/solve.sh create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/task.toml create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/tests/test.sh create mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/builder-images.md create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/debugging-patterns.md create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/dynamic-validation.md create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/human-in-the-loop.md create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/image-selection-criteria.md create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/prerequisites.md create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/rhel-deployment.md create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/skills/containerize-deploy/SKILL.md create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/buildconfig.yaml.template create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/deployment.yaml.template create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/Chart.yaml.template create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/NOTES.txt.template create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/_helpers.tpl.template create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/deployment.yaml.template create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/route.yaml.template create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/service.yaml.template create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/values.yaml.template create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/imagestream.yaml.template create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/route.yaml.template create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/service.yaml.template create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/systemd/systemd-container-rootful.service create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/systemd/systemd-container-rootless.service create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/systemd/systemd-native.service create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/instruction.md create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/solution/solve.sh create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/task.toml create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/tests/test.sh create mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/docs/builder-images.md create mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/docs/debugging-patterns.md create mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/docs/dynamic-validation.md create mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/docs/human-in-the-loop.md create mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/docs/image-selection-criteria.md create mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/docs/prerequisites.md create mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/docs/rhel-deployment.md create mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/skills/debug-build/SKILL.md create mode 100644 evaluation/with_skills/rh-developer__debug-build/instruction.md create mode 100644 evaluation/with_skills/rh-developer__debug-build/solution/solve.sh create mode 100644 evaluation/with_skills/rh-developer__debug-build/task.toml create mode 100644 evaluation/with_skills/rh-developer__debug-build/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-developer__debug-build/tests/test.sh create mode 100644 evaluation/with_skills/rh-developer__debug-build/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/docs/builder-images.md create mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/docs/debugging-patterns.md create mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/docs/dynamic-validation.md create mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/docs/human-in-the-loop.md create mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/docs/image-selection-criteria.md create mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/docs/prerequisites.md create mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/docs/rhel-deployment.md create mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/mcp-servers/mock-podman-mcp.py create mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/skills/debug-container/SKILL.md create mode 100644 evaluation/with_skills/rh-developer__debug-container/instruction.md create mode 100644 evaluation/with_skills/rh-developer__debug-container/solution/solve.sh create mode 100644 evaluation/with_skills/rh-developer__debug-container/task.toml create mode 100644 evaluation/with_skills/rh-developer__debug-container/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-developer__debug-container/tests/test.sh create mode 100644 evaluation/with_skills/rh-developer__debug-container/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/docs/builder-images.md create mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/docs/debugging-patterns.md create mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/docs/dynamic-validation.md create mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/docs/human-in-the-loop.md create mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/docs/image-selection-criteria.md create mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/docs/prerequisites.md create mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/docs/rhel-deployment.md create mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/skills/debug-network/SKILL.md create mode 100644 evaluation/with_skills/rh-developer__debug-network/instruction.md create mode 100644 evaluation/with_skills/rh-developer__debug-network/solution/solve.sh create mode 100644 evaluation/with_skills/rh-developer__debug-network/task.toml create mode 100644 evaluation/with_skills/rh-developer__debug-network/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-developer__debug-network/tests/test.sh create mode 100644 evaluation/with_skills/rh-developer__debug-network/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/builder-images.md create mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/debugging-patterns.md create mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/dynamic-validation.md create mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/human-in-the-loop.md create mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/image-selection-criteria.md create mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/prerequisites.md create mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/rhel-deployment.md create mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/skills/debug-pipeline/SKILL.md create mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/instruction.md create mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/solution/solve.sh create mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/task.toml create mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/tests/test.sh create mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/docs/builder-images.md create mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/docs/debugging-patterns.md create mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/docs/dynamic-validation.md create mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/docs/human-in-the-loop.md create mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/docs/image-selection-criteria.md create mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/docs/prerequisites.md create mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/docs/rhel-deployment.md create mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/skills/debug-pod/SKILL.md create mode 100644 evaluation/with_skills/rh-developer__debug-pod/instruction.md create mode 100644 evaluation/with_skills/rh-developer__debug-pod/solution/solve.sh create mode 100644 evaluation/with_skills/rh-developer__debug-pod/task.toml create mode 100644 evaluation/with_skills/rh-developer__debug-pod/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-developer__debug-pod/tests/test.sh create mode 100644 evaluation/with_skills/rh-developer__debug-pod/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/docs/builder-images.md create mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/docs/debugging-patterns.md create mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/docs/dynamic-validation.md create mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/docs/human-in-the-loop.md create mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/docs/image-selection-criteria.md create mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/docs/prerequisites.md create mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/docs/rhel-deployment.md create mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/mcp-servers/mock-rhel-mcp.py create mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/skills/debug-rhel/SKILL.md create mode 100644 evaluation/with_skills/rh-developer__debug-rhel/instruction.md create mode 100644 evaluation/with_skills/rh-developer__debug-rhel/solution/solve.sh create mode 100644 evaluation/with_skills/rh-developer__debug-rhel/task.toml create mode 100644 evaluation/with_skills/rh-developer__debug-rhel/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-developer__debug-rhel/tests/test.sh create mode 100644 evaluation/with_skills/rh-developer__debug-rhel/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/docs/builder-images.md create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/docs/debugging-patterns.md create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/docs/dynamic-validation.md create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/docs/human-in-the-loop.md create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/docs/image-selection-criteria.md create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/docs/prerequisites.md create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/docs/rhel-deployment.md create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/skills/deploy/SKILL.md create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/templates/buildconfig.yaml.template create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/templates/deployment.yaml.template create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/templates/helm/Chart.yaml.template create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/NOTES.txt.template create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/_helpers.tpl.template create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/deployment.yaml.template create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/route.yaml.template create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/service.yaml.template create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/templates/helm/values.yaml.template create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/templates/imagestream.yaml.template create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/templates/route.yaml.template create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/templates/service.yaml.template create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/templates/systemd/systemd-container-rootful.service create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/templates/systemd/systemd-container-rootless.service create mode 100644 evaluation/with_skills/rh-developer__deploy/environment/templates/systemd/systemd-native.service create mode 100644 evaluation/with_skills/rh-developer__deploy/instruction.md create mode 100644 evaluation/with_skills/rh-developer__deploy/solution/solve.sh create mode 100644 evaluation/with_skills/rh-developer__deploy/task.toml create mode 100644 evaluation/with_skills/rh-developer__deploy/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-developer__deploy/tests/test.sh create mode 100644 evaluation/with_skills/rh-developer__deploy/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/docs/builder-images.md create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/docs/debugging-patterns.md create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/docs/dynamic-validation.md create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/docs/human-in-the-loop.md create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/docs/image-selection-criteria.md create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/docs/prerequisites.md create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/docs/rhel-deployment.md create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/sample-project/.s2i/environment create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/sample-project/Dockerfile create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/sample-project/app.py create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/sample-project/requirements.txt create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/sample-project/tests/test_app.py create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/skills/detect-project/SKILL.md create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/templates/buildconfig.yaml.template create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/templates/deployment.yaml.template create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/Chart.yaml.template create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/NOTES.txt.template create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/_helpers.tpl.template create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/deployment.yaml.template create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/route.yaml.template create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/service.yaml.template create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/values.yaml.template create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/templates/imagestream.yaml.template create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/templates/route.yaml.template create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/templates/service.yaml.template create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/templates/systemd/systemd-container-rootful.service create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/templates/systemd/systemd-container-rootless.service create mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/templates/systemd/systemd-native.service create mode 100644 evaluation/with_skills/rh-developer__detect-project/instruction.md create mode 100644 evaluation/with_skills/rh-developer__detect-project/solution/solve.sh create mode 100644 evaluation/with_skills/rh-developer__detect-project/task.toml create mode 100644 evaluation/with_skills/rh-developer__detect-project/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-developer__detect-project/tests/test.sh create mode 100644 evaluation/with_skills/rh-developer__detect-project/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/docs/builder-images.md create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/docs/debugging-patterns.md create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/docs/dynamic-validation.md create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/docs/human-in-the-loop.md create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/docs/image-selection-criteria.md create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/docs/prerequisites.md create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/docs/rhel-deployment.md create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/mcp-servers/mock-helm-mcp.py create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/skills/helm-deploy/SKILL.md create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/templates/buildconfig.yaml.template create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/templates/deployment.yaml.template create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/Chart.yaml.template create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/NOTES.txt.template create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/_helpers.tpl.template create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/deployment.yaml.template create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/route.yaml.template create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/service.yaml.template create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/values.yaml.template create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/templates/imagestream.yaml.template create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/templates/route.yaml.template create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/templates/service.yaml.template create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-container-rootful.service create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-container-rootless.service create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-native.service create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/instruction.md create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/solution/solve.sh create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/task.toml create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/tests/test.sh create mode 100644 evaluation/with_skills/rh-developer__helm-deploy/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/docs/builder-images.md create mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/docs/debugging-patterns.md create mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/docs/dynamic-validation.md create mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/docs/human-in-the-loop.md create mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/docs/image-selection-criteria.md create mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/docs/prerequisites.md create mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/docs/rhel-deployment.md create mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/skills/recommend-image/SKILL.md create mode 100644 evaluation/with_skills/rh-developer__recommend-image/instruction.md create mode 100644 evaluation/with_skills/rh-developer__recommend-image/solution/solve.sh create mode 100644 evaluation/with_skills/rh-developer__recommend-image/task.toml create mode 100644 evaluation/with_skills/rh-developer__recommend-image/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-developer__recommend-image/tests/test.sh create mode 100644 evaluation/with_skills/rh-developer__recommend-image/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/builder-images.md create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/debugging-patterns.md create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/dynamic-validation.md create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/human-in-the-loop.md create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/image-selection-criteria.md create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/prerequisites.md create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/rhel-deployment.md create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/mcp-servers/mock-rhel-host-mcp.py create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/skills/rhel-deploy/SKILL.md create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/buildconfig.yaml.template create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/deployment.yaml.template create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/Chart.yaml.template create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/NOTES.txt.template create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/_helpers.tpl.template create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/deployment.yaml.template create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/route.yaml.template create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/service.yaml.template create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/values.yaml.template create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/imagestream.yaml.template create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/route.yaml.template create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/service.yaml.template create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-container-rootful.service create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-container-rootless.service create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-native.service create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/instruction.md create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/solution/solve.sh create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/task.toml create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/tests/test.sh create mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/docs/builder-images.md create mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/docs/debugging-patterns.md create mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/docs/dynamic-validation.md create mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/docs/human-in-the-loop.md create mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/docs/image-selection-criteria.md create mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/docs/prerequisites.md create mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/docs/rhel-deployment.md create mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/skills/s2i-build/SKILL.md create mode 100644 evaluation/with_skills/rh-developer__s2i-build/instruction.md create mode 100644 evaluation/with_skills/rh-developer__s2i-build/solution/solve.sh create mode 100644 evaluation/with_skills/rh-developer__s2i-build/task.toml create mode 100644 evaluation/with_skills/rh-developer__s2i-build/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-developer__s2i-build/tests/test.sh create mode 100644 evaluation/with_skills/rh-developer__s2i-build/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/docs/builder-images.md create mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/docs/debugging-patterns.md create mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/docs/dynamic-validation.md create mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/docs/human-in-the-loop.md create mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/docs/image-selection-criteria.md create mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/docs/prerequisites.md create mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/docs/rhel-deployment.md create mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/skills/validate-environment/SKILL.md create mode 100644 evaluation/with_skills/rh-developer__validate-environment/instruction.md create mode 100644 evaluation/with_skills/rh-developer__validate-environment/solution/solve.sh create mode 100644 evaluation/with_skills/rh-developer__validate-environment/task.toml create mode 100644 evaluation/with_skills/rh-developer__validate-environment/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-developer__validate-environment/tests/test.sh create mode 100644 evaluation/with_skills/rh-developer__validate-environment/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/INDEX.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/SOURCES.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/README.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/insights/README.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/README.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/skill-invocation.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/rhel/README.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/rhel/package-management.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/01-account-cves.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/02-system-all-cves.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/03-system-remediatable-cves.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/01-cve-response-parser.py create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/02-cve-parsing-guide.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/03-output-templates.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/04-examples.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/05-error-handling.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/mcp-lightspeed-validator/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/instruction.md create mode 100644 evaluation/with_skills/rh-sre__cve-impact/solution/solve.sh create mode 100644 evaluation/with_skills/rh-sre__cve-impact/task.toml create mode 100644 evaluation/with_skills/rh-sre__cve-impact/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-sre__cve-impact/tests/test.sh create mode 100644 evaluation/with_skills/rh-sre__cve-impact/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/INDEX.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/SOURCES.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/README.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/insights/README.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/README.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/skill-invocation.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/rhel/README.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/rhel/package-management.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/01-remediation-indicators.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/03-output-template.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/04-examples.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/05-error-handling.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/skills/mcp-lightspeed-validator/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/instruction.md create mode 100644 evaluation/with_skills/rh-sre__cve-validation/solution/solve.sh create mode 100644 evaluation/with_skills/rh-sre__cve-validation/task.toml create mode 100644 evaluation/with_skills/rh-sre__cve-validation/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-sre__cve-validation/tests/test.sh create mode 100644 evaluation/with_skills/rh-sre__cve-validation/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/INDEX.md create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/SOURCES.md create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/README.md create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/insights/README.md create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/README.md create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/skill-invocation.md create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/rhel/README.md create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/rhel/package-management.md create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/skills/execution-summary/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__execution-summary/instruction.md create mode 100644 evaluation/with_skills/rh-sre__execution-summary/solution/solve.sh create mode 100644 evaluation/with_skills/rh-sre__execution-summary/task.toml create mode 100644 evaluation/with_skills/rh-sre__execution-summary/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-sre__execution-summary/tests/test.sh create mode 100644 evaluation/with_skills/rh-sre__execution-summary/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/INDEX.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/SOURCES.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/README.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/insights/README.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/README.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/skill-invocation.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/rhel/README.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/rhel/package-management.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/01-parameter-reference.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/03-output-templates.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/04-examples.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/05-error-handling.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/mcp-lightspeed-validator/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/instruction.md create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/solution/solve.sh create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/task.toml create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/tests/test.sh create mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/INDEX.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/SOURCES.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/README.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/insights/README.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/README.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/skill-invocation.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/rhel/README.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/rhel/package-management.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/mcp-servers/mock-aap-mcp.py create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/01-git-setup.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/02-web-ui-form.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/03-output-template.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/04-examples.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/mcp-aap-validator/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/01-execution-report-templates.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/02-error-handling-guide.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/03-workflow-examples.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/04-dry-run-display-templates.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/05-git-flow-prompts.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/instruction.md create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/solution/solve.sh create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/task.toml create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/tests/test.sh create mode 100644 evaluation/with_skills/rh-sre__job-template-creator/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/INDEX.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/SOURCES.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/README.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/README.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/README.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/skill-invocation.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/README.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/package-management.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/mcp-servers/mock-aap-mcp.py create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/01-git-setup.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/02-web-ui-form.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/03-output-template.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/04-examples.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-remediation-validator/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/mcp-aap-validator/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/01-execution-report-templates.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/02-error-handling-guide.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/03-workflow-examples.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/04-dry-run-display-templates.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/05-git-flow-prompts.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/instruction.md create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/solution/solve.sh create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/task.toml create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/tests/test.sh create mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/INDEX.md create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/SOURCES.md create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/README.md create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/insights/README.md create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/README.md create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/skill-invocation.md create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/README.md create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/package-management.md create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/mcp-servers/mock-aap-mcp.py create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/skills/mcp-aap-validator/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/instruction.md create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/solution/solve.sh create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/task.toml create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/tests/test.sh create mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/INDEX.md create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/SOURCES.md create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/README.md create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/README.md create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/README.md create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/skill-invocation.md create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/README.md create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/package-management.md create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/skills/mcp-lightspeed-validator/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/instruction.md create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/solution/solve.sh create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/task.toml create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/tests/test.sh create mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/INDEX.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/SOURCES.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/README.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/insights/README.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/README.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/skill-invocation.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/rhel/README.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/rhel/package-management.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/mcp-servers/mock-aap-mcp.py create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/skills/mcp-aap-validator/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/01-execution-report-templates.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/02-error-handling-guide.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/03-workflow-examples.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/04-dry-run-display-templates.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/05-git-flow-prompts.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/instruction.md create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/solution/solve.sh create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/task.toml create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/tests/test.sh create mode 100644 evaluation/with_skills/rh-sre__playbook-executor/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/INDEX.md create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/SOURCES.md create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/README.md create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/insights/README.md create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/README.md create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/skill-invocation.md create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/rhel/README.md create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/rhel/package-management.md create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/skills/playbook-generator/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/instruction.md create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/solution/solve.sh create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/task.toml create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/tests/test.sh create mode 100644 evaluation/with_skills/rh-sre__playbook-generator/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/INDEX.md create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/SOURCES.md create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/README.md create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/insights/README.md create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/README.md create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/skill-invocation.md create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/rhel/README.md create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/rhel/package-management.md create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/skills/remediation-verifier/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/instruction.md create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/solution/solve.sh create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/task.toml create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/tests/test.sh create mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/INDEX.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/SOURCES.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/README.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/insights/README.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/references/README.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/references/skill-invocation.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/rhel/README.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/rhel/package-management.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/01-remediation-indicators.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/03-output-template.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/04-examples.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/05-error-handling.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/skills/remediation/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__remediation/environment/skills/remediation/references/01-remediation-plan-template.md create mode 100644 evaluation/with_skills/rh-sre__remediation/instruction.md create mode 100644 evaluation/with_skills/rh-sre__remediation/solution/solve.sh create mode 100644 evaluation/with_skills/rh-sre__remediation/task.toml create mode 100644 evaluation/with_skills/rh-sre__remediation/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-sre__remediation/tests/test.sh create mode 100644 evaluation/with_skills/rh-sre__remediation/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/INDEX.md create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/SOURCES.md create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/README.md create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/insights/README.md create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/references/README.md create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/references/skill-invocation.md create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/rhel/README.md create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/rhel/package-management.md create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/with_skills/rh-sre__system-context/environment/skills/system-context/SKILL.md create mode 100644 evaluation/with_skills/rh-sre__system-context/instruction.md create mode 100644 evaluation/with_skills/rh-sre__system-context/solution/solve.sh create mode 100644 evaluation/with_skills/rh-sre__system-context/task.toml create mode 100644 evaluation/with_skills/rh-sre__system-context/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-sre__system-context/tests/test.sh create mode 100644 evaluation/with_skills/rh-sre__system-context/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-virt__vm-clone/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/INDEX.md create mode 100644 evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/lifecycle-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/network-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/runtime-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/scheduling-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/storage-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-clone/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/with_skills/rh-virt__vm-clone/environment/skills/vm-clone/SKILL.md create mode 100644 evaluation/with_skills/rh-virt__vm-clone/instruction.md create mode 100644 evaluation/with_skills/rh-virt__vm-clone/solution/solve.sh create mode 100644 evaluation/with_skills/rh-virt__vm-clone/task.toml create mode 100644 evaluation/with_skills/rh-virt__vm-clone/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-virt__vm-clone/tests/test.sh create mode 100644 evaluation/with_skills/rh-virt__vm-clone/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-virt__vm-create/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/INDEX.md create mode 100644 evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/lifecycle-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/network-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/runtime-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/scheduling-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/storage-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-create/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/with_skills/rh-virt__vm-create/environment/skills/vm-create/SKILL.md create mode 100644 evaluation/with_skills/rh-virt__vm-create/instruction.md create mode 100644 evaluation/with_skills/rh-virt__vm-create/solution/solve.sh create mode 100644 evaluation/with_skills/rh-virt__vm-create/task.toml create mode 100644 evaluation/with_skills/rh-virt__vm-create/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-virt__vm-create/tests/test.sh create mode 100644 evaluation/with_skills/rh-virt__vm-create/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-virt__vm-delete/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/INDEX.md create mode 100644 evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/lifecycle-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/network-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/runtime-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/scheduling-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/storage-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-delete/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/with_skills/rh-virt__vm-delete/environment/skills/vm-delete/SKILL.md create mode 100644 evaluation/with_skills/rh-virt__vm-delete/instruction.md create mode 100644 evaluation/with_skills/rh-virt__vm-delete/solution/solve.sh create mode 100644 evaluation/with_skills/rh-virt__vm-delete/task.toml create mode 100644 evaluation/with_skills/rh-virt__vm-delete/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-virt__vm-delete/tests/test.sh create mode 100644 evaluation/with_skills/rh-virt__vm-delete/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-virt__vm-inventory/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/INDEX.md create mode 100644 evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/lifecycle-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/network-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/runtime-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/scheduling-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/storage-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-inventory/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/with_skills/rh-virt__vm-inventory/environment/skills/vm-inventory/SKILL.md create mode 100644 evaluation/with_skills/rh-virt__vm-inventory/instruction.md create mode 100644 evaluation/with_skills/rh-virt__vm-inventory/solution/solve.sh create mode 100644 evaluation/with_skills/rh-virt__vm-inventory/task.toml create mode 100644 evaluation/with_skills/rh-virt__vm-inventory/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-virt__vm-inventory/tests/test.sh create mode 100644 evaluation/with_skills/rh-virt__vm-inventory/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/INDEX.md create mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/lifecycle-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/network-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/runtime-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/scheduling-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/storage-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/skills/vm-lifecycle-manager/SKILL.md create mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/instruction.md create mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/solution/solve.sh create mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/task.toml create mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/tests/test.sh create mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/INDEX.md create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/lifecycle-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/network-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/runtime-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/scheduling-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/storage-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/REBALANCE_AUTOMATIC.md create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/REBALANCE_MANUAL.md create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/SKILL.md create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/anti-patterns.md create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/live-migration-best-practices.md create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/performance-tuning.md create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/production-considerations.md create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/instruction.md create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/solution/solve.sh create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/task.toml create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/tests/test.sh create mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/INDEX.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/lifecycle-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/network-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/runtime-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/scheduling-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/storage-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/environment/skills/vm-snapshot-create/SKILL.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/instruction.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/solution/solve.sh create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/task.toml create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/tests/test.sh create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/INDEX.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/lifecycle-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/network-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/runtime-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/scheduling-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/storage-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/skills/vm-snapshot-delete/SKILL.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/instruction.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/solution/solve.sh create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/task.toml create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/tests/test.sh create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/INDEX.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/lifecycle-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/network-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/runtime-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/scheduling-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/storage-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/environment/skills/vm-snapshot-list/SKILL.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/instruction.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/solution/solve.sh create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/task.toml create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/tests/test.sh create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/tests/test_outputs.py create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/Dockerfile create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/.ai-index/semantic-index.json create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/INDEX.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/lifecycle-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/network-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/runtime-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/scheduling-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/storage-errors.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/skills/vm-snapshot-restore/SKILL.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/instruction.md create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/solution/solve.sh create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/task.toml create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/tests/llm_judge.py create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/tests/test.sh create mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/tests/test_outputs.py create mode 100644 evaluation/without_skills/ocp-admin__cluster-report/environment/Dockerfile create mode 100644 evaluation/without_skills/ocp-admin__cluster-report/environment/docs/multi-cluster-auth.md create mode 100644 evaluation/without_skills/ocp-admin__cluster-report/environment/mcp-servers/mock-ocp-mcp.py create mode 100644 evaluation/without_skills/ocp-admin__cluster-report/instruction.md create mode 100644 evaluation/without_skills/ocp-admin__cluster-report/solution/solve.sh create mode 100644 evaluation/without_skills/ocp-admin__cluster-report/task.toml create mode 100644 evaluation/without_skills/ocp-admin__cluster-report/tests/llm_judge.py create mode 100644 evaluation/without_skills/ocp-admin__cluster-report/tests/test.sh create mode 100644 evaluation/without_skills/ocp-admin__cluster-report/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/common-issues.md create mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/examples/model-deploy.md create mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/examples/nim-setup.md create mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/live-doc-lookup.md create mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/references/known-model-profiles.md create mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/references/supported-runtimes.md create mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/skill-conventions.md create mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-observability-mcp.py create mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-rhoai-mcp.py create mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/instruction.md create mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/solution/solve.sh create mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/task.toml create mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/tests/test.sh create mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/common-issues.md create mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/examples/model-deploy.md create mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/examples/nim-setup.md create mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/live-doc-lookup.md create mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/references/known-model-profiles.md create mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/references/supported-runtimes.md create mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/skill-conventions.md create mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/environment/mcp-servers/mock-rhoai-mcp.py create mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/instruction.md create mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/solution/solve.sh create mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/task.toml create mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/tests/test.sh create mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/common-issues.md create mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/model-deploy.md create mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/nim-setup.md create mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/live-doc-lookup.md create mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/known-model-profiles.md create mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/supported-runtimes.md create mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/skill-conventions.md create mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/mcp-servers/mock-rhoai-mcp.py create mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/instruction.md create mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/solution/solve.sh create mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/task.toml create mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/tests/test.sh create mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/common-issues.md create mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/examples/model-deploy.md create mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/examples/nim-setup.md create mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/live-doc-lookup.md create mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/references/known-model-profiles.md create mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/references/supported-runtimes.md create mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/skill-conventions.md create mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/environment/mcp-servers/mock-rhoai-mcp.py create mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/instruction.md create mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/solution/solve.sh create mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/task.toml create mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/tests/test.sh create mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/common-issues.md create mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/examples/model-deploy.md create mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/examples/nim-setup.md create mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/live-doc-lookup.md create mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/references/known-model-profiles.md create mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/references/supported-runtimes.md create mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/skill-conventions.md create mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/environment/mcp-servers/mock-rhoai-mcp.py create mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/instruction.md create mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/solution/solve.sh create mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/task.toml create mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/tests/test.sh create mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/common-issues.md create mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/model-deploy.md create mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/nim-setup.md create mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/live-doc-lookup.md create mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/known-model-profiles.md create mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/supported-runtimes.md create mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/skill-conventions.md create mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/mcp-servers/mock-rhoai-mcp.py create mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/instruction.md create mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/solution/solve.sh create mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/task.toml create mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/tests/test.sh create mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/common-issues.md create mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/model-deploy.md create mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/nim-setup.md create mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/live-doc-lookup.md create mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/references/known-model-profiles.md create mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/references/supported-runtimes.md create mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/skill-conventions.md create mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/mcp-servers/mock-rhoai-mcp.py create mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/instruction.md create mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/solution/solve.sh create mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/task.toml create mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/tests/test.sh create mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/builder-images.md create mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/debugging-patterns.md create mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/dynamic-validation.md create mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/human-in-the-loop.md create mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/image-selection-criteria.md create mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/prerequisites.md create mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/rhel-deployment.md create mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/instruction.md create mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/solution/solve.sh create mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/task.toml create mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/tests/test.sh create mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/docs/builder-images.md create mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/docs/debugging-patterns.md create mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/docs/dynamic-validation.md create mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/docs/human-in-the-loop.md create mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/docs/image-selection-criteria.md create mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/docs/prerequisites.md create mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/docs/rhel-deployment.md create mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-developer__debug-build/instruction.md create mode 100644 evaluation/without_skills/rh-developer__debug-build/solution/solve.sh create mode 100644 evaluation/without_skills/rh-developer__debug-build/task.toml create mode 100644 evaluation/without_skills/rh-developer__debug-build/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-developer__debug-build/tests/test.sh create mode 100644 evaluation/without_skills/rh-developer__debug-build/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/docs/builder-images.md create mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/docs/debugging-patterns.md create mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/docs/dynamic-validation.md create mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/docs/human-in-the-loop.md create mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/docs/image-selection-criteria.md create mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/docs/prerequisites.md create mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/docs/rhel-deployment.md create mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/mcp-servers/mock-podman-mcp.py create mode 100644 evaluation/without_skills/rh-developer__debug-container/instruction.md create mode 100644 evaluation/without_skills/rh-developer__debug-container/solution/solve.sh create mode 100644 evaluation/without_skills/rh-developer__debug-container/task.toml create mode 100644 evaluation/without_skills/rh-developer__debug-container/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-developer__debug-container/tests/test.sh create mode 100644 evaluation/without_skills/rh-developer__debug-container/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/docs/builder-images.md create mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/docs/debugging-patterns.md create mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/docs/dynamic-validation.md create mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/docs/human-in-the-loop.md create mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/docs/image-selection-criteria.md create mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/docs/prerequisites.md create mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/docs/rhel-deployment.md create mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-developer__debug-network/instruction.md create mode 100644 evaluation/without_skills/rh-developer__debug-network/solution/solve.sh create mode 100644 evaluation/without_skills/rh-developer__debug-network/task.toml create mode 100644 evaluation/without_skills/rh-developer__debug-network/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-developer__debug-network/tests/test.sh create mode 100644 evaluation/without_skills/rh-developer__debug-network/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/builder-images.md create mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/debugging-patterns.md create mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/dynamic-validation.md create mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/human-in-the-loop.md create mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/image-selection-criteria.md create mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/prerequisites.md create mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/rhel-deployment.md create mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/instruction.md create mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/solution/solve.sh create mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/task.toml create mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/tests/test.sh create mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/docs/builder-images.md create mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/docs/debugging-patterns.md create mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/docs/dynamic-validation.md create mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/docs/human-in-the-loop.md create mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/docs/image-selection-criteria.md create mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/docs/prerequisites.md create mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/docs/rhel-deployment.md create mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-developer__debug-pod/instruction.md create mode 100644 evaluation/without_skills/rh-developer__debug-pod/solution/solve.sh create mode 100644 evaluation/without_skills/rh-developer__debug-pod/task.toml create mode 100644 evaluation/without_skills/rh-developer__debug-pod/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-developer__debug-pod/tests/test.sh create mode 100644 evaluation/without_skills/rh-developer__debug-pod/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/docs/builder-images.md create mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/docs/debugging-patterns.md create mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/docs/dynamic-validation.md create mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/docs/human-in-the-loop.md create mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/docs/image-selection-criteria.md create mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/docs/prerequisites.md create mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/docs/rhel-deployment.md create mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/mcp-servers/mock-rhel-mcp.py create mode 100644 evaluation/without_skills/rh-developer__debug-rhel/instruction.md create mode 100644 evaluation/without_skills/rh-developer__debug-rhel/solution/solve.sh create mode 100644 evaluation/without_skills/rh-developer__debug-rhel/task.toml create mode 100644 evaluation/without_skills/rh-developer__debug-rhel/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-developer__debug-rhel/tests/test.sh create mode 100644 evaluation/without_skills/rh-developer__debug-rhel/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-developer__deploy/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-developer__deploy/environment/docs/builder-images.md create mode 100644 evaluation/without_skills/rh-developer__deploy/environment/docs/debugging-patterns.md create mode 100644 evaluation/without_skills/rh-developer__deploy/environment/docs/dynamic-validation.md create mode 100644 evaluation/without_skills/rh-developer__deploy/environment/docs/human-in-the-loop.md create mode 100644 evaluation/without_skills/rh-developer__deploy/environment/docs/image-selection-criteria.md create mode 100644 evaluation/without_skills/rh-developer__deploy/environment/docs/prerequisites.md create mode 100644 evaluation/without_skills/rh-developer__deploy/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/without_skills/rh-developer__deploy/environment/docs/rhel-deployment.md create mode 100644 evaluation/without_skills/rh-developer__deploy/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/without_skills/rh-developer__deploy/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-developer__deploy/instruction.md create mode 100644 evaluation/without_skills/rh-developer__deploy/solution/solve.sh create mode 100644 evaluation/without_skills/rh-developer__deploy/task.toml create mode 100644 evaluation/without_skills/rh-developer__deploy/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-developer__deploy/tests/test.sh create mode 100644 evaluation/without_skills/rh-developer__deploy/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/docs/builder-images.md create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/docs/debugging-patterns.md create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/docs/dynamic-validation.md create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/docs/human-in-the-loop.md create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/docs/image-selection-criteria.md create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/docs/prerequisites.md create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/docs/rhel-deployment.md create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/sample-project/.s2i/environment create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/sample-project/Dockerfile create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/sample-project/app.py create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/sample-project/requirements.txt create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/sample-project/tests/test_app.py create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/templates/buildconfig.yaml.template create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/templates/deployment.yaml.template create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/Chart.yaml.template create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/NOTES.txt.template create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/_helpers.tpl.template create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/deployment.yaml.template create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/route.yaml.template create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/service.yaml.template create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/values.yaml.template create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/templates/imagestream.yaml.template create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/templates/route.yaml.template create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/templates/service.yaml.template create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/templates/systemd/systemd-container-rootful.service create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/templates/systemd/systemd-container-rootless.service create mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/templates/systemd/systemd-native.service create mode 100644 evaluation/without_skills/rh-developer__detect-project/instruction.md create mode 100644 evaluation/without_skills/rh-developer__detect-project/solution/solve.sh create mode 100644 evaluation/without_skills/rh-developer__detect-project/task.toml create mode 100644 evaluation/without_skills/rh-developer__detect-project/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-developer__detect-project/tests/test.sh create mode 100644 evaluation/without_skills/rh-developer__detect-project/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/docs/builder-images.md create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/docs/debugging-patterns.md create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/docs/dynamic-validation.md create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/docs/human-in-the-loop.md create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/docs/image-selection-criteria.md create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/docs/prerequisites.md create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/docs/rhel-deployment.md create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/mcp-servers/mock-helm-mcp.py create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/templates/buildconfig.yaml.template create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/templates/deployment.yaml.template create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/Chart.yaml.template create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/NOTES.txt.template create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/_helpers.tpl.template create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/deployment.yaml.template create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/route.yaml.template create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/service.yaml.template create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/values.yaml.template create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/templates/imagestream.yaml.template create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/templates/route.yaml.template create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/templates/service.yaml.template create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-container-rootful.service create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-container-rootless.service create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-native.service create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/instruction.md create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/solution/solve.sh create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/task.toml create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/tests/test.sh create mode 100644 evaluation/without_skills/rh-developer__helm-deploy/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/docs/builder-images.md create mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/docs/debugging-patterns.md create mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/docs/dynamic-validation.md create mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/docs/human-in-the-loop.md create mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/docs/image-selection-criteria.md create mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/docs/prerequisites.md create mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/docs/rhel-deployment.md create mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-developer__recommend-image/instruction.md create mode 100644 evaluation/without_skills/rh-developer__recommend-image/solution/solve.sh create mode 100644 evaluation/without_skills/rh-developer__recommend-image/task.toml create mode 100644 evaluation/without_skills/rh-developer__recommend-image/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-developer__recommend-image/tests/test.sh create mode 100644 evaluation/without_skills/rh-developer__recommend-image/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/builder-images.md create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/debugging-patterns.md create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/dynamic-validation.md create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/human-in-the-loop.md create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/image-selection-criteria.md create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/prerequisites.md create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/rhel-deployment.md create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/mcp-servers/mock-rhel-host-mcp.py create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/buildconfig.yaml.template create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/deployment.yaml.template create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/Chart.yaml.template create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/NOTES.txt.template create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/_helpers.tpl.template create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/deployment.yaml.template create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/route.yaml.template create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/service.yaml.template create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/values.yaml.template create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/imagestream.yaml.template create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/route.yaml.template create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/service.yaml.template create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-container-rootful.service create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-container-rootless.service create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-native.service create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/instruction.md create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/solution/solve.sh create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/task.toml create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/tests/test.sh create mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/docs/builder-images.md create mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/docs/debugging-patterns.md create mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/docs/dynamic-validation.md create mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/docs/human-in-the-loop.md create mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/docs/image-selection-criteria.md create mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/docs/prerequisites.md create mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/docs/rhel-deployment.md create mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-developer__s2i-build/instruction.md create mode 100644 evaluation/without_skills/rh-developer__s2i-build/solution/solve.sh create mode 100644 evaluation/without_skills/rh-developer__s2i-build/task.toml create mode 100644 evaluation/without_skills/rh-developer__s2i-build/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-developer__s2i-build/tests/test.sh create mode 100644 evaluation/without_skills/rh-developer__s2i-build/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/docs/builder-images.md create mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/docs/debugging-patterns.md create mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/docs/dynamic-validation.md create mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/docs/human-in-the-loop.md create mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/docs/image-selection-criteria.md create mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/docs/prerequisites.md create mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/docs/python-s2i-entrypoints.md create mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/docs/rhel-deployment.md create mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/docs/selinux-troubleshooting.md create mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/mcp-servers/mock-openshift-mcp.py create mode 100644 evaluation/without_skills/rh-developer__validate-environment/instruction.md create mode 100644 evaluation/without_skills/rh-developer__validate-environment/solution/solve.sh create mode 100644 evaluation/without_skills/rh-developer__validate-environment/task.toml create mode 100644 evaluation/without_skills/rh-developer__validate-environment/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-developer__validate-environment/tests/test.sh create mode 100644 evaluation/without_skills/rh-developer__validate-environment/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/INDEX.md create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/SOURCES.md create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/README.md create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/insights/README.md create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/README.md create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/skill-invocation.md create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/rhel/README.md create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/rhel/package-management.md create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/without_skills/rh-sre__cve-impact/instruction.md create mode 100644 evaluation/without_skills/rh-sre__cve-impact/solution/solve.sh create mode 100644 evaluation/without_skills/rh-sre__cve-impact/task.toml create mode 100644 evaluation/without_skills/rh-sre__cve-impact/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-sre__cve-impact/tests/test.sh create mode 100644 evaluation/without_skills/rh-sre__cve-impact/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/INDEX.md create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/SOURCES.md create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/README.md create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/insights/README.md create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/README.md create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/skill-invocation.md create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/rhel/README.md create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/rhel/package-management.md create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/without_skills/rh-sre__cve-validation/instruction.md create mode 100644 evaluation/without_skills/rh-sre__cve-validation/solution/solve.sh create mode 100644 evaluation/without_skills/rh-sre__cve-validation/task.toml create mode 100644 evaluation/without_skills/rh-sre__cve-validation/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-sre__cve-validation/tests/test.sh create mode 100644 evaluation/without_skills/rh-sre__cve-validation/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/INDEX.md create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/SOURCES.md create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/README.md create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/insights/README.md create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/README.md create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/skill-invocation.md create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/rhel/README.md create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/rhel/package-management.md create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/without_skills/rh-sre__execution-summary/instruction.md create mode 100644 evaluation/without_skills/rh-sre__execution-summary/solution/solve.sh create mode 100644 evaluation/without_skills/rh-sre__execution-summary/task.toml create mode 100644 evaluation/without_skills/rh-sre__execution-summary/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-sre__execution-summary/tests/test.sh create mode 100644 evaluation/without_skills/rh-sre__execution-summary/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/INDEX.md create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/SOURCES.md create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/README.md create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/insights/README.md create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/README.md create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/skill-invocation.md create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/rhel/README.md create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/rhel/package-management.md create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/instruction.md create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/solution/solve.sh create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/task.toml create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/tests/test.sh create mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/INDEX.md create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/SOURCES.md create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/README.md create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/insights/README.md create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/README.md create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/skill-invocation.md create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/rhel/README.md create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/rhel/package-management.md create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/mcp-servers/mock-aap-mcp.py create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/instruction.md create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/solution/solve.sh create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/task.toml create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/tests/test.sh create mode 100644 evaluation/without_skills/rh-sre__job-template-creator/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/INDEX.md create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/SOURCES.md create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/README.md create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/README.md create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/README.md create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/skill-invocation.md create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/README.md create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/package-management.md create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/mcp-servers/mock-aap-mcp.py create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/instruction.md create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/solution/solve.sh create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/task.toml create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/tests/test.sh create mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/INDEX.md create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/SOURCES.md create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/README.md create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/insights/README.md create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/README.md create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/skill-invocation.md create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/README.md create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/package-management.md create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/mcp-servers/mock-aap-mcp.py create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/instruction.md create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/solution/solve.sh create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/task.toml create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/tests/test.sh create mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/INDEX.md create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/SOURCES.md create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/README.md create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/README.md create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/README.md create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/skill-invocation.md create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/README.md create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/package-management.md create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/instruction.md create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/solution/solve.sh create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/task.toml create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/tests/test.sh create mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/INDEX.md create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/SOURCES.md create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/README.md create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/insights/README.md create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/README.md create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/skill-invocation.md create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/rhel/README.md create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/rhel/package-management.md create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/mcp-servers/mock-aap-mcp.py create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/instruction.md create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/solution/solve.sh create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/task.toml create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/tests/test.sh create mode 100644 evaluation/without_skills/rh-sre__playbook-executor/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/INDEX.md create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/SOURCES.md create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/README.md create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/insights/README.md create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/README.md create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/skill-invocation.md create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/rhel/README.md create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/rhel/package-management.md create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/instruction.md create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/solution/solve.sh create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/task.toml create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/tests/test.sh create mode 100644 evaluation/without_skills/rh-sre__playbook-generator/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/INDEX.md create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/SOURCES.md create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/README.md create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/insights/README.md create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/README.md create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/skill-invocation.md create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/rhel/README.md create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/rhel/package-management.md create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/instruction.md create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/solution/solve.sh create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/task.toml create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/tests/test.sh create mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/INDEX.md create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/SOURCES.md create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/README.md create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/insights/README.md create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/references/README.md create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/references/skill-invocation.md create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/rhel/README.md create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/rhel/package-management.md create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/without_skills/rh-sre__remediation/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/without_skills/rh-sre__remediation/instruction.md create mode 100644 evaluation/without_skills/rh-sre__remediation/solution/solve.sh create mode 100644 evaluation/without_skills/rh-sre__remediation/task.toml create mode 100644 evaluation/without_skills/rh-sre__remediation/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-sre__remediation/tests/test.sh create mode 100644 evaluation/without_skills/rh-sre__remediation/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/cross-reference-graph.json create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/semantic-index.json create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/task-to-docs-mapping.json create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/INDEX.md create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/SOURCES.md create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/README.md create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/aap-job-execution.md create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/cve-remediation-templates.md create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/playbook-integration-aap.md create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/insights/README.md create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/insights/vulnerability-logic.md create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/references/README.md create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/references/cvss-scoring.md create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-parameters.md create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-tool-failures.md create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/references/skill-invocation.md create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/rhel/README.md create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/rhel/package-management.md create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/testing/aap-integration-test-guide.md create mode 100644 evaluation/without_skills/rh-sre__system-context/environment/mcp-servers/mock-lightspeed-mcp.py create mode 100644 evaluation/without_skills/rh-sre__system-context/instruction.md create mode 100644 evaluation/without_skills/rh-sre__system-context/solution/solve.sh create mode 100644 evaluation/without_skills/rh-sre__system-context/task.toml create mode 100644 evaluation/without_skills/rh-sre__system-context/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-sre__system-context/tests/test.sh create mode 100644 evaluation/without_skills/rh-sre__system-context/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-virt__vm-clone/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-virt__vm-clone/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/without_skills/rh-virt__vm-clone/instruction.md create mode 100644 evaluation/without_skills/rh-virt__vm-clone/solution/solve.sh create mode 100644 evaluation/without_skills/rh-virt__vm-clone/task.toml create mode 100644 evaluation/without_skills/rh-virt__vm-clone/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-virt__vm-clone/tests/test.sh create mode 100644 evaluation/without_skills/rh-virt__vm-clone/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-virt__vm-create/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/.ai-index/semantic-index.json create mode 100644 evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/INDEX.md create mode 100644 evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/lifecycle-errors.md create mode 100644 evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/network-errors.md create mode 100644 evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/runtime-errors.md create mode 100644 evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/scheduling-errors.md create mode 100644 evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/storage-errors.md create mode 100644 evaluation/without_skills/rh-virt__vm-create/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/without_skills/rh-virt__vm-create/instruction.md create mode 100644 evaluation/without_skills/rh-virt__vm-create/solution/solve.sh create mode 100644 evaluation/without_skills/rh-virt__vm-create/task.toml create mode 100644 evaluation/without_skills/rh-virt__vm-create/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-virt__vm-create/tests/test.sh create mode 100644 evaluation/without_skills/rh-virt__vm-create/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-virt__vm-delete/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-virt__vm-delete/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/without_skills/rh-virt__vm-delete/instruction.md create mode 100644 evaluation/without_skills/rh-virt__vm-delete/solution/solve.sh create mode 100644 evaluation/without_skills/rh-virt__vm-delete/task.toml create mode 100644 evaluation/without_skills/rh-virt__vm-delete/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-virt__vm-delete/tests/test.sh create mode 100644 evaluation/without_skills/rh-virt__vm-delete/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-virt__vm-inventory/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-virt__vm-inventory/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/without_skills/rh-virt__vm-inventory/instruction.md create mode 100644 evaluation/without_skills/rh-virt__vm-inventory/solution/solve.sh create mode 100644 evaluation/without_skills/rh-virt__vm-inventory/task.toml create mode 100644 evaluation/without_skills/rh-virt__vm-inventory/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-virt__vm-inventory/tests/test.sh create mode 100644 evaluation/without_skills/rh-virt__vm-inventory/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-virt__vm-lifecycle-manager/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-virt__vm-lifecycle-manager/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/without_skills/rh-virt__vm-lifecycle-manager/instruction.md create mode 100644 evaluation/without_skills/rh-virt__vm-lifecycle-manager/solution/solve.sh create mode 100644 evaluation/without_skills/rh-virt__vm-lifecycle-manager/task.toml create mode 100644 evaluation/without_skills/rh-virt__vm-lifecycle-manager/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-virt__vm-lifecycle-manager/tests/test.sh create mode 100644 evaluation/without_skills/rh-virt__vm-lifecycle-manager/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-virt__vm-rebalance/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-virt__vm-rebalance/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/without_skills/rh-virt__vm-rebalance/instruction.md create mode 100644 evaluation/without_skills/rh-virt__vm-rebalance/solution/solve.sh create mode 100644 evaluation/without_skills/rh-virt__vm-rebalance/task.toml create mode 100644 evaluation/without_skills/rh-virt__vm-rebalance/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-virt__vm-rebalance/tests/test.sh create mode 100644 evaluation/without_skills/rh-virt__vm-rebalance/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/.ai-index/semantic-index.json create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/INDEX.md create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/lifecycle-errors.md create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/network-errors.md create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/runtime-errors.md create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/scheduling-errors.md create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/storage-errors.md create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/instruction.md create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/solution/solve.sh create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/task.toml create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/tests/test.sh create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-delete/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-delete/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-delete/instruction.md create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-delete/solution/solve.sh create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-delete/task.toml create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-delete/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-delete/tests/test.sh create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-delete/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-list/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-list/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-list/instruction.md create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-list/solution/solve.sh create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-list/task.toml create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-list/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-list/tests/test.sh create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-list/tests/test_outputs.py create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-restore/environment/Dockerfile create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-restore/environment/mcp-servers/mock-virt-mcp.py create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-restore/instruction.md create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-restore/solution/solve.sh create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-restore/task.toml create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-restore/tests/llm_judge.py create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-restore/tests/test.sh create mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-restore/tests/test_outputs.py diff --git a/evaluation/with_skills/PER_SKILL_REVIEW_REPORT.md b/evaluation/with_skills/PER_SKILL_REVIEW_REPORT.md new file mode 100644 index 00000000..3698d689 --- /dev/null +++ b/evaluation/with_skills/PER_SKILL_REVIEW_REPORT.md @@ -0,0 +1,556 @@ +# Per-Skill Evaluation Review Report + +Review of each task under `tasks/per_skill_eval/` covering instructions, tests, skills, docs, and mock MCP. Criteria: instructions clear/realistic/fair/not overfitting; tests fair/not overfitting; mock MCP proper and realistic. + +--- + +## ocp-admin__cluster-report + +**Instructions:** Clear and realistic. Asks for cluster health and inventory report with version, nodes, projects, pods. Explicitly asks to document methodology. Does not mention skills. Fair scope. + +**Tests:** Conceptual checks (cluster version, node status, resource utilization, projects, workload stats, context awareness). No exact tool or field names. Fair and not overfitting. + +**Mock MCP:** mock-ocp-mcp provides multiple contexts (prod-us-east, prod-eu-west, staging-central, dev-k8s, legacy-dc), ClusterVersion, nodes, projects, pods. Realistic. Supports both OpenShift and non-OpenShift contexts. Good. + +**Remarks:** None. + +--- + +## rh-ai-engineer__ai-observability + +**Instructions:** Clear. Set up monitoring for AI/ML models: metrics, GPU utilization, right-sizing. Does not mention skills. Realistic. + +**Tests:** Conceptual (GPU monitoring, model metrics, right-sizing, Prometheus/Grafana, alerting). No tool-name matching. Fair. + +**Mock MCP:** Uses rhoai and openshift mocks; not ai-observability MCP. Skill expects get_gpu_info, analyze_vllm, etc. Agent can still describe methodology from the skill. Rhoai mock has inference services, projects; openshift has resources. Adequate for methodology documentation. + +**Remarks:** Mock does not implement ai-observability MCP tools. Agent relies on skill/docs and available rhoai/openshift tools. Acceptable for report-based evaluation. + +--- + +## rh-ai-engineer__debug-inference + +**Instructions:** Clear. Debug failing InferenceService: readiness, pod scheduling, resources, recommend fix. Does not mention skills. Realistic. + +**Tests:** Conceptual (readiness, scheduling, logs, resources, events, fix recommendation). Fair. + +**Mock MCP:** Rhoai mock has broken deployments (text-gen-legacy OOMKilled, nim-llama-prod failing). Openshift mock has pods, events, logs. Good for debugging. + +**Remarks:** None. + +--- + +## rh-ai-engineer__ds-project-setup + +**Instructions:** Clear. Set up data science project with storage, model serving, data connections. Does not mention skills. Realistic. + +**Tests:** Conceptual (project creation, data connections, model serving, credentials, dashboard). Fair. + +**Mock MCP:** Rhoai mock has projects, data connections, serving runtimes, inference services. Good coverage. + +**Remarks:** None. + +--- + +## rh-ai-engineer__model-deploy + +**Instructions:** Clear. Deploy ML model: serving runtime, InferenceService, GPU, common issues. Does not mention skills. Realistic. + +**Tests:** Conceptual (serving runtime, InferenceService, storage, GPU/resource, verification). Fair. + +**Mock MCP:** Rhoai mock has serving runtimes, inference services, deploy_model. Good. + +**Remarks:** None. + +--- + +## rh-ai-engineer__nim-setup + +**Instructions:** Clear. Set up NVIDIA NIM: prerequisites (GPU Operator, NFD), NGC auth, NIM Account. Does not mention skills. Realistic. + +**Tests:** Conceptual (GPU Operator, NFD, NGC auth, image pull secret, NIM Account). Fair. + +**Mock MCP:** Rhoai and openshift. NIM Account is a CR; agent can describe setup. Adequate. + +**Remarks:** None. + +--- + +## rh-ai-engineer__serving-runtime-config + +**Instructions:** Clear. Configure ServingRuntime: model format, container, platform integration. Does not mention skills. Realistic. + +**Tests:** Conceptual (API group, model format, multi-model, container config, platform integration). Fair. + +**Mock MCP:** Rhoai mock has list_serving_runtimes, serving runtime templates. Good. + +**Remarks:** None. + +--- + +## rh-ai-engineer__workbench-manage + +**Instructions:** Clear. Manage workbench: notebook image, resources, storage, lifecycle. Does not mention skills. Realistic. + +**Tests:** Conceptual (notebook image, resources, storage, lifecycle, data loss warning). Fair. + +**Mock MCP:** Rhoai mock may not expose workbench-specific tools (list_workbenches, create_workbench, etc.). Agent documents methodology from skill. Adequate for report-based eval. + +**Remarks:** Verify mock has workbench tools if agent is expected to call them. Otherwise methodology-only is acceptable. + +--- + +## rh-developer__containerize-deploy + +**Instructions:** Clear. Plan containerization (S2I, Dockerfile, Helm) and deployment. Does not mention skills. Realistic. + +**Tests:** Conceptual (strategy evaluation, deployment config). Fair. + +**Mock MCP:** Openshift mock with deployments, builds, projects. Good. + +**Remarks:** None. + +--- + +## rh-developer__debug-build + +**Instructions:** Clear. S2I build failing; examine config/logs, identify phase, recommend fix. Does not mention skills. Realistic. + +**Tests:** Conceptual (build config, phase, fix). Fair. + +**Mock MCP:** Openshift mock has builds with status Complete; api-service pod crashes at runtime (entry point), not during build. No failing S2I build in mock. Agent documents methodology from skill. Adequate for report-based eval. + +**Remarks:** Mock has no failing build; agent relies on skill/docs for build-debug methodology. Consider adding a failing build (e.g., failed pip install) for richer execution-based eval. + +--- + +## rh-developer__debug-container + +**Instructions:** Clear. Container failing at startup; inspect image/config, find cause, recommend fix. Does not mention skills. Realistic. + +**Tests:** Conceptual (image inspection, root cause, fix). Fair. + +**Mock MCP:** Openshift mock has containers. Adequate. + +**Remarks:** None. + +--- + +## rh-developer__debug-network + +**Instructions:** Clear. HTTP 503 via Route; trace Route→Service→Pod, find misconfiguration. Does not mention skills. Realistic. + +**Tests:** Conceptual (request path, misconfiguration, fix). Fair. + +**Mock MCP:** Openshift mock has order-system with 503 (selector mismatch). Good. + +**Remarks:** None. + +--- + +## rh-developer__debug-pipeline + +**Instructions:** Clear. Tekton PipelineRun failed; examine status, find failing task, recommend fix. Does not mention skills. Realistic. + +**Tests:** Conceptual (PipelineRun, task, fix/retry). Fair. + +**Mock MCP:** Openshift mock has pipeline data. Good. + +**Remarks:** None. + +--- + +## rh-developer__debug-pod + +**Instructions:** Clear. Pod in web-frontend namespace crashing; investigate, find cause, recommend fix. Does not mention skills. Aligned with mock (web-frontend has OOMKilled). Realistic. + +**Tests:** Conceptual (OOM/memory, exit code, previous logs, resource limits, events, remediation). Fair. + +**Mock MCP:** Openshift mock has web-frontend with OOMKilled (exit 137, 64Mi limit). Good alignment. + +**Remarks:** None. + +--- + +## rh-developer__debug-rhel + +**Instructions:** Clear. RHEL service failing; check service, SELinux, firewall, recommend fix. Does not mention skills. Realistic. + +**Tests:** Conceptual (service, SELinux, firewall, fix). Fair. + +**Mock MCP:** Uses available tools; RHEL debugging may be more doc/skill-driven. Adequate. + +**Remarks:** None. + +--- + +## rh-developer__deploy + +**Instructions:** Clear. Plan deployment: strategy, Service, Route, image, ports. Does not mention skills. Realistic. + +**Tests:** Conceptual (Deployment, Service, Route, image, ports). Fair. + +**Mock MCP:** Openshift mock has deployments, services, routes. Good. + +**Remarks:** None. + +--- + +## rh-developer__detect-project + +**Instructions:** Clear. Detect project type, language, framework from source. Does not mention skills. Realistic. + +**Tests:** Conceptual (language, framework, deployment strategy). Fair. + +**Mock MCP:** May use Read tool for source; MCP for cluster context. Adequate. + +**Remarks:** None. + +--- + +## rh-developer__helm-deploy + +**Instructions:** Clear. Plan Helm deployment: chart, values, OpenShift specifics. Does not mention skills. Realistic. + +**Tests:** Conceptual (Helm chart, values, OpenShift). Fair. + +**Mock MCP:** Openshift mock. Adequate. + +**Remarks:** None. + +--- + +## rh-developer__recommend-image + +**Instructions:** Clear. Recommend base image for project type (UBI, security, size). Does not mention skills. Realistic. + +**Tests:** Conceptual (base image, UBI, selection criteria). Fair. + +**Mock MCP:** May use project metadata from mock. Adequate. + +**Remarks:** None. + +--- + +## rh-developer__rhel-deploy + +**Instructions:** Clear. Plan RHEL deployment: systemd, SELinux, volumes, networking. Does not mention skills. Realistic. + +**Tests:** Conceptual (systemd, SELinux, volumes, networking). Fair. + +**Mock MCP:** Adequate for methodology documentation. + +**Remarks:** None. + +--- + +## rh-developer__s2i-build + +**Instructions:** Clear. Configure S2I for Python app: builder, build process, entry point. Does not mention skills. Realistic. + +**Tests:** Conceptual (builder image, entry point, BuildConfig, dependencies). Fair. + +**Mock MCP:** Openshift mock has builds, api-platform. Good. + +**Remarks:** None. + +--- + +## rh-developer__validate-environment + +**Instructions:** Clear. Validate OpenShift: connectivity, permissions, resources, readiness. Does not mention skills. Realistic. + +**Tests:** Conceptual (connectivity, permissions, resources, readiness). Fair. + +**Mock MCP:** Openshift mock. Adequate. + +**Remarks:** None. + +--- + +## rh-sre__cve-impact + +**Instructions:** Clear. Analyze CVE impact: affected systems, scope, pagination. Does not mention skills. Realistic. + +**Tests:** Conceptual (affected systems count, pagination, environment breakdown, remediation readiness, severity). Fair. + +**Mock MCP:** mock-lightspeed-mcp has 63 systems, 5 CVEs, get_cves, get_cve, get_cve_systems, get_system_cves. Realistic fleet and CVE data. Good. + +**Remarks:** None. + +--- + +## rh-sre__cve-validation + +**Instructions:** Clear. Validate CVEs: identifiers, severity, fixes, remediation status. Does not mention skills. Realistic. + +**Tests:** Conceptual (CVE validation, advisories, classification). Fair. + +**Mock MCP:** Lightspeed mock. Good. + +**Remarks:** None. + +--- + +## rh-sre__execution-summary + +**Instructions:** Minimal. "Complete the execution summary analysis." Vague but does not overfit. Agent discovers scope from skill. + +**Tests:** Conceptual (execution summary concepts). Fair. + +**Mock MCP:** AAP and Lightspeed mocks. Adequate. + +**Remarks:** Instruction could be slightly more specific (e.g., "document tools and steps used in a remediation workflow") without overfitting. + +--- + +## rh-sre__fleet-inventory + +**Instructions:** Minimal. "Complete the fleet inventory analysis." Vague but fair. Agent discovers scope from skill. + +**Tests:** Conceptual (fleet inventory concepts). Fair. + +**Mock MCP:** Lightspeed mock with 63 systems. Good. + +**Remarks:** Same as execution-summary: optional minor clarification. + +--- + +## rh-sre__job-template-creator + +**Instructions:** Minimal. "Create an AAP job template for CVE remediation." Fair. Agent discovers details from skill. + +**Tests:** Conceptual (job template creation). Fair. + +**Mock MCP:** AAP mock with job templates, projects. Good. + +**Remarks:** None. + +--- + +## rh-sre__job-template-remediation-validator + +**Instructions:** Minimal. "Validate an AAP job template for CVE remediation." Fair. + +**Tests:** Conceptual (template validation). Fair. + +**Mock MCP:** AAP mock. Good. + +**Remarks:** None. + +--- + +## rh-sre__mcp-aap-validator + +**Instructions:** Clear. Validate AAP MCP connectivity and functionality. Does not mention skills. Realistic. + +**Tests:** Conceptual (connectivity, auth, tool availability, error diagnostics, structured output). Fair. + +**Mock MCP:** AAP mock. Agent validates by calling tools. Good. + +**Remarks:** None. + +--- + +## rh-sre__mcp-lightspeed-validator + +**Instructions:** Clear. Validate Lightspeed MCP connectivity and functionality. Does not mention skills. Realistic. + +**Tests:** Conceptual (connectivity, auth, tools, diagnostics). Fair. + +**Mock MCP:** Lightspeed mock. Good. + +**Remarks:** None. + +--- + +## rh-sre__playbook-executor + +**Instructions:** Clear. Execute remediation playbook via AAP, pre-flight, monitoring. Does not mention skills. Realistic. + +**Tests:** Conceptual (pre-flight, dry run, monitoring, validation, git/source). Fair. + +**Mock MCP:** AAP mock with job templates, projects, jobs, launch. Good. + +**Remarks:** None. + +--- + +## rh-sre__playbook-generator + +**Instructions:** Minimal. "Generate a CVE remediation playbook using Red Hat Insights/Lightspeed." Fair. + +**Tests:** Conceptual (playbook generation). Fair. + +**Mock MCP:** Lightspeed mock with create_vulnerability_playbook. Good. + +**Remarks:** None. + +--- + +## rh-sre__remediation + +**Instructions:** Minimal. "Orchestrate CVE remediation from validation through execution and verification." Fair. + +**Tests:** Conceptual (remediation orchestration). Fair. + +**Mock MCP:** AAP and Lightspeed. Good. + +**Remarks:** None. + +--- + +## rh-sre__remediation-verifier + +**Instructions:** Minimal. "Verify CVE remediation was applied." Fair. + +**Tests:** Conceptual (verification). Fair. + +**Mock MCP:** Lightspeed mock. Good. + +**Remarks:** None. + +--- + +## rh-sre__system-context + +**Instructions:** Minimal. "Gather system context for remediation decisions." Fair. + +**Tests:** Conceptual (system context). Fair. + +**Mock MCP:** Lightspeed mock with system data. Good. + +**Remarks:** None. + +--- + +## rh-virt__vm-clone + +**Instructions:** Clear. Clone production-db (prod-vms) to test-db-clone (test-env). Does not mention skills. Realistic. + +**Tests:** Conceptual (cloning strategy, storage, independence). Fair. + +**Mock MCP:** mock-virt-mcp has VMs but not production-db in prod-vms. Uses virt-prod-dc1, virt-prod-dc2, etc. Agent documents methodology for the given scenario. Adequate. + +**Remarks:** Instruction VM/namespace (production-db, prod-vms) not in mock. Acceptable for methodology documentation. + +--- + +## rh-virt__vm-create + +**Instructions:** Clear. Plan VM test-vm in vm-testing. Does not mention skills. Realistic. + +**Tests:** Conceptual (VM spec, storage, error handling). Fair. + +**Mock MCP:** Virt mock. Agent can describe creation plan. Good. + +**Remarks:** test-vm and vm-testing not in mock; acceptable for planning task. + +--- + +## rh-virt__vm-delete + +**Instructions:** Clear. Plan deletion of legacy-app in decommission. Does not mention skills. Realistic. + +**Tests:** Conceptual (safety checks, scope, safeguards). Fair. + +**Mock MCP:** Virt mock. Adequate. + +**Remarks:** legacy-app and decommission not in mock; acceptable for planning. + +--- + +## rh-virt__vm-inventory + +**Instructions:** Clear. Produce VM inventory: all namespaces, status, resources, OS, IPs, organization. Does not mention skills. Realistic. + +**Tests:** Conceptual (VM status, CPU/memory, OS, network, storage, node, sort). No tool/field names. Fair. + +**Mock MCP:** mock-virt-mcp has 32 VMs across namespaces, VM/VMI, nodes, PVCs. Good. VMI may lack volumeStatus; agent can still produce inventory from VM and VMI data. + +**Remarks:** None. + +--- + +## rh-virt__vm-lifecycle-manager + +**Instructions:** Clear. Stop web-frontend, restart production-db in prod-vms. Does not mention skills. Realistic. + +**Tests:** Conceptual (lifecycle procedures, sequencing, verification). Fair. + +**Mock MCP:** Virt mock. Adequate. + +**Remarks:** web-frontend, production-db, prod-vms not in mock; acceptable for methodology. + +--- + +## rh-virt__vm-rebalance + +**Instructions:** Clear. Migrate production-db from overloaded node. Does not mention skills. Realistic. + +**Tests:** Conceptual (migration feasibility, target node, safety). Fair. + +**Mock MCP:** Virt mock has nodes and utilization. Good. + +**Remarks:** production-db not in mock; acceptable. + +--- + +## rh-virt__vm-snapshot-create + +**Instructions:** Clear. Snapshot production-db in prod-vms; prerequisites, spec, consistency. Does not mention skills. Realistic. + +**Tests:** Conceptual (prerequisites, consistency, spec, monitoring, volume check). Baseline requires "production-db" (from instruction). Fair. + +**Mock MCP:** Virt mock does not implement VirtualMachineSnapshot in resources_list. Agent documents plan from skill. Adequate for methodology documentation. + +**Remarks:** production-db and prod-vms not in mock; test expects "production-db" from instruction. Consistent. Snapshot CRs not in mock; agent works from skill. + +--- + +## rh-virt__vm-snapshot-delete + +**Instructions:** Clear. Delete snapshot production-db-backup-20240215 for production-db in prod-vms. Does not mention skills. Realistic. + +**Tests:** Conceptual (safety, confirmation, verification). Fair. + +**Mock MCP:** Virt mock. Adequate. + +**Remarks:** None. + +--- + +## rh-virt__vm-snapshot-list + +**Instructions:** Clear. List snapshots for production-db in prod-vms. Does not mention skills. Realistic. + +**Tests:** Conceptual (snapshot list, status, timestamps). Fair. + +**Mock MCP:** Virt mock. Check if VirtualMachineSnapshot is supported. Adequate. + +**Remarks:** None. + +--- + +## rh-virt__vm-snapshot-restore + +**Instructions:** Clear. Restore production-db from snapshot production-db-backup-20240301. Does not mention skills. Realistic. + +**Tests:** Conceptual (readiness, VM state, safeguards). Fair. + +**Mock MCP:** Virt mock. Adequate. + +**Remarks:** None. + +--- + +## Summary + +**Overall:** Instructions are clear, realistic, and do not mention skills. Tests use conceptual checks and avoid exact tool/field matching. Mocks are generally appropriate. + +**Notable points:** +- ai-observability: Mock uses rhoai/openshift, not ai-observability MCP; acceptable for methodology documentation. +- workbench-manage: Confirm workbench tools exist in mock if execution is expected. +- debug-build: Confirm mock includes a failing build scenario. +- rh-sre minimal instructions (execution-summary, fleet-inventory, etc.): Vague but fair; optional minor clarification. +- rh-virt: Several tasks reference VMs/namespaces (production-db, prod-vms, etc.) not in mock; acceptable for planning/methodology tasks. diff --git a/evaluation/with_skills/SKILL_PATH_FIXES.md b/evaluation/with_skills/SKILL_PATH_FIXES.md new file mode 100644 index 00000000..2daa163e --- /dev/null +++ b/evaluation/with_skills/SKILL_PATH_FIXES.md @@ -0,0 +1,180 @@ +# Per-Skill Evaluation: Skill Path Fixes + +This document records all modifications made to SKILL.md files and environment +directories to ensure paths resolve correctly when the agent runs inside the +Harbor container. + +## Container Layout + +The Dockerfile copies environment contents into: + +``` +/root/ +├── .claude/skills//SKILL.md # from environment/skills/ +├── .claude/docs/... # from environment/docs/ +├── docs/... # second copy of docs +├── .mcp.json # generated or copied +└── .mcp-servers/ # from environment/mcp-servers/ +``` + +From a SKILL.md at `/root/.claude/skills//SKILL.md`: +- `../../docs/` resolves to `/root/.claude/docs/` +- `../../../docs/` resolves to `/root/docs/` (second copy) +- `../references/` resolves to `/root/.claude/skills/references/` +- `./` resolves to `/root/.claude/skills//` + +--- + +## Fixes Applied + +### 1. rh-ai-engineer (7 tasks): Added shared `skills/references/` + +**Tasks**: ai-observability, debug-inference, ds-project-setup, model-deploy, +nim-setup, serving-runtime-config, workbench-manage + +**Problem**: SKILL.md files reference `../references/skill-conventions.md`, +`../references/live-doc-lookup.md`, and `../references/common-issues.md`. +These expect `environment/skills/references/` to exist. It was missing. + +**Fix**: Copied `agentic-collections/rh-ai-engineer/skills/references/` into +each task's `environment/skills/references/` directory. + +**Files added** (per task): +- `environment/skills/references/skill-conventions.md` +- `environment/skills/references/live-doc-lookup.md` +- `environment/skills/references/common-issues.md` + +--- + +### 2. ocp-admin__cluster-report: Added `scripts/` and `.mcp.json` + +**Problem**: SKILL.md references `../../scripts/cluster-report/assemble.py`, +`../../scripts/cluster-report/aggregate.py`, +`../../scripts/cluster-report/build-kubeconfig.py`, and `../../.mcp.json`. +None were present in the environment. + +**Fix**: Copied from `agentic-collections/ocp-admin/`: +- `scripts/cluster-report/` (6 files) into `environment/scripts/cluster-report/` +- `.mcp.json` into `environment/.mcp.json` + +--- + +### 3. rh-sre (7 tasks): Added cross-referenced skill directories + +**Problem**: Several SRE skills reference other skills via `../other-skill/SKILL.md`. +In the per-skill evaluation, only the evaluated skill is included, so cross-refs +broke. + +**Fix**: Copied the referenced skill directories from +`agentic-collections/rh-sre/skills/` into each task's `environment/skills/`: + +| Task | Added skills | +|------|-------------| +| rh-sre__cve-impact | mcp-lightspeed-validator | +| rh-sre__cve-validation | mcp-lightspeed-validator | +| rh-sre__fleet-inventory | mcp-lightspeed-validator | +| rh-sre__job-template-creator | mcp-aap-validator, playbook-executor | +| rh-sre__job-template-remediation-validator | mcp-aap-validator, playbook-executor, job-template-creator | +| rh-sre__playbook-executor | mcp-aap-validator | +| rh-sre__remediation | cve-validation | + +--- + +### 4. rh-developer (5 tasks): Added `templates/` + +**Tasks**: containerize-deploy, deploy, detect-project, helm-deploy, rhel-deploy + +**Problem**: SKILL.md files reference `templates/deployment.yaml.template`, +`templates/helm/`, `templates/systemd/`, etc. The templates directory was +not present in the environment. + +**Fix**: Copied `agentic-collections/rh-developer/templates/` into each +task's `environment/templates/` directory. + +--- + +### 5. rh-sre__cve-impact: Fixed dangling doc references (SKILL.md modified) + +**Problem**: SKILL.md referenced `insights-api.md` and `fleet-management.md` +in `../../docs/insights/`. These files do not exist in the source +agentic-collections repository. + +**Fix**: Replaced broken links with references to +`vulnerability-logic.md` (which exists at `../../docs/insights/vulnerability-logic.md` +and covers related content): + +| Original reference | Replaced with | +|-------------------|---------------| +| `../../docs/insights/insights-api.md` | `../../docs/insights/vulnerability-logic.md` | +| `../../docs/insights/fleet-management.md` | `../../docs/insights/vulnerability-logic.md` | + +Lines changed: 221-222, 252-253, 394-395 + +--- + +### 6. rh-sre__fleet-inventory: Fixed dangling doc references (SKILL.md modified) + +**Problem**: Same as cve-impact — references to non-existent `insights-api.md` +and `fleet-management.md`. + +**Fix**: Same replacement to `vulnerability-logic.md`. + +Lines changed: 101-102, 127-128, 219-220 + +--- + +### 7. rh-sre__cve-impact: Fixed path depth `../../../docs/` → `../../docs/` + +**Problem**: Two references used `../../../docs/references/` (three levels up) +instead of `../../docs/references/` (two levels up). Both paths work inside +the container (docs is at both `/root/.claude/docs/` and `/root/docs/`), but +`../../docs/` is the canonical path. + +**Fix**: Changed `../../../docs/` to `../../docs/` in two places: +- Line 23: `skill-invocation.md` +- Line 325: `lightspeed-mcp-tool-failures.md` + +--- + +### 8. rh-sre__cve-validation: Fixed path depth `../../../docs/` → `../../docs/` + +**Problem**: Same path depth issue as cve-impact. + +**Fix**: Changed `../../../docs/references/skill-invocation.md` path from +`../../../docs/` to `../../docs/`. + +Line changed: 24 + +--- + +### 9. rh-virt__vm-rebalance: Fixed citation paths (SKILL.md modified) + +**Problem**: SKILL.md uses absolute-style paths +`rh-virt/skills/vm-rebalance/REBALANCE_MANUAL.md` in agent output citation +text. These don't resolve from the skill directory. The actual Read +instructions correctly use `./REBALANCE_MANUAL.md`. + +**Fix**: Changed citation paths to use relative `./` prefix: +- `rh-virt/skills/vm-rebalance/REBALANCE_MANUAL.md` → `./REBALANCE_MANUAL.md` +- `rh-virt/skills/vm-rebalance/REBALANCE_AUTOMATIC.md` → `./REBALANCE_AUTOMATIC.md` + +Lines changed: 94, 103 + +--- + +## Remaining Non-Issues (false positives) + +| Task | Pattern | Explanation | +|------|---------|-------------| +| rh-developer__debug-rhel | `[path](/.*)? ` | SELinux fcontext regex, not a file link | +| rh-developer__rhel-deploy | `[app-name](/.*)? ` | SELinux fcontext regex, not a file link | + +These appear in `semanage fcontext` shell command examples. The markdown +link syntax parser matches them, but they are regex patterns, not file +references. + +--- + +## Validation Results + +After all fixes: **269 paths OK, 0 real broken references**. diff --git a/evaluation/with_skills/ocp-admin__cluster-report/environment/.mcp.json b/evaluation/with_skills/ocp-admin__cluster-report/environment/.mcp.json new file mode 100644 index 00000000..5cd15768 --- /dev/null +++ b/evaluation/with_skills/ocp-admin__cluster-report/environment/.mcp.json @@ -0,0 +1,20 @@ +{ + "mcpServers": { + "openshift": { + "command": "bash", + "args": [ + "-c", + "U=(); [ \"$(uname -s)\" = Linux ] && U=(--userns=keep-id:uid=65532,gid=65532); exec podman run \"${U[@]}\" --rm -i --network=host -v \"${KUBECONFIG}:/kubeconfig:ro,Z\" --entrypoint /app/kubernetes-mcp-server quay.io/ecosystem-appeng/openshift-mcp-server:latest --kubeconfig /kubeconfig --read-only --toolsets core,config" + ], + "env": { + "KUBECONFIG": "${KUBECONFIG}" + }, + "description": "Red Hat OpenShift MCP server for multi-cluster administration and reporting", + "security": { + "isolation": "container", + "network": "local", + "credentials": "env-only" + } + } + } +} diff --git a/evaluation/with_skills/ocp-admin__cluster-report/environment/Dockerfile b/evaluation/with_skills/ocp-admin__cluster-report/environment/Dockerfile new file mode 100644 index 00000000..b49ea754 --- /dev/null +++ b/evaluation/with_skills/ocp-admin__cluster-report/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-ocp-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/ocp-admin__cluster-report/environment/docs/multi-cluster-auth.md b/evaluation/with_skills/ocp-admin__cluster-report/environment/docs/multi-cluster-auth.md new file mode 100644 index 00000000..e187471b --- /dev/null +++ b/evaluation/with_skills/ocp-admin__cluster-report/environment/docs/multi-cluster-auth.md @@ -0,0 +1,248 @@ +# Multi-Cluster Authentication with Service Account Tokens + +Set up non-interactive, long-lived authentication for running `cluster-report` across many OpenShift clusters without repeated `oc login` sessions. + +## Overview + +The `cluster-report` skill requires valid kubeconfig contexts for every cluster it reports on. Interactive `oc login --web` opens a browser for each cluster and produces tokens that expire in ~24 hours which make it difficult to do at scale. + +**Solution**: Create a read-only ServiceAccount on each cluster with a non-expiring token. A builder script assembles these tokens into a single merged kubeconfig that the skill uses unchanged. + +## Prerequisites + +- `oc` or `kubectl` CLI +- `python3` (stdlib only, no extra packages) +- `cluster-admin` access on each target cluster (one-time setup only) + +## Quick Start (Automated) + +If you're currently logged into all the clusters you would like to get a report for via `oc login`: + +```bash +# Step 1: Setup — applies RBAC to each cluster, extracts SA tokens +python3 ocp-admin/scripts/cluster-report/build-kubeconfig.py setup --all-contexts + +# Step 2: Build — assembles a merged kubeconfig from the inventory +python3 ocp-admin/scripts/cluster-report/build-kubeconfig.py \ + build --clusters ~/.ocp-clusters/clusters.json --verify + +# Step 3: Use — export and run the skill +export KUBECONFIG=/tmp/cluster-report-kubeconfig +# Then in Claude Code use the skill: /cluster-report +``` + +After the one-time setup, only Steps 2–3 are needed for future report sessions. + +## Manual Setup (Per Cluster) + +If you prefer to set up each cluster individually: + +### 1. Apply RBAC + +> **Required permissions**: The manifest creates cluster-scoped resources (ClusterRole, ClusterRoleBinding), so the user applying it needs `cluster-admin` privileges. This is a one-time setup step. + +```bash +oc login +oc apply -f ocp-admin/scripts/cluster-report/cluster-reporter-rbac.yaml +``` + +This creates: + +- Namespace `cluster-reporter-system` +- ServiceAccount `cluster-reporter` with a read-only ClusterRole +- ClusterRoleBinding `cluster-reporter-binding` (binds the SA to the ClusterRole) +- Token Secret `cluster-reporter-token` (non-expiring) + +### 2. Extract the Token + +```bash +oc get secret cluster-reporter-token -n cluster-reporter-system \ + -o jsonpath='{.data.token}' | base64 -d +``` + +Save this token securely. It grants read-only access to nodes, pods, namespaces, projects, cluster version, and metrics. + +> **AI Safety**: Never display token values in conversation output. Verify tokens are set, but never print or echo their contents. + +### 3. Add to Inventory File + +Create or edit `~/.ocp-clusters/clusters.json`: + +```json +{ + "clusters": [ + { + "name": "prod-us-east", + "api_url": "https://api.prod-us-east.example.com:6443", + "token": "sha256~your-token-here" + } + ] +} +``` + +Set permissions: `chmod 600 ~/.ocp-clusters/clusters.json` + +### 4. Build Kubeconfig + +```bash +python3 ocp-admin/scripts/cluster-report/build-kubeconfig.py \ + build --clusters ~/.ocp-clusters/clusters.json --output ~/.kube/cluster-report-kubeconfig +``` + +## RBAC Permissions + +The `cluster-reporter-readonly` ClusterRole grants the minimum permissions required by the `cluster-report` skill: + + +| Resource | API Group | Verbs | Used By | +| ----------------------- | -------------------- | --------- | ------------------------------------------------------------- | +| nodes, namespaces, pods | core | get, list | `nodes_top`, `resources_list`, `namespaces_list`, `pods_list` | +| clusterversions | config.openshift.io | get | `resources_get` (OpenShift verification) | +| projects | project.openshift.io | list | `projects_list` | +| nodes, pods (metrics) | metrics.k8s.io | get, list | `nodes_top` | + + +No create, update, delete, or watch permissions are granted. + +## Clusters Inventory Format + +The inventory file (`clusters.json`) supports two token modes: + +### Inline Tokens (Simple) + +```json +{ + "clusters": [ + { + "name": "prod-us-east", + "api_url": "https://api.prod-us-east.example.com:6443", + "token": "sha256~abc123..." + } + ] +} +``` + +The file itself contains secrets — keep it out of git and set `chmod 600`. + +### Environment Variable References (More Secure) + +```json +{ + "clusters": [ + { + "name": "prod-us-east", + "api_url": "https://api.prod-us-east.example.com:6443", + "token_env": "CLUSTER_TOKEN_PROD_US_EAST" + } + ] +} +``` + +The file contains no secrets. Load tokens into environment variables from your secrets manager before running `--build`. + +### Optional: CA Certificate + +```json +{ + "clusters": [ + { + "name": "prod-us-east", + "api_url": "https://api.prod-us-east.example.com:6443", + "token": "sha256~abc123...", + "ca_cert": "/path/to/prod-us-east-ca.crt" + } + ] +} +``` + +If `ca_cert` is omitted, TLS verification is skipped (`--insecure-skip-tls-verify`). + +## Script Reference + +### `setup` Subcommand + +```bash +python3 build-kubeconfig.py setup [OPTIONS] +``` + + +| Flag | Description | Default | +| --------------------------- | ----------------------------- | ------------------------------- | +| `--all-contexts` | Setup all kubeconfig contexts | Lists contexts and exits | +| `--contexts ctx1,ctx2` | Setup only specified contexts | — | +| `--output-inventory ` | Inventory file path | `~/.ocp-clusters/clusters.json` | + + +Behavior: + +- Applies `cluster-reporter-rbac.yaml` to each cluster +- Waits up to 15 seconds for the token Secret to populate +- Extracts and saves the token to the inventory file +- Skips unreachable clusters with an error message +- Appends to existing inventory (deduplicates by name) + +### `build` Subcommand + +```bash +python3 build-kubeconfig.py build --clusters [OPTIONS] +``` + + +| Flag | Description | Default | +| ------------------- | -------------------------------- | -------------------------------- | +| `--clusters ` | Inventory file path (required) | — | +| `--output ` | Kubeconfig output path | `/tmp/cluster-report-kubeconfig` | +| `--verify` | Test each context after building | Off | + + +Behavior: + +- Reads inventory, resolves tokens (inline or env var) +- Builds kubeconfig with `kubectl config set-cluster/set-credentials/set-context` +- Partial success: continues on individual failures +- `--verify` tests each context with `cluster-info` +- Outputs JSON summary with success/error counts + +## Token Rotation + +SA token Secrets do not expire, but you may want to rotate them periodically: + +```bash +oc delete secret cluster-reporter-token -n cluster-reporter-system +oc apply -f ocp-admin/scripts/cluster-report/cluster-reporter-rbac.yaml + +oc get secret cluster-reporter-token -n cluster-reporter-system \ + -o jsonpath='{.data.token}' | base64 -d + +python3 build-kubeconfig.py build --clusters ~/.ocp-clusters/clusters.json --verify +``` + +To detect expired or invalid tokens: + +```bash +python3 build-kubeconfig.py build --clusters ~/.ocp-clusters/clusters.json --verify +``` + +## Security Best Practices + +1. **Never commit tokens to git** — add `clusters.json` to `.gitignore` +2. **File permissions** — `chmod 600` on both `clusters.json` and the generated kubeconfig +3. **Prefer `token_env`** — store actual tokens in a secrets manager, not in files +4. **Minimum RBAC** — the ClusterRole grants read-only access only +5. **Dedicated namespace** — the SA lives in `cluster-reporter-system`, not `kube-system` +6. **Generated kubeconfig is ephemeral** — `/tmp/` is fine for session use; for persistent storage use `~/.kube/` with `chmod 600` +7. **Never display tokens in AI conversations** — verify tokens are set but never print, echo, or expose their values in output + +## Troubleshooting + + +| Problem | Cause | Fix | +| ---------------------------------------- | ----------------------------------------- | ------------------------------------------------------------- | +| `--setup` skips a cluster | Not logged in or auth expired | `oc login ` first, then re-run setup | +| `--verify` fails for a cluster | Token expired or Secret deleted | Re-run `--setup --contexts ` for that cluster | +| `cluster-report` shows 401 for a cluster | Token invalid | Same as above — re-run setup for that cluster | +| `cluster-report` shows 403 | SA missing permissions | Re-apply `cluster-reporter-rbac.yaml` on that cluster | +| Token Secret not populated | Token controller slow or SA doesn't exist | Wait and retry; verify SA exists in `cluster-reporter-system` | +| `--build` says "env var not set" | Using `token_env` but env not loaded | Export the token env vars before running `--build` | + + diff --git a/evaluation/with_skills/ocp-admin__cluster-report/environment/mcp-servers/mock-ocp-mcp.py b/evaluation/with_skills/ocp-admin__cluster-report/environment/mcp-servers/mock-ocp-mcp.py new file mode 100644 index 00000000..65e0b6b5 --- /dev/null +++ b/evaluation/with_skills/ocp-admin__cluster-report/environment/mcp-servers/mock-ocp-mcp.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 + +import json +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + +CONTEXTS = [ + ("prod-us-east", "https://api.prod-us-east.example.com:6443", "OpenShift 4.16.3", 6, "high"), + ("prod-eu-west", "https://api.prod-eu-west.example.com:6443", "OpenShift 4.15.12", 4, "moderate"), + ("staging-central", "https://api.staging-central.example.com:6443", "OpenShift 4.16.1", 3, "low"), + ("dev-k8s", "https://dev-k8s.internal.example.com:6443", "Kubernetes", 2, "low"), + ("legacy-dc", "https://legacy-dc.example.com:6443", "OpenShift 4.14", 5, "unknown"), +] + +UNREACHABLE = {"legacy-dc"} +OPENSHIFT_CONTEXTS = {"prod-us-east", "prod-eu-west", "staging-central", "legacy-dc"} +NON_OPENSHIFT = {"dev-k8s"} + + +def _check_context(context): + ctx = (context or "prod-us-east").strip() + if ctx in UNREACHABLE: + raise ConnectionError(f"Connection refused to {ctx}") + valid = {c[0] for c in CONTEXTS} + if ctx not in valid: + raise ValueError(f"Unknown context: {ctx}") + return ctx + + +def _format_tabular(headers, rows): + if not headers or not rows: + return "" + widths = [len(h) for h in headers] + for row in rows: + for i, h in enumerate(headers): + val = str(row.get(h, "")) + widths[i] = max(widths[i], len(val)) + lines = [] + header_line = "".join(h.ljust(w + 2) for h, w in zip(headers, widths)) + lines.append(header_line.rstrip()) + for row in rows: + line = "".join(str(row.get(h, "")).ljust(w + 2) for h, w in zip(headers, widths)) + lines.append(line.rstrip()) + return "\n".join(lines) + + +# Node data for resources_get (Node kind) +NODE_DATA = { + "prod-us-east": { + "node-us-master-1": { + "metadata": {"name": "node-us-master-1", "labels": {"node-role.kubernetes.io/master": ""}}, + "status": {"allocatable": {"cpu": "4", "memory": "16Gi", "pods": "250"}, "conditions": []}, + }, + "node-us-master-2": { + "metadata": {"name": "node-us-master-2", "labels": {"node-role.kubernetes.io/master": ""}}, + "status": {"allocatable": {"cpu": "4", "memory": "16Gi", "pods": "250"}, "conditions": []}, + }, + "node-us-master-3": { + "metadata": {"name": "node-us-master-3", "labels": {"node-role.kubernetes.io/master": ""}}, + "status": {"allocatable": {"cpu": "4", "memory": "16Gi", "pods": "250"}, "conditions": []}, + }, + "node-us-worker-1": { + "metadata": {"name": "node-us-worker-1", "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": { + "allocatable": {"cpu": "32", "memory": "128Gi", "pods": "250", "nvidia.com/gpu": "4"}, + "conditions": [], + }, + }, + "node-us-worker-2": { + "metadata": {"name": "node-us-worker-2", "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": {"allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, "conditions": []}, + }, + "node-us-worker-3": { + "metadata": {"name": "node-us-worker-3", "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": { + "allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250", "nvidia.com/gpu": "4"}, + "conditions": [], + }, + }, + }, + "prod-eu-west": { + "node-eu-master-1": { + "metadata": {"name": "node-eu-master-1", "labels": {"node-role.kubernetes.io/master": ""}}, + "status": {"allocatable": {"cpu": "4", "memory": "16Gi", "pods": "250"}, "conditions": []}, + }, + "node-eu-worker-1": { + "metadata": {"name": "node-eu-worker-1", "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": {"allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, "conditions": []}, + }, + "node-eu-worker-2": { + "metadata": {"name": "node-eu-worker-2", "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": {"allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, "conditions": []}, + }, + "node-eu-worker-3": { + "metadata": {"name": "node-eu-worker-3", "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": {"allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, "conditions": []}, + }, + }, + "staging-central": { + "node-staging-master-1": { + "metadata": {"name": "node-staging-master-1", "labels": {"node-role.kubernetes.io/master": ""}}, + "status": {"allocatable": {"cpu": "4", "memory": "16Gi", "pods": "250"}, "conditions": []}, + }, + "node-staging-worker-1": { + "metadata": {"name": "node-staging-worker-1", "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": {"allocatable": {"cpu": "8", "memory": "32Gi", "pods": "250"}, "conditions": []}, + }, + "node-staging-worker-2": { + "metadata": {"name": "node-staging-worker-2", "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": {"allocatable": {"cpu": "8", "memory": "32Gi", "pods": "250"}, "conditions": []}, + }, + }, + "dev-k8s": { + "node-dev-1": { + "metadata": {"name": "node-dev-1", "labels": {"node-role.kubernetes.io/control-plane": ""}}, + "status": {"allocatable": {"cpu": "4", "memory": "8Gi", "pods": "110"}, "conditions": []}, + }, + "node-dev-2": { + "metadata": {"name": "node-dev-2", "labels": {}}, + "status": {"allocatable": {"cpu": "4", "memory": "8Gi", "pods": "110"}, "conditions": []}, + }, + }, +} + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all kubeconfig contexts with server URLs and cluster info.""" + headers = ["CONTEXT", "SERVER", "VERSION", "NODES", "UTILIZATION"] + rows = [{"CONTEXT": c[0], "SERVER": c[1], "VERSION": c[2], "NODES": str(c[3]), "UTILIZATION": c[4]} for c in CONTEXTS] + return _format_tabular(headers, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: str | None = None, + context: str | None = None, +) -> str: + """Get a single Kubernetes resource by apiVersion, kind, and name.""" + ctx = _check_context(context) + + if apiVersion == "config.openshift.io/v1" and kind == "ClusterVersion": + if ctx in NON_OPENSHIFT: + raise ValueError("ClusterVersion not found (non-OpenShift cluster)") + versions = { + "prod-us-east": "4.16.3", + "prod-eu-west": "4.15.12", + "staging-central": "4.16.1", + "legacy-dc": "4.14", + } + ver = versions.get(ctx, "4.16.0") + return f'{{"apiVersion":"config.openshift.io/v1","kind":"ClusterVersion","metadata":{{"name":"version"}},"status":{{"desired":{{"version":"{ver}"}}}}}}' + + if apiVersion == "v1" and kind == "Node": + nodes = NODE_DATA.get(ctx, {}) + if name not in nodes: + raise ValueError(f"Node {name} not found") + return json.dumps(nodes[name]) + + raise ValueError(f"Unsupported resource: {apiVersion}/{kind}") + + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: str | None = None, + context: str | None = None, +) -> str: + """List Kubernetes resources by apiVersion and kind.""" + ctx = _check_context(context) + + if apiVersion == "v1" and kind == "Node": + nodes = NODE_DATA.get(ctx, {}) + return json.dumps(list(nodes.values())) + + if apiVersion == "v1" and kind == "Namespace": + return namespaces_list(context=ctx) + + raise ValueError(f"Unsupported list: {apiVersion}/{kind}") + + +@mcp.tool() +def nodes_top(context: str | None = None) -> str: + """Return node CPU and memory usage from Metrics Server.""" + ctx = _check_context(context) + + # prod-us-east: node-us-worker-1 (28.4/32=89%, 112.6/128=88%), node-us-worker-3 (14.2/16=89%, 56.8/64=89%) + if ctx == "prod-us-east": + rows = [ + {"NAME": "node-us-master-1", "CPU(cores)": "1.2", "MEMORY(bytes)": "4Gi"}, + {"NAME": "node-us-master-2", "CPU(cores)": "1.1", "MEMORY(bytes)": "3.8Gi"}, + {"NAME": "node-us-master-3", "CPU(cores)": "1.0", "MEMORY(bytes)": "3.6Gi"}, + {"NAME": "node-us-worker-1", "CPU(cores)": "28.4", "MEMORY(bytes)": "112.6Gi"}, + {"NAME": "node-us-worker-2", "CPU(cores)": "8.2", "MEMORY(bytes)": "32Gi"}, + {"NAME": "node-us-worker-3", "CPU(cores)": "14.2", "MEMORY(bytes)": "56.8Gi"}, + ] + elif ctx == "prod-eu-west": + rows = [ + {"NAME": "node-eu-master-1", "CPU(cores)": "0.8", "MEMORY(bytes)": "3Gi"}, + {"NAME": "node-eu-worker-1", "CPU(cores)": "6.2", "MEMORY(bytes)": "24Gi"}, + {"NAME": "node-eu-worker-2", "CPU(cores)": "5.8", "MEMORY(bytes)": "22Gi"}, + {"NAME": "node-eu-worker-3", "CPU(cores)": "7.1", "MEMORY(bytes)": "28Gi"}, + ] + elif ctx == "staging-central": + rows = [ + {"NAME": "node-staging-master-1", "CPU(cores)": "0.5", "MEMORY(bytes)": "2Gi"}, + {"NAME": "node-staging-worker-1", "CPU(cores)": "2.1", "MEMORY(bytes)": "8Gi"}, + {"NAME": "node-staging-worker-2", "CPU(cores)": "1.8", "MEMORY(bytes)": "7Gi"}, + ] + elif ctx == "dev-k8s": + rows = [ + {"NAME": "node-dev-1", "CPU(cores)": "1.2", "MEMORY(bytes)": "3Gi"}, + {"NAME": "node-dev-2", "CPU(cores)": "2.0", "MEMORY(bytes)": "5Gi"}, + ] + else: + rows = [] + + headers = ["NAME", "CPU(cores)", "MEMORY(bytes)"] + return _format_tabular(headers, rows) + + +@mcp.tool() +def pods_list(namespace: str | None = None, context: str | None = None) -> str: + """List pods across namespaces.""" + ctx = _check_context(context) + + if ctx == "prod-us-east": + rows = [ + {"NAMESPACE": "batch-jobs", "NAME": "data-pipeline-batch-abc", "STATUS": "Failed"}, + {"NAMESPACE": "batch-jobs", "NAME": "data-pipeline-batch-def", "STATUS": "Failed"}, + {"NAMESPACE": "ci-cd", "NAME": "image-builder", "STATUS": "CrashLoopBackOff"}, + {"NAMESPACE": "app-platform", "NAME": "deploy-canary", "STATUS": "Pending"}, + {"NAMESPACE": "default", "NAME": "api-server", "STATUS": "Running"}, + {"NAMESPACE": "default", "NAME": "web-frontend", "STATUS": "Running"}, + {"NAMESPACE": "openshift-monitoring", "NAME": "prometheus-0", "STATUS": "Running"}, + ] + elif ctx == "prod-eu-west": + rows = [ + {"NAMESPACE": "security", "NAME": "compliance-scanner-failed", "STATUS": "Failed"}, + {"NAMESPACE": "default", "NAME": "api-eu", "STATUS": "Running"}, + ] + elif ctx == "staging-central": + rows = [ + {"NAMESPACE": "staging-apps", "NAME": "image-pull-broken-pod", "STATUS": "ImagePullBackOff"}, + {"NAMESPACE": "default", "NAME": "staging-api", "STATUS": "Running"}, + ] + elif ctx == "dev-k8s": + rows = [ + {"NAMESPACE": "default", "NAME": "dev-pod-1", "STATUS": "Running"}, + {"NAMESPACE": "kube-system", "NAME": "coredns-xyz", "STATUS": "Running"}, + ] + else: + rows = [] + + headers = ["NAMESPACE", "NAME", "STATUS"] + return _format_tabular(headers, rows) + + +@mcp.tool() +def projects_list(context: str | None = None) -> str: + """List OpenShift projects.""" + ctx = _check_context(context) + if ctx in NON_OPENSHIFT: + raise ValueError("projects_list is OpenShift-only; use namespaces_list for vanilla Kubernetes") + + counts = {"prod-us-east": 21, "prod-eu-west": 16, "staging-central": 12, "legacy-dc": 8} + n = counts.get(ctx, 5) + rows = [{"NAME": f"project-{i}"} for i in range(1, n + 1)] + headers = ["NAME"] + return _format_tabular(headers, rows) + + +@mcp.tool() +def namespaces_list(context: str | None = None) -> str: + """List all namespaces in a cluster.""" + ctx = _check_context(context) + + if ctx == "dev-k8s": + # 6 namespaces for vanilla Kubernetes + rows = [ + {"NAME": "default"}, + {"NAME": "kube-system"}, + {"NAME": "kube-public"}, + {"NAME": "kube-node-lease"}, + {"NAME": "app-dev"}, + {"NAME": "monitoring"}, + ] + else: + # OpenShift: projects map to namespaces + counts = {"prod-us-east": 21, "prod-eu-west": 16, "staging-central": 12} + n = counts.get(ctx, 5) + rows = [{"NAME": f"project-{i}"} for i in range(1, n + 1)] + + headers = ["NAME"] + return _format_tabular(headers, rows) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/aggregate.py b/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/aggregate.py new file mode 100644 index 00000000..031f0bbe --- /dev/null +++ b/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/aggregate.py @@ -0,0 +1,601 @@ +#!/usr/bin/env python3 + +import json +import math +import re +import sys + + +def parse_cpu(value): + if value is None: + return 0.0 + s = str(value).strip() + if s.endswith("m"): + return float(s[:-1]) / 1000.0 + if s.endswith("n"): + return float(s[:-1]) / 1e9 + if s.endswith("u"): + return float(s[:-1]) / 1e6 + return float(s) + + +def parse_memory(value): + if value is None: + return 0.0 + s = str(value).strip() + multipliers = { + "Ki": 1024, + "Mi": 1024 ** 2, + "Gi": 1024 ** 3, + "Ti": 1024 ** 4, + "K": 1000, + "M": 1000 ** 2, + "G": 1000 ** 3, + "T": 1000 ** 4, + } + for suffix, mult in sorted(multipliers.items(), key=lambda x: -len(x[0])): + if s.endswith(suffix): + num = float(s[: -len(suffix)]) + return (num * mult) / (1024 ** 3) + return float(s) / (1024 ** 3) + + +def detect_node_role(labels): + if not labels: + return "worker" + prefix = "node-role.kubernetes.io/" + roles = [] + for key in labels: + if key.startswith(prefix): + role = key[len(prefix):] + if role: + roles.append(role) + if not roles: + return "worker" + priority = ["control-plane", "master", "infra", "worker"] + for p in priority: + if p in roles: + return p + return roles[0] + + +GPU_KEYS = ["nvidia.com/gpu", "amd.com/gpu", "intel.com/gpu"] + + +def detect_gpus(allocatable): + if not allocatable: + return 0, "" + for key in GPU_KEYS: + val = allocatable.get(key) + if val is not None: + count = int(val) + if count > 0: + return count, key + return 0, "" + + +def parse_tabular(text): + if not text or not isinstance(text, str): + return [] + + lines = text.splitlines() + non_blank = [line for line in lines if line.strip()] + if len(non_blank) < 2: + return [] + + header_line = non_blank[0] + data_lines = non_blank[1:] + + starts = [0] + i = 0 + while i < len(header_line): + if header_line[i] == " ": + space_start = i + while i < len(header_line) and header_line[i] == " ": + i += 1 + if i < len(header_line) and (i - space_start) >= 2: + starts.append(i) + else: + i += 1 + + headers = [] + for idx, start in enumerate(starts): + end = starts[idx + 1] if idx + 1 < len(starts) else len(header_line) + headers.append(header_line[start:end].strip()) + + result = [] + for line in data_lines: + row = {} + for idx, start in enumerate(starts): + end = starts[idx + 1] if idx + 1 < len(starts) else len(line) + value = line[start:end].strip() if start < len(line) else "" + row[headers[idx]] = value + result.append(row) + + return result + + +def parse_labels_string(labels_str): + if not labels_str or labels_str == "": + return {} + result = {} + for item in labels_str.split(","): + item = item.strip() + if not item: + continue + if "=" in item: + key, val = item.split("=", 1) + result[key] = val + else: + result[item] = "" + return result + + +def _col(row, name, default=""): + if name in row: + return row[name] + name_lower = name.lower() + for key in row: + if key.lower() == name_lower: + return row[key] + return default + + +def parse_pods_tabular(text): + rows = parse_tabular(text) + result = [] + for row in rows: + result.append({ + "namespace": _col(row, "NAMESPACE", "unknown"), + "name": _col(row, "NAME", "unknown"), + "status": _col(row, "STATUS", "Unknown"), + }) + return result + + +def parse_nodes_list_tabular(text): + rows = parse_tabular(text) + result = [] + for row in rows: + name = _col(row, "NAME", "unknown") + roles_str = _col(row, "ROLES", "") + labels_str = _col(row, "LABELS", "") + + labels = parse_labels_string(labels_str) + if roles_str and roles_str != "": + for role in roles_str.split(","): + role = role.strip() + if role: + label_key = f"node-role.kubernetes.io/{role}" + if label_key not in labels: + labels[label_key] = "" + + result.append({ + "metadata": {"name": name, "labels": labels}, + "status": {}, + }) + return result + + +def parse_nodes_top_tabular(text): + rows = parse_tabular(text) + result = [] + for row in rows: + result.append({ + "name": _col(row, "NAME", "unknown"), + "cpu_usage": _col(row, "CPU(cores)") or None, + "memory_usage": _col(row, "MEMORY(bytes)") or None, + }) + return result + + +def parse_projects_tabular(text): + rows = parse_tabular(text) + return [{"name": _col(row, "NAME", "unknown")} for row in rows] + + +def parse_namespaces_tabular(text): + rows = parse_tabular(text) + return [{"name": _col(row, "NAME", "unknown")} for row in rows] + + +def classify_pod_status(pod): + if isinstance(pod, dict) and "status" in pod and isinstance(pod["status"], str): + return pod["status"] + + status_obj = pod.get("status", {}) + if isinstance(status_obj, str): + return status_obj + + phase = status_obj.get("phase", "Unknown") + + container_statuses = status_obj.get("containerStatuses", []) + if not container_statuses: + container_statuses = status_obj.get("initContainerStatuses", []) + + for cs in container_statuses or []: + state = cs.get("state", {}) + waiting = state.get("waiting", {}) + reason = waiting.get("reason", "") + if reason in ( + "CrashLoopBackOff", + "ImagePullBackOff", + "ErrImagePull", + "CreateContainerError", + "CreateContainerConfigError", + "RunContainerError", + ): + return reason + + if phase == "Completed": + return "Succeeded" + + return phase + + +def aggregate_pods_by_namespace(pods, top_n=10): + if not pods: + return [] + + ns_data = {} + for pod in pods: + if "metadata" in pod: + ns = pod["metadata"].get("namespace", "unknown") + else: + ns = pod.get("namespace", "unknown") + + status = classify_pod_status(pod) + + if ns not in ns_data: + ns_data[ns] = {"namespace": ns, "pods_total": 0, "running": 0, + "pending": 0, "failed": 0, "succeeded": 0, "other": 0} + + ns_data[ns]["pods_total"] += 1 + if status == "Running": + ns_data[ns]["running"] += 1 + elif status == "Pending": + ns_data[ns]["pending"] += 1 + elif status in ("Failed", "Error"): + ns_data[ns]["failed"] += 1 + elif status in ("Succeeded", "Completed"): + ns_data[ns]["succeeded"] += 1 + else: + ns_data[ns]["other"] += 1 + + sorted_ns = sorted(ns_data.values(), key=lambda x: x["pods_total"], reverse=True) + return sorted_ns[:top_n] + + +def process_nodes(nodes_top, nodes_list): + nodes = {} + metrics_available = nodes_top is not None + + if nodes_list: + for node in nodes_list: + if isinstance(node, dict): + meta = node.get("metadata", {}) + name = meta.get("name", node.get("name", "unknown")) + labels = meta.get("labels", node.get("labels", {})) + status = node.get("status", {}) + allocatable = status.get("allocatable", {}) + capacity = status.get("capacity", {}) + + role = detect_node_role(labels) + gpu_count, gpu_type = detect_gpus(allocatable) + + cpu_total = parse_cpu(allocatable.get("cpu") or capacity.get("cpu")) + mem_total = parse_memory(allocatable.get("memory") or capacity.get("memory")) + + nodes[name] = { + "name": name, + "role": role, + "cpu_used": None, + "cpu_total": round(cpu_total, 2), + "memory_used": None, + "memory_total": round(mem_total, 2), + "gpus": gpu_count, + "gpu_type": gpu_type, + } + + if nodes_top: + for entry in nodes_top: + if isinstance(entry, dict): + name = entry.get("name", entry.get("NAME", "unknown")) + cpu_used = entry.get("cpu_usage") or entry.get("CPU(cores)") or entry.get("cpu") + mem_used = entry.get("memory_usage") or entry.get("MEMORY(bytes)") or entry.get("memory") + + if name in nodes: + if cpu_used is not None: + nodes[name]["cpu_used"] = round(parse_cpu(str(cpu_used)), 2) + if mem_used is not None: + nodes[name]["memory_used"] = round(parse_memory(str(mem_used)), 2) + else: + nodes[name] = { + "name": name, + "role": "worker", + "cpu_used": round(parse_cpu(str(cpu_used)), 2) if cpu_used else None, + "cpu_total": None, + "memory_used": round(parse_memory(str(mem_used)), 2) if mem_used else None, + "memory_total": None, + "gpus": 0, + "gpu_type": "", + } + + return list(nodes.values()), metrics_available + + +def process_cluster(cluster_data): + errors = cluster_data.get("errors", []) + nodes_top = cluster_data.get("nodes_top") + nodes_list = cluster_data.get("nodes_list") + projects = cluster_data.get("projects") + namespaces = cluster_data.get("namespaces") + pods = cluster_data.get("pods") + + if isinstance(pods, str): + pods = parse_pods_tabular(pods) + if isinstance(nodes_top, str): + nodes_top = parse_nodes_top_tabular(nodes_top) + if isinstance(nodes_list, str): + nodes_list = parse_nodes_list_tabular(nodes_list) + if isinstance(projects, str): + projects = parse_projects_tabular(projects) + if isinstance(namespaces, str): + namespaces = parse_namespaces_tabular(namespaces) + + nodes_detail, metrics_available = process_nodes(nodes_top, nodes_list) + + cpu_used = None + cpu_total = 0.0 + mem_used = None + mem_total = 0.0 + gpu_total = 0 + + for node in nodes_detail: + if node["cpu_total"] is not None: + cpu_total += node["cpu_total"] + if node["memory_total"] is not None: + mem_total += node["memory_total"] + if node["cpu_used"] is not None: + cpu_used = (cpu_used or 0.0) + node["cpu_used"] + if node["memory_used"] is not None: + mem_used = (mem_used or 0.0) + node["memory_used"] + gpu_total += node["gpus"] + + cpu_percent = None + if cpu_used is not None and cpu_total > 0: + cpu_percent = round((cpu_used / cpu_total) * 100) + + mem_percent = None + if mem_used is not None and mem_total > 0: + mem_percent = round((mem_used / mem_total) * 100) + + project_count = 0 + if projects is not None: + project_count = len(projects) if isinstance(projects, list) else 0 + elif namespaces is not None: + project_count = len(namespaces) if isinstance(namespaces, list) else 0 + + pod_status = { + "Running": 0, + "Pending": 0, + "Succeeded": 0, + "Failed": 0, + "Unknown": 0, + "CrashLoopBackOff": 0, + "ImagePullBackOff": 0, + "ErrImagePull": 0, + "Other": 0, + } + pods_running = 0 + pods_total = 0 + + if pods and isinstance(pods, list): + pods_total = len(pods) + for pod in pods: + status = classify_pod_status(pod) + if status in pod_status: + pod_status[status] += 1 + else: + pod_status["Other"] += 1 + if status == "Running": + pods_running += 1 + + top_namespaces = aggregate_pods_by_namespace(pods or []) + + return { + "overview": { + "cluster": cluster_data.get("context", "unknown"), + "server": cluster_data.get("server", "unknown"), + "node_count": len(nodes_detail), + "cpu_used_cores": round(cpu_used, 1) if cpu_used is not None else None, + "cpu_total_cores": round(cpu_total, 1), + "cpu_percent": cpu_percent, + "memory_used_gib": round(mem_used, 1) if mem_used is not None else None, + "memory_total_gib": round(mem_total, 1), + "memory_percent": mem_percent, + "gpu_total": gpu_total, + "project_count": project_count, + "pods_running": pods_running, + "pods_total": pods_total, + "metrics_available": metrics_available, + }, + "nodes": nodes_detail, + "pod_status": {k: v for k, v in pod_status.items() if v > 0}, + "top_namespaces": top_namespaces, + "errors": errors, + } + + +def compute_totals(overview_list): + totals = { + "node_count": 0, + "cpu_used_cores": None, + "cpu_total_cores": 0.0, + "memory_used_gib": None, + "memory_total_gib": 0.0, + "gpu_total": 0, + "project_count": 0, + "pods_running": 0, + "pods_total": 0, + } + + for ov in overview_list: + totals["node_count"] += ov.get("node_count", 0) + totals["cpu_total_cores"] += ov.get("cpu_total_cores", 0) + totals["memory_total_gib"] += ov.get("memory_total_gib", 0) + totals["gpu_total"] += ov.get("gpu_total", 0) + totals["project_count"] += ov.get("project_count", 0) + totals["pods_running"] += ov.get("pods_running", 0) + totals["pods_total"] += ov.get("pods_total", 0) + + if ov.get("cpu_used_cores") is not None: + totals["cpu_used_cores"] = (totals["cpu_used_cores"] or 0) + ov["cpu_used_cores"] + if ov.get("memory_used_gib") is not None: + totals["memory_used_gib"] = (totals["memory_used_gib"] or 0) + ov["memory_used_gib"] + + totals["cpu_total_cores"] = round(totals["cpu_total_cores"], 1) + totals["memory_total_gib"] = round(totals["memory_total_gib"], 1) + + if totals["cpu_used_cores"] is not None: + totals["cpu_used_cores"] = round(totals["cpu_used_cores"], 1) + if totals["memory_used_gib"] is not None: + totals["memory_used_gib"] = round(totals["memory_used_gib"], 1) + + if totals["cpu_used_cores"] is not None and totals["cpu_total_cores"] > 0: + totals["cpu_percent"] = round((totals["cpu_used_cores"] / totals["cpu_total_cores"]) * 100) + else: + totals["cpu_percent"] = None + + if totals["memory_used_gib"] is not None and totals["memory_total_gib"] > 0: + totals["memory_percent"] = round((totals["memory_used_gib"] / totals["memory_total_gib"]) * 100) + else: + totals["memory_percent"] = None + + return totals + + +def detect_attention_items(overview_list, per_cluster): + items = [] + + for ov in overview_list: + cluster = ov["cluster"] + pc = per_cluster.get(cluster, {}) + + if ov.get("cpu_percent") is not None and ov["cpu_percent"] > 85: + items.append(f"{cluster}: Cluster CPU usage at {ov['cpu_percent']}% (>85% threshold)") + + if ov.get("memory_percent") is not None and ov["memory_percent"] > 85: + items.append(f"{cluster}: Cluster memory usage at {ov['memory_percent']}% (>85% threshold)") + + for node in pc.get("nodes", []): + if (node.get("cpu_used") is not None and node.get("cpu_total") + and node["cpu_total"] > 0): + node_cpu_pct = (node["cpu_used"] / node["cpu_total"]) * 100 + if node_cpu_pct > 85: + items.append( + f"{cluster}: Node {node['name']} CPU at {round(node_cpu_pct)}% (>85%)" + ) + if (node.get("memory_used") is not None and node.get("memory_total") + and node["memory_total"] > 0): + node_mem_pct = (node["memory_used"] / node["memory_total"]) * 100 + if node_mem_pct > 85: + items.append( + f"{cluster}: Node {node['name']} memory at {round(node_mem_pct)}% (>85%)" + ) + + pod_status = pc.get("pod_status", {}) + failed = pod_status.get("Failed", 0) + pod_status.get("Error", 0) + if failed > 0: + items.append(f"{cluster}: {failed} pods in Failed/Error state") + + unknown = pod_status.get("Unknown", 0) + if unknown > 0: + items.append(f"{cluster}: {unknown} pods in Unknown state") + + pending = pod_status.get("Pending", 0) + if pending > 0: + items.append(f"{cluster}: {pending} pods in Pending state (possible resource constraints)") + + crash = pod_status.get("CrashLoopBackOff", 0) + if crash > 0: + items.append(f"{cluster}: {crash} pods in CrashLoopBackOff") + + img_pull = pod_status.get("ImagePullBackOff", 0) + pod_status.get("ErrImagePull", 0) + if img_pull > 0: + items.append(f"{cluster}: {img_pull} pods with image pull errors") + + if not ov.get("metrics_available", True): + items.append(f"{cluster}: Metrics Server not available — no CPU/memory usage data") + + for err in pc.get("errors", []): + items.append(f"{cluster}: {err}") + + return items + + +def main(): + try: + raw = sys.stdin.read() + except Exception as e: + json.dump({"error": f"Failed to read stdin: {e}"}, sys.stdout, indent=2) + sys.exit(1) + + try: + data = json.loads(raw) + except json.JSONDecodeError as e: + json.dump({"error": f"Invalid JSON input: {e}"}, sys.stdout, indent=2) + sys.exit(1) + + clusters_input = data.get("clusters", {}) + if not clusters_input: + json.dump({"error": "No clusters found in input"}, sys.stdout, indent=2) + sys.exit(1) + + overview_list = [] + per_cluster = {} + failed_clusters = [] + + for ctx_name, cluster_data in clusters_input.items(): + cluster_data.setdefault("context", ctx_name) + result = process_cluster(cluster_data) + overview_list.append(result["overview"]) + per_cluster[ctx_name] = { + "nodes": result["nodes"], + "pod_status": result["pod_status"], + "top_namespaces": result["top_namespaces"], + "errors": result["errors"], + } + if result["errors"]: + for err in result["errors"]: + failed_clusters.append({ + "context": ctx_name, + "server": cluster_data.get("server", "unknown"), + "error": err, + }) + + clusters_reported = sum( + 1 for ov in overview_list + if ov["node_count"] > 0 or ov["pods_total"] > 0 or ov["project_count"] > 0 + ) + clusters_failed = len(overview_list) - clusters_reported + + totals = compute_totals(overview_list) + attention = detect_attention_items(overview_list, per_cluster) + + output = { + "generated_at": data.get("generated_at", ""), + "clusters_reported": clusters_reported, + "clusters_failed": clusters_failed, + "overview": overview_list, + "totals": totals, + "per_cluster": per_cluster, + "attention": attention, + "failed_clusters": failed_clusters, + } + + json.dump(output, sys.stdout, indent=2) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/assemble.py b/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/assemble.py new file mode 100644 index 00000000..ee0f9f67 --- /dev/null +++ b/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/assemble.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +import json +import os +import subprocess +import sys + +DATA_FIELDS = ("nodes_top", "nodes_list", "projects", "namespaces", "pods") + + +def unwrap_persisted_output(raw_content): + try: + data = json.loads(raw_content) + except (json.JSONDecodeError, ValueError): + return raw_content + + if isinstance(data, list) and len(data) > 0: + if all(isinstance(item, dict) and "type" in item for item in data): + texts = [] + for item in data: + if item.get("type") == "text" and "text" in item: + texts.append(item["text"]) + if texts: + return "\n".join(texts) + return None + + return data + + +def resolve_file_ref(file_path): + if not os.path.exists(file_path): + return None, f"File not found: {file_path}" + + try: + with open(file_path, "r") as f: + raw = f.read() + except PermissionError: + return None, f"Permission denied reading: {file_path}" + except OSError as e: + return None, f"Error reading {file_path}: {e}" + + if not raw.strip(): + return None, f"Empty file: {file_path}" + + content = unwrap_persisted_output(raw) + + if content is None: + return None, f"No text content in envelope: {file_path}" + + return content, None + + +def resolve_cluster(cluster_data): + errors = list(cluster_data.get("errors", [])) + + for field in DATA_FIELDS: + value = cluster_data.get(field) + if isinstance(value, dict) and "$file" in value: + file_path = value["$file"] + content, error = resolve_file_ref(file_path) + if error: + cluster_data[field] = None + errors.append(error) + else: + cluster_data[field] = content + + cluster_data["errors"] = errors + return cluster_data + + +def main(): + aggregate_mode = "--aggregate" in sys.argv + + try: + raw = sys.stdin.read() + except Exception as e: + json.dump({"error": f"Failed to read stdin: {e}"}, sys.stdout, indent=2) + sys.exit(1) + + try: + manifest = json.loads(raw) + except json.JSONDecodeError as e: + json.dump({"error": f"Invalid manifest JSON: {e}"}, sys.stdout, indent=2) + sys.exit(1) + + clusters = manifest.get("clusters", {}) + for cluster_data in clusters.values(): + resolve_cluster(cluster_data) + + resolved_json = json.dumps(manifest, indent=2) + + if aggregate_mode: + script_dir = os.path.dirname(os.path.abspath(__file__)) + aggregate_script = os.path.join(script_dir, "aggregate.py") + proc = subprocess.run( + [sys.executable, aggregate_script], + input=resolved_json, + capture_output=True, + text=True, + ) + sys.stdout.write(proc.stdout) + if proc.stderr: + sys.stderr.write(proc.stderr) + sys.exit(proc.returncode) + else: + sys.stdout.write(resolved_json) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/build-kubeconfig.py b/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/build-kubeconfig.py new file mode 100644 index 00000000..a4e06bc2 --- /dev/null +++ b/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/build-kubeconfig.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python3 +"""Multi-cluster kubeconfig builder for cluster-report. + +Two subcommands: + setup Apply RBAC and extract SA tokens for clusters you're logged into + build Build a merged kubeconfig from a clusters inventory file + +Usage: + python3 build-kubeconfig.py setup [--all-contexts] [--contexts ctx1,ctx2] + [--output-inventory ] + + python3 build-kubeconfig.py build --clusters + [--output ] [--verify] + +Requires: oc or kubectl, python3 (stdlib only) +""" + +import argparse +import base64 +import json +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +RBAC_MANIFEST = SCRIPT_DIR / "cluster-reporter-rbac.yaml" + +SA_NAMESPACE = "cluster-reporter-system" +SECRET_NAME = "cluster-reporter-token" + +DEFAULT_INVENTORY = Path.home() / ".ocp-clusters" / "clusters.json" +DEFAULT_OUTPUT = Path("/tmp/cluster-report-kubeconfig") + + +def find_kube_cmd(): + """Detect oc (preferred) or kubectl in PATH.""" + if shutil.which("oc"): + return "oc" + if shutil.which("kubectl"): + print("WARNING: 'oc' not found – falling back to 'kubectl'. " + "Install the OpenShift CLI (oc) for full compatibility: " + "https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/", + file=sys.stderr) + return "kubectl" + print('{"error": "Neither oc nor kubectl found in PATH. ' + 'Install oc: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/"}', + file=sys.stderr) + sys.exit(1) + + +# --------------------------------------------------------------------------- +# Setup mode +# --------------------------------------------------------------------------- + +def run_setup(args): + kube_cmd = find_kube_cmd() + inventory_file = Path(args.output_inventory) + + if not args.skip_rbac and not RBAC_MANIFEST.is_file(): + print(f"Error: RBAC manifest not found at {RBAC_MANIFEST}", file=sys.stderr) + sys.exit(1) + + try: + all_ctx = subprocess.check_output( + [kube_cmd, "config", "get-contexts", "-o", "name"], + text=True, stderr=subprocess.DEVNULL + ).strip().splitlines() + except subprocess.CalledProcessError: + all_ctx = [] + + if not all_ctx: + print('{"error": "No kubeconfig contexts found. Log in to at least one cluster first."}', + file=sys.stderr) + sys.exit(1) + + if args.contexts: + contexts = args.contexts.split(",") + unknown = [c for c in contexts if c not in all_ctx] + if unknown: + print(f"Error: unknown context(s): {', '.join(unknown)}", file=sys.stderr) + print(f"Available: {', '.join(all_ctx)}", file=sys.stderr) + sys.exit(1) + elif args.all_contexts: + contexts = all_ctx + else: + print("Available contexts:") + for i, ctx in enumerate(all_ctx, 1): + print(f" {i}. {ctx}") + print() + print("Run with --all-contexts to setup all, or --contexts ctx1,ctx2 to select specific ones.") + sys.exit(0) + + print(f"Pre-flight: checking {len(contexts)} cluster(s)...\n") + reachable = {} + for ctx in contexts: + server = _get_server_url(kube_cmd, ctx) + if not server: + print(f" {ctx}: SKIP (no server URL in kubeconfig)") + continue + try: + subprocess.run( + [kube_cmd, "cluster-info", "--context", ctx], + capture_output=True, text=True, timeout=15, check=True + ) + reachable[ctx] = server + print(f" {ctx}: reachable ({server})") + except (subprocess.CalledProcessError, subprocess.TimeoutExpired): + print(f" {ctx}: SKIP (unreachable – try '{kube_cmd} login {server}' first)") + continue + + if not args.skip_rbac: + try: + result = subprocess.run( + [kube_cmd, "auth", "can-i", "create", "clusterroles", + "--context", ctx], + capture_output=True, text=True, timeout=10 + ) + if result.stdout.strip().lower() != "yes": + print(f" {ctx}: SKIP (insufficient permissions – " + f"cluster-admin required for RBAC setup, " + f"or use --skip-rbac if RBAC is already applied)") + del reachable[ctx] + except (subprocess.CalledProcessError, subprocess.TimeoutExpired): + print(f" {ctx}: SKIP (could not verify permissions)") + del reachable[ctx] + + if not reachable: + print("\nError: no eligible clusters found. Nothing to do.", file=sys.stderr) + sys.exit(1) + + print(f"\n{len(reachable)}/{len(contexts)} cluster(s) ready. " + f"Proceeding with setup...\n") + + inventory_file.parent.mkdir(parents=True, exist_ok=True) + + existing_by_name = {} + if inventory_file.is_file(): + try: + with open(inventory_file) as f: + existing_by_name = {c["name"]: c for c in json.load(f).get("clusters", [])} + except (json.JSONDecodeError, KeyError): + pass + + results = {"setup": [], "errors": []} + + for ctx, server in reachable.items(): + print(f"--- {ctx} ---") + print(f" Server: {server}") + + if args.skip_rbac: + print(" Skipping RBAC apply (--skip-rbac)") + else: + print(" Applying RBAC...") + try: + subprocess.run( + [kube_cmd, "apply", "-f", str(RBAC_MANIFEST), "--context", ctx], + capture_output=True, text=True, timeout=30, check=True + ) + except subprocess.CalledProcessError as e: + results["errors"].append(f"{ctx}: RBAC apply failed: {e.stderr.strip()}") + print(f" FAIL: RBAC apply failed: {e.stderr.strip()}") + continue + + print(" Waiting for token...") + token = _wait_for_token(kube_cmd, ctx) + if not token: + results["errors"].append(f"{ctx}: token not populated after 15s") + print(" FAIL: token Secret not populated") + continue + + try: + decoded_token = base64.b64decode(token).decode("utf-8") + except Exception: + decoded_token = token + + existing_by_name[ctx] = {"name": ctx, "api_url": server, "token": decoded_token} + results["setup"].append(ctx) + print(" OK: token extracted") + + with open(inventory_file, "w") as f: + json.dump({"clusters": list(existing_by_name.values())}, f, indent=2) + os.chmod(inventory_file, 0o600) + + print() + print("=" * 50) + print(f"Setup complete: {len(results['setup'])} succeeded, {len(results['errors'])} failed") + if results["errors"]: + print("Errors:") + for e in results["errors"]: + print(f" - {e}") + print(f"Inventory written to: {inventory_file}") + print() + print("Next step:") + print(f" python3 {__file__} build --clusters {inventory_file} --verify") + + json.dump(results, sys.stderr, indent=2) + + +def _get_server_url(kube_cmd, ctx): + """Resolve the API server URL for a kubeconfig context.""" + try: + server = subprocess.check_output( + [kube_cmd, "config", "view", "-o", + f'jsonpath={{.clusters[?(@.name=="{ctx}")].cluster.server}}'], + text=True, stderr=subprocess.DEVNULL + ).strip() + if server: + return server + + cluster_ref = subprocess.check_output( + [kube_cmd, "config", "view", "-o", + f'jsonpath={{.contexts[?(@.name=="{ctx}")].context.cluster}}'], + text=True, stderr=subprocess.DEVNULL + ).strip() + if cluster_ref: + return subprocess.check_output( + [kube_cmd, "config", "view", "-o", + f'jsonpath={{.clusters[?(@.name=="{cluster_ref}")].cluster.server}}'], + text=True, stderr=subprocess.DEVNULL + ).strip() or None + except subprocess.CalledProcessError: + pass + return None + + +def _wait_for_token(kube_cmd, ctx, timeout_secs=15): + """Poll for the SA token Secret to be populated.""" + for _ in range(timeout_secs): + try: + token = subprocess.check_output( + [kube_cmd, "get", "secret", SECRET_NAME, + "-n", SA_NAMESPACE, "--context", ctx, + "-o", "jsonpath={.data.token}"], + text=True, stderr=subprocess.DEVNULL, timeout=10 + ).strip() + if token: + return token + except (subprocess.CalledProcessError, subprocess.TimeoutExpired): + pass + time.sleep(1) + return None + + +# --------------------------------------------------------------------------- +# Build mode +# --------------------------------------------------------------------------- + +def run_build(args): + kube_cmd = find_kube_cmd() + clusters_file = Path(args.clusters) + output_file = Path(args.output) + + if not clusters_file.is_file(): + print(f'{{"error": "Clusters file not found: {clusters_file}"}}', file=sys.stderr) + sys.exit(1) + + with open(clusters_file) as f: + config = json.load(f) + + clusters = config.get("clusters", []) + if not clusters: + print('{"error": "No clusters in inventory file"}', file=sys.stderr) + sys.exit(1) + + output_file.unlink(missing_ok=True) + output_file.touch(mode=0o600) + + env = {**os.environ, "KUBECONFIG": str(output_file)} + errors = [] + success = 0 + + for c in clusters: + name = c.get("name", "") + api_url = c.get("api_url", "") + + if not name or not api_url: + errors.append(f"Entry missing name or api_url: {c}") + continue + + token = _resolve_token(c, errors) + if token is None: + continue + + ca_args = (["--certificate-authority", c["ca_cert"]] + if c.get("ca_cert") + else ["--insecure-skip-tls-verify=true"]) + try: + subprocess.run( + [kube_cmd, "config", "set-cluster", name, "--server", api_url] + ca_args, + check=True, capture_output=True, env=env + ) + except subprocess.CalledProcessError as e: + errors.append(f"{name}: set-cluster failed: {e.stderr.decode().strip()}") + continue + + try: + subprocess.run( + [kube_cmd, "config", "set-credentials", f"{name}-reporter", "--token", token], + check=True, capture_output=True, env=env + ) + except subprocess.CalledProcessError as e: + errors.append(f"{name}: set-credentials failed: {e.stderr.decode().strip()}") + continue + + try: + subprocess.run( + [kube_cmd, "config", "set-context", name, + "--cluster", name, "--user", f"{name}-reporter"], + check=True, capture_output=True, env=env + ) + except subprocess.CalledProcessError as e: + errors.append(f"{name}: set-context failed: {e.stderr.decode().strip()}") + continue + + if success == 0: + subprocess.run( + [kube_cmd, "config", "use-context", name], + check=False, capture_output=True, env=env + ) + + success += 1 + + verify_results = {} + if args.verify and success > 0: + print(f"Verifying {success} context(s)...") + for c in clusters: + name = c.get("name", "") + if not name: + continue + try: + subprocess.run( + [kube_cmd, "get", "nodes", "--context", name, "-o", "name", "--no-headers"], + capture_output=True, text=True, timeout=15, check=True, env=env + ) + verify_results[name] = "ok" + print(f" {name}: OK") + except subprocess.TimeoutExpired: + verify_results[name] = "timeout" + errors.append(f"{name}: verification timed out") + print(f" {name}: TIMEOUT") + except subprocess.CalledProcessError: + verify_results[name] = "failed" + errors.append(f"{name}: verification failed (likely expired token)") + print(f" {name}: FAILED (re-run setup for this cluster)") + + result = { + "clusters_configured": success, + "clusters_failed": len(errors), + "kubeconfig": str(output_file), + "errors": errors, + } + if args.verify: + result["verification"] = verify_results + + print() + print(json.dumps(result, indent=2)) + print() + print(f"Kubeconfig written to: {output_file}") + print() + print("To use with cluster-report:") + print(f" export KUBECONFIG={output_file}") + + if success == 0: + sys.exit(1) + + +def _resolve_token(cluster_entry, errors): + """Resolve token from inline value or environment variable. Returns None on failure.""" + name = cluster_entry.get("name", "") + if "token_env" in cluster_entry: + token = os.environ.get(cluster_entry["token_env"]) + if not token: + errors.append(f"{name}: env var {cluster_entry['token_env']} not set") + return None + return token + if "token" in cluster_entry: + return cluster_entry["token"] + errors.append(f"{name}: no token or token_env specified") + return None + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser( + description="Multi-cluster kubeconfig builder for cluster-report", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + # -- setup -- + setup_parser = subparsers.add_parser( + "setup", + help="Apply RBAC to clusters you're logged into, extract SA tokens, " + "and write a clusters inventory file.", + ) + setup_parser.add_argument( + "--all-contexts", action="store_true", + help="Setup all kubeconfig contexts without prompting.", + ) + setup_parser.add_argument( + "--contexts", type=str, default=None, + help="Comma-separated list of contexts to setup.", + ) + setup_parser.add_argument( + "--skip-rbac", action="store_true", + help="Skip RBAC apply and only extract tokens (use when RBAC is already configured).", + ) + setup_parser.add_argument( + "--output-inventory", type=str, default=str(DEFAULT_INVENTORY), + help=f"Path for the clusters inventory file (default: {DEFAULT_INVENTORY}).", + ) + + # -- build -- + build_parser = subparsers.add_parser( + "build", + help="Read a clusters inventory file and build a merged kubeconfig.", + ) + build_parser.add_argument( + "--clusters", type=str, required=True, + help="Path to the clusters inventory JSON file.", + ) + build_parser.add_argument( + "--output", type=str, default=str(DEFAULT_OUTPUT), + help=f"Path for the generated kubeconfig (default: {DEFAULT_OUTPUT}).", + ) + build_parser.add_argument( + "--verify", action="store_true", + help="Test each context after building the kubeconfig.", + ) + + args = parser.parse_args() + + if args.command == "setup": + run_setup(args) + elif args.command == "build": + run_build(args) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/cluster-reporter-rbac.yaml b/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/cluster-reporter-rbac.yaml new file mode 100644 index 00000000..4fe9eeb8 --- /dev/null +++ b/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/cluster-reporter-rbac.yaml @@ -0,0 +1,72 @@ +--- +# cluster-reporter-rbac.yaml +# One-time per-cluster setup for multi-cluster reporting with SA tokens. +# Apply with: oc apply -f cluster-reporter-rbac.yaml +# +# Creates a read-only ServiceAccount with the minimum permissions required +# by the cluster-report skill. The token Secret does not expire until deleted. +# +# After applying, extract the token: +# oc get secret cluster-reporter-token -n cluster-reporter-system \ +# -o jsonpath='{.data.token}' | base64 -d + +apiVersion: v1 +kind: Namespace +metadata: + name: cluster-reporter-system + labels: + app.kubernetes.io/part-of: cluster-reporter +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cluster-reporter + namespace: cluster-reporter-system + labels: + app.kubernetes.io/part-of: cluster-reporter +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cluster-reporter-readonly + labels: + app.kubernetes.io/part-of: cluster-reporter +rules: + - apiGroups: [""] + resources: ["nodes", "namespaces", "pods"] + verbs: ["get", "list"] + - apiGroups: ["config.openshift.io"] + resources: ["clusterversions"] + verbs: ["get"] + - apiGroups: ["project.openshift.io"] + resources: ["projects"] + verbs: ["list"] + - apiGroups: ["metrics.k8s.io"] + resources: ["nodes", "pods"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cluster-reporter-binding + labels: + app.kubernetes.io/part-of: cluster-reporter +subjects: + - kind: ServiceAccount + name: cluster-reporter + namespace: cluster-reporter-system +roleRef: + kind: ClusterRole + name: cluster-reporter-readonly + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: Secret +metadata: + name: cluster-reporter-token + namespace: cluster-reporter-system + annotations: + kubernetes.io/service-account.name: cluster-reporter + labels: + app.kubernetes.io/part-of: cluster-reporter +type: kubernetes.io/service-account-token diff --git a/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/test_aggregate.py b/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/test_aggregate.py new file mode 100644 index 00000000..db3fa535 --- /dev/null +++ b/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/test_aggregate.py @@ -0,0 +1,863 @@ +#!/usr/bin/env python3 + +import json +import subprocess +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) +import aggregate + + +class TestParseCpu(unittest.TestCase): + def test_whole_cores(self): + self.assertEqual(aggregate.parse_cpu("4"), 4.0) + + def test_millicores(self): + self.assertEqual(aggregate.parse_cpu("500m"), 0.5) + + def test_millicores_whole(self): + self.assertEqual(aggregate.parse_cpu("4000m"), 4.0) + + def test_nanocores(self): + self.assertAlmostEqual(aggregate.parse_cpu("1000000000n"), 1.0) + + def test_microcores(self): + self.assertAlmostEqual(aggregate.parse_cpu("1000000u"), 1.0) + + def test_none(self): + self.assertEqual(aggregate.parse_cpu(None), 0.0) + + def test_integer(self): + self.assertEqual(aggregate.parse_cpu(8), 8.0) + + def test_fractional(self): + self.assertEqual(aggregate.parse_cpu("0.5"), 0.5) + + +class TestParseMemory(unittest.TestCase): + def test_gibibytes(self): + self.assertEqual(aggregate.parse_memory("16Gi"), 16.0) + + def test_mebibytes(self): + self.assertEqual(aggregate.parse_memory("16384Mi"), 16.0) + + def test_kibibytes(self): + self.assertAlmostEqual(aggregate.parse_memory("16777216Ki"), 16.0) + + def test_raw_bytes(self): + self.assertAlmostEqual(aggregate.parse_memory("17179869184"), 16.0, places=0) + + def test_tebibytes(self): + self.assertEqual(aggregate.parse_memory("1Ti"), 1024.0) + + def test_decimal_gigabytes(self): + val = aggregate.parse_memory("16G") + self.assertAlmostEqual(val, 16000000000 / (1024 ** 3), places=1) + + def test_none(self): + self.assertEqual(aggregate.parse_memory(None), 0.0) + + +class TestDetectNodeRole(unittest.TestCase): + def test_worker(self): + labels = {"node-role.kubernetes.io/worker": ""} + self.assertEqual(aggregate.detect_node_role(labels), "worker") + + def test_control_plane(self): + labels = {"node-role.kubernetes.io/control-plane": ""} + self.assertEqual(aggregate.detect_node_role(labels), "control-plane") + + def test_master(self): + labels = {"node-role.kubernetes.io/master": ""} + self.assertEqual(aggregate.detect_node_role(labels), "master") + + def test_infra(self): + labels = {"node-role.kubernetes.io/infra": ""} + self.assertEqual(aggregate.detect_node_role(labels), "infra") + + def test_multiple_roles_prefers_control_plane(self): + labels = { + "node-role.kubernetes.io/worker": "", + "node-role.kubernetes.io/control-plane": "", + } + self.assertEqual(aggregate.detect_node_role(labels), "control-plane") + + def test_no_role_labels(self): + labels = {"kubernetes.io/hostname": "node-1"} + self.assertEqual(aggregate.detect_node_role(labels), "worker") + + def test_empty_labels(self): + self.assertEqual(aggregate.detect_node_role({}), "worker") + + def test_none_labels(self): + self.assertEqual(aggregate.detect_node_role(None), "worker") + + +class TestDetectGpus(unittest.TestCase): + def test_nvidia_gpu(self): + alloc = {"cpu": "8", "memory": "32Gi", "nvidia.com/gpu": "2"} + count, gpu_type = aggregate.detect_gpus(alloc) + self.assertEqual(count, 2) + self.assertEqual(gpu_type, "nvidia.com/gpu") + + def test_amd_gpu(self): + alloc = {"amd.com/gpu": "4"} + count, gpu_type = aggregate.detect_gpus(alloc) + self.assertEqual(count, 4) + self.assertEqual(gpu_type, "amd.com/gpu") + + def test_intel_gpu(self): + alloc = {"intel.com/gpu": "1"} + count, gpu_type = aggregate.detect_gpus(alloc) + self.assertEqual(count, 1) + self.assertEqual(gpu_type, "intel.com/gpu") + + def test_no_gpus(self): + alloc = {"cpu": "8", "memory": "32Gi"} + count, gpu_type = aggregate.detect_gpus(alloc) + self.assertEqual(count, 0) + self.assertEqual(gpu_type, "") + + def test_zero_gpus(self): + alloc = {"nvidia.com/gpu": "0"} + count, gpu_type = aggregate.detect_gpus(alloc) + self.assertEqual(count, 0) + self.assertEqual(gpu_type, "") + + def test_none_allocatable(self): + count, gpu_type = aggregate.detect_gpus(None) + self.assertEqual(count, 0) + self.assertEqual(gpu_type, "") + + +class TestParseTabular(unittest.TestCase): + def test_basic_table(self): + text = "NAME STATUS\nnode-1 Ready\nnode-2 NotReady" + result = aggregate.parse_tabular(text) + self.assertEqual(len(result), 2) + self.assertEqual(result[0]["NAME"], "node-1") + self.assertEqual(result[0]["STATUS"], "Ready") + self.assertEqual(result[1]["NAME"], "node-2") + self.assertEqual(result[1]["STATUS"], "NotReady") + + def test_multiword_header(self): + text = "NAME DISPLAY NAME STATUS\nproj-1 My Project Active" + result = aggregate.parse_tabular(text) + self.assertEqual(len(result), 1) + self.assertIn("DISPLAY NAME", result[0]) + self.assertEqual(result[0]["DISPLAY NAME"], "My Project") + + def test_empty_input(self): + self.assertEqual(aggregate.parse_tabular(""), []) + self.assertEqual(aggregate.parse_tabular(None), []) + + def test_header_only(self): + self.assertEqual(aggregate.parse_tabular("NAME STATUS"), []) + + def test_short_data_line(self): + text = "NAME STATUS LABELS\nnode-1 Ready" + result = aggregate.parse_tabular(text) + self.assertEqual(result[0]["NAME"], "node-1") + self.assertEqual(result[0]["STATUS"], "Ready") + self.assertEqual(result[0]["LABELS"], "") + + def test_varying_column_widths(self): + text = "NAME STATUS AGE\nshort OK 1d\nvery-long Fail 30d" + result = aggregate.parse_tabular(text) + self.assertEqual(result[0]["NAME"], "short") + self.assertEqual(result[1]["NAME"], "very-long") + self.assertEqual(result[0]["STATUS"], "OK") + self.assertEqual(result[1]["STATUS"], "Fail") + + def test_blank_lines_skipped(self): + text = "NAME STATUS\n\nnode-1 Ready\n\n" + result = aggregate.parse_tabular(text) + self.assertEqual(len(result), 1) + + def test_real_mcp_pod_header(self): + text = ( + "NAMESPACE APIVERSION KIND NAME " + "READY STATUS RESTARTS AGE\n" + "openshift-dns v1 Pod dns-default-abc12 " + "1/1 Running 0 5d\n" + "aistor v1 Pod webhook-69496784f7 " + "0/1 ErrImagePull 0 4d" + ) + result = aggregate.parse_tabular(text) + self.assertEqual(len(result), 2) + self.assertEqual(result[0]["NAMESPACE"], "openshift-dns") + self.assertEqual(result[0]["STATUS"], "Running") + self.assertEqual(result[1]["STATUS"], "ErrImagePull") + + +class TestParseLabelsString(unittest.TestCase): + def test_basic_labels(self): + result = aggregate.parse_labels_string( + "node-role.kubernetes.io/worker=,kubernetes.io/hostname=node-1" + ) + self.assertEqual(result["node-role.kubernetes.io/worker"], "") + self.assertEqual(result["kubernetes.io/hostname"], "node-1") + + def test_empty_string(self): + self.assertEqual(aggregate.parse_labels_string(""), {}) + + def test_none(self): + self.assertEqual(aggregate.parse_labels_string(None), {}) + + def test_label_with_value(self): + result = aggregate.parse_labels_string("beta.kubernetes.io/arch=amd64") + self.assertEqual(result["beta.kubernetes.io/arch"], "amd64") + + def test_none_literal(self): + self.assertEqual(aggregate.parse_labels_string(""), {}) + + +class TestParsePodsTabular(unittest.TestCase): + def test_basic_pods(self): + text = ( + "NAMESPACE APIVERSION KIND NAME " + "READY STATUS RESTARTS AGE\n" + "openshift-mon v1 Pod prometheus-0 " + "1/1 Running 0 5d\n" + "default v1 Pod failing-pod " + "0/1 CrashLoopBackOff 15 1d" + ) + result = aggregate.parse_pods_tabular(text) + self.assertEqual(len(result), 2) + self.assertEqual(result[0]["namespace"], "openshift-mon") + self.assertEqual(result[0]["status"], "Running") + self.assertEqual(result[1]["namespace"], "default") + self.assertEqual(result[1]["status"], "CrashLoopBackOff") + + def test_empty_input(self): + self.assertEqual(aggregate.parse_pods_tabular(""), []) + + def test_various_statuses(self): + for status in ["Running", "Pending", "Failed", "Succeeded", + "ErrImagePull", "ImagePullBackOff", "Completed"]: + text = f"NAMESPACE NAME STATUS\ndefault pod-x {status}" + result = aggregate.parse_pods_tabular(text) + self.assertEqual(result[0]["status"], status, f"Failed for {status}") + + +class TestParseNodesListTabular(unittest.TestCase): + def test_basic_nodes(self): + text = ( + "APIVERSION KIND NAME STATUS ROLES AGE " + "VERSION LABELS\n" + "v1 Node worker-0 Ready worker 30d " + "v1.28 node-role.kubernetes.io/worker=,kubernetes.io/hostname=worker-0" + ) + result = aggregate.parse_nodes_list_tabular(text) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["metadata"]["name"], "worker-0") + self.assertIn("node-role.kubernetes.io/worker", + result[0]["metadata"]["labels"]) + + def test_role_from_roles_column(self): + text = ( + "APIVERSION KIND NAME STATUS ROLES AGE " + "VERSION LABELS\n" + "v1 Node master-0 Ready control-plane 30d " + "v1.28 kubernetes.io/hostname=master-0" + ) + result = aggregate.parse_nodes_list_tabular(text) + labels = result[0]["metadata"]["labels"] + role = aggregate.detect_node_role(labels) + self.assertEqual(role, "control-plane") + + def test_no_allocatable_data(self): + text = "APIVERSION KIND NAME STATUS ROLES LABELS\nv1 Node n1 Ready worker app=test" + result = aggregate.parse_nodes_list_tabular(text) + self.assertEqual(result[0]["status"], {}) + + +class TestParseNodesTopTabular(unittest.TestCase): + def test_basic_top(self): + text = ( + "NAME CPU(cores) CPU(%) MEMORY(bytes) MEMORY(%)\n" + "node-1 4000m 50% 16Gi 50%\n" + "node-2 2000m 25% 8Gi 25%" + ) + result = aggregate.parse_nodes_top_tabular(text) + self.assertEqual(len(result), 2) + self.assertEqual(result[0]["name"], "node-1") + self.assertEqual(result[0]["cpu_usage"], "4000m") + self.assertEqual(result[0]["memory_usage"], "16Gi") + self.assertEqual(result[1]["name"], "node-2") + + def test_empty(self): + self.assertEqual(aggregate.parse_nodes_top_tabular(""), []) + + +class TestParseProjectsTabular(unittest.TestCase): + def test_basic(self): + text = ( + "APIVERSION KIND NAME DISPLAY NAME STATUS LABELS\n" + "project.openshift.io/v1 Project my-project My Project Active app=test\n" + "project.openshift.io/v1 Project default Active " + ) + result = aggregate.parse_projects_tabular(text) + self.assertEqual(len(result), 2) + self.assertEqual(result[0]["name"], "my-project") + self.assertEqual(result[1]["name"], "default") + + def test_empty(self): + self.assertEqual(aggregate.parse_projects_tabular(""), []) + + +class TestParseNamespacesTabular(unittest.TestCase): + def test_basic(self): + text = ( + "APIVERSION KIND NAME STATUS AGE LABELS\n" + "v1 Namespace kube-system Active 90d kubernetes.io/metadata.name=kube-system\n" + "v1 Namespace default Active 90d kubernetes.io/metadata.name=default" + ) + result = aggregate.parse_namespaces_tabular(text) + self.assertEqual(len(result), 2) + self.assertEqual(result[0]["name"], "kube-system") + self.assertEqual(result[1]["name"], "default") + + +class TestClassifyPodStatus(unittest.TestCase): + def test_running(self): + pod = {"status": {"phase": "Running", "containerStatuses": []}} + self.assertEqual(aggregate.classify_pod_status(pod), "Running") + + def test_pending(self): + pod = {"status": {"phase": "Pending"}} + self.assertEqual(aggregate.classify_pod_status(pod), "Pending") + + def test_succeeded(self): + pod = {"status": {"phase": "Succeeded"}} + self.assertEqual(aggregate.classify_pod_status(pod), "Succeeded") + + def test_failed(self): + pod = {"status": {"phase": "Failed"}} + self.assertEqual(aggregate.classify_pod_status(pod), "Failed") + + def test_completed_maps_to_succeeded(self): + pod = {"status": {"phase": "Completed"}} + self.assertEqual(aggregate.classify_pod_status(pod), "Succeeded") + + def test_crashloopbackoff_override(self): + pod = { + "status": { + "phase": "Running", + "containerStatuses": [ + {"state": {"waiting": {"reason": "CrashLoopBackOff"}}} + ], + } + } + self.assertEqual(aggregate.classify_pod_status(pod), "CrashLoopBackOff") + + def test_imagepullbackoff_override(self): + pod = { + "status": { + "phase": "Pending", + "containerStatuses": [ + {"state": {"waiting": {"reason": "ImagePullBackOff"}}} + ], + } + } + self.assertEqual(aggregate.classify_pod_status(pod), "ImagePullBackOff") + + def test_errimagepull(self): + pod = { + "status": { + "phase": "Pending", + "containerStatuses": [ + {"state": {"waiting": {"reason": "ErrImagePull"}}} + ], + } + } + self.assertEqual(aggregate.classify_pod_status(pod), "ErrImagePull") + + def test_flat_status_string(self): + pod = {"name": "my-pod", "namespace": "default", "status": "CrashLoopBackOff"} + self.assertEqual(aggregate.classify_pod_status(pod), "CrashLoopBackOff") + + def test_unknown_default(self): + pod = {"status": {}} + self.assertEqual(aggregate.classify_pod_status(pod), "Unknown") + + +class TestAggregatePodsByNamespace(unittest.TestCase): + def test_basic_aggregation(self): + pods = [ + {"metadata": {"namespace": "ns-a"}, "status": {"phase": "Running"}}, + {"metadata": {"namespace": "ns-a"}, "status": {"phase": "Running"}}, + {"metadata": {"namespace": "ns-b"}, "status": {"phase": "Pending"}}, + ] + result = aggregate.aggregate_pods_by_namespace(pods) + self.assertEqual(len(result), 2) + self.assertEqual(result[0]["namespace"], "ns-a") + self.assertEqual(result[0]["pods_total"], 2) + self.assertEqual(result[0]["running"], 2) + self.assertEqual(result[1]["namespace"], "ns-b") + self.assertEqual(result[1]["pods_total"], 1) + self.assertEqual(result[1]["pending"], 1) + + def test_top_10_limit(self): + pods = [] + for i in range(15): + for j in range(15 - i): + pods.append({ + "metadata": {"namespace": f"ns-{i:02d}"}, + "status": {"phase": "Running"}, + }) + result = aggregate.aggregate_pods_by_namespace(pods) + self.assertEqual(len(result), 10) + self.assertEqual(result[0]["pods_total"], 15) + self.assertEqual(result[9]["pods_total"], 6) + + def test_empty_pods(self): + self.assertEqual(aggregate.aggregate_pods_by_namespace([]), []) + self.assertEqual(aggregate.aggregate_pods_by_namespace(None), []) + + def test_flat_pod_structure(self): + pods = [ + {"namespace": "ns-a", "status": "Running"}, + {"namespace": "ns-a", "status": "Failed"}, + ] + result = aggregate.aggregate_pods_by_namespace(pods) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["running"], 1) + self.assertEqual(result[0]["failed"], 1) + + +class TestProcessCluster(unittest.TestCase): + def _make_cluster(self, **overrides): + base = { + "context": "test-cluster", + "server": "https://api.test.example.com:6443", + "nodes_top": None, + "nodes_list": None, + "projects": None, + "namespaces": None, + "pods": None, + "errors": [], + } + base.update(overrides) + return base + + def test_full_data(self): + cluster = self._make_cluster( + nodes_top=[ + {"name": "node-1", "cpu_usage": "4000m", "memory_usage": "16Gi"}, + ], + nodes_list=[ + { + "metadata": { + "name": "node-1", + "labels": {"node-role.kubernetes.io/worker": ""}, + }, + "status": { + "allocatable": {"cpu": "8", "memory": "32Gi", "nvidia.com/gpu": "2"}, + }, + } + ], + projects=[{"name": f"proj-{i}"} for i in range(5)], + pods=[ + {"metadata": {"namespace": "default"}, "status": {"phase": "Running"}}, + {"metadata": {"namespace": "default"}, "status": {"phase": "Running"}}, + {"metadata": {"namespace": "kube-system"}, "status": {"phase": "Failed"}}, + ], + ) + result = aggregate.process_cluster(cluster) + + ov = result["overview"] + self.assertEqual(ov["node_count"], 1) + self.assertEqual(ov["cpu_used_cores"], 4.0) + self.assertEqual(ov["cpu_total_cores"], 8.0) + self.assertEqual(ov["cpu_percent"], 50) + self.assertEqual(ov["gpu_total"], 2) + self.assertEqual(ov["project_count"], 5) + self.assertEqual(ov["pods_running"], 2) + self.assertEqual(ov["pods_total"], 3) + self.assertTrue(ov["metrics_available"]) + + self.assertEqual(result["pod_status"]["Running"], 2) + self.assertEqual(result["pod_status"]["Failed"], 1) + self.assertEqual(len(result["top_namespaces"]), 2) + + def test_no_metrics_server(self): + cluster = self._make_cluster( + nodes_top=None, + nodes_list=[ + { + "metadata": {"name": "node-1", "labels": {}}, + "status": {"allocatable": {"cpu": "8", "memory": "32Gi"}}, + } + ], + ) + result = aggregate.process_cluster(cluster) + + ov = result["overview"] + self.assertFalse(ov["metrics_available"]) + self.assertIsNone(ov["cpu_used_cores"]) + self.assertIsNone(ov["cpu_percent"]) + self.assertEqual(ov["cpu_total_cores"], 8.0) + + def test_empty_cluster(self): + cluster = self._make_cluster() + result = aggregate.process_cluster(cluster) + + ov = result["overview"] + self.assertEqual(ov["node_count"], 0) + self.assertEqual(ov["pods_total"], 0) + self.assertEqual(ov["project_count"], 0) + self.assertEqual(result["pod_status"], {}) + self.assertEqual(result["top_namespaces"], []) + + def test_namespaces_fallback(self): + cluster = self._make_cluster( + projects=None, + namespaces=[{"name": f"ns-{i}"} for i in range(3)], + ) + result = aggregate.process_cluster(cluster) + self.assertEqual(result["overview"]["project_count"], 3) + + def test_tabular_pods_input(self): + tabular_pods = ( + "NAMESPACE APIVERSION KIND NAME " + "READY STATUS RESTARTS AGE\n" + "default v1 Pod pod-1 " + "1/1 Running 0 1d\n" + "default v1 Pod pod-2 " + "1/1 Running 0 1d\n" + "kube-system v1 Pod pod-3 " + "0/1 Failed 5 3d" + ) + cluster = self._make_cluster(pods=tabular_pods) + result = aggregate.process_cluster(cluster) + self.assertEqual(result["overview"]["pods_total"], 3) + self.assertEqual(result["overview"]["pods_running"], 2) + self.assertEqual(result["pod_status"]["Running"], 2) + self.assertEqual(result["pod_status"]["Failed"], 1) + self.assertEqual(len(result["top_namespaces"]), 2) + + def test_tabular_projects_input(self): + tabular_projects = ( + "APIVERSION KIND NAME DISPLAY NAME STATUS LABELS\n" + "project.openshift.io/v1 Project proj-1 Project 1 Active \n" + "project.openshift.io/v1 Project proj-2 Project 2 Active \n" + "project.openshift.io/v1 Project proj-3 Active " + ) + cluster = self._make_cluster(projects=tabular_projects) + result = aggregate.process_cluster(cluster) + self.assertEqual(result["overview"]["project_count"], 3) + + def test_mixed_tabular_and_json(self): + tabular_pods = ( + "NAMESPACE NAME STATUS\n" + "default pod-1 Running" + ) + json_nodes = [{ + "metadata": {"name": "n1", "labels": {}}, + "status": {"allocatable": {"cpu": "4", "memory": "16Gi"}}, + }] + cluster = self._make_cluster( + pods=tabular_pods, nodes_list=json_nodes, + projects=[{"name": "proj-1"}]) + result = aggregate.process_cluster(cluster) + self.assertEqual(result["overview"]["pods_total"], 1) + self.assertEqual(result["overview"]["node_count"], 1) + self.assertEqual(result["overview"]["project_count"], 1) + + def test_tabular_nodes_list_known_limitation(self): + tabular_nodes = ( + "APIVERSION KIND NAME STATUS ROLES AGE " + "VERSION LABELS\n" + "v1 Node worker-0 Ready worker 30d " + "v1.28 node-role.kubernetes.io/worker=" + ) + cluster = self._make_cluster(nodes_list=tabular_nodes) + result = aggregate.process_cluster(cluster) + self.assertEqual(result["overview"]["node_count"], 1) + self.assertEqual(result["overview"]["cpu_total_cores"], 0.0) + self.assertEqual(result["overview"]["memory_total_gib"], 0.0) + self.assertEqual(result["overview"]["gpu_total"], 0) + self.assertEqual(result["nodes"][0]["role"], "worker") + + def test_tabular_nodes_top_with_json_nodes_list(self): + tabular_top = ( + "NAME CPU(cores) CPU(%) MEMORY(bytes) MEMORY(%)\n" + "node-1 4000m 50% 16Gi 50%" + ) + json_nodes = [{ + "metadata": {"name": "node-1", + "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": {"allocatable": {"cpu": "8", "memory": "32Gi"}}, + }] + cluster = self._make_cluster( + nodes_top=tabular_top, nodes_list=json_nodes) + result = aggregate.process_cluster(cluster) + self.assertEqual(result["overview"]["cpu_used_cores"], 4.0) + self.assertEqual(result["overview"]["cpu_total_cores"], 8.0) + self.assertTrue(result["overview"]["metrics_available"]) + + +class TestComputeTotals(unittest.TestCase): + def test_two_clusters(self): + overview = [ + { + "node_count": 3, "cpu_used_cores": 10.0, "cpu_total_cores": 24.0, + "memory_used_gib": 40.0, "memory_total_gib": 96.0, + "gpu_total": 2, "project_count": 10, + "pods_running": 50, "pods_total": 60, + }, + { + "node_count": 5, "cpu_used_cores": 20.0, "cpu_total_cores": 40.0, + "memory_used_gib": 60.0, "memory_total_gib": 160.0, + "gpu_total": 4, "project_count": 20, + "pods_running": 100, "pods_total": 120, + }, + ] + totals = aggregate.compute_totals(overview) + self.assertEqual(totals["node_count"], 8) + self.assertEqual(totals["cpu_used_cores"], 30.0) + self.assertEqual(totals["cpu_total_cores"], 64.0) + self.assertEqual(totals["cpu_percent"], 47) + self.assertEqual(totals["memory_used_gib"], 100.0) + self.assertEqual(totals["memory_total_gib"], 256.0) + self.assertEqual(totals["memory_percent"], 39) + self.assertEqual(totals["gpu_total"], 6) + self.assertEqual(totals["project_count"], 30) + self.assertEqual(totals["pods_running"], 150) + self.assertEqual(totals["pods_total"], 180) + + def test_mixed_metrics_availability(self): + overview = [ + {"node_count": 3, "cpu_used_cores": 10.0, "cpu_total_cores": 24.0, + "memory_used_gib": 40.0, "memory_total_gib": 96.0, + "gpu_total": 0, "project_count": 5, "pods_running": 20, "pods_total": 25}, + {"node_count": 2, "cpu_used_cores": None, "cpu_total_cores": 16.0, + "memory_used_gib": None, "memory_total_gib": 64.0, + "gpu_total": 0, "project_count": 3, "pods_running": 10, "pods_total": 15}, + ] + totals = aggregate.compute_totals(overview) + self.assertEqual(totals["cpu_used_cores"], 10.0) + self.assertEqual(totals["cpu_total_cores"], 40.0) + + +class TestDetectAttentionItems(unittest.TestCase): + def test_high_cpu(self): + overview = [{"cluster": "prod", "cpu_percent": 90, "memory_percent": 50, + "metrics_available": True, "server": "x"}] + per_cluster = {"prod": {"nodes": [], "pod_status": {}, "errors": []}} + items = aggregate.detect_attention_items(overview, per_cluster) + self.assertTrue(any("CPU usage at 90%" in i for i in items)) + + def test_failed_pods(self): + overview = [{"cluster": "prod", "cpu_percent": 50, "memory_percent": 50, + "metrics_available": True, "server": "x"}] + per_cluster = {"prod": {"nodes": [], "pod_status": {"Failed": 3}, "errors": []}} + items = aggregate.detect_attention_items(overview, per_cluster) + self.assertTrue(any("3 pods in Failed" in i for i in items)) + + def test_pending_pods(self): + overview = [{"cluster": "dev", "cpu_percent": 30, "memory_percent": 30, + "metrics_available": True, "server": "x"}] + per_cluster = {"dev": {"nodes": [], "pod_status": {"Pending": 5}, "errors": []}} + items = aggregate.detect_attention_items(overview, per_cluster) + self.assertTrue(any("5 pods in Pending" in i for i in items)) + + def test_crashloopbackoff(self): + overview = [{"cluster": "prod", "cpu_percent": None, "memory_percent": None, + "metrics_available": False, "server": "x"}] + per_cluster = {"prod": {"nodes": [], "pod_status": {"CrashLoopBackOff": 2}, "errors": []}} + items = aggregate.detect_attention_items(overview, per_cluster) + self.assertTrue(any("CrashLoopBackOff" in i for i in items)) + self.assertTrue(any("Metrics Server" in i for i in items)) + + def test_no_issues(self): + overview = [{"cluster": "prod", "cpu_percent": 30, "memory_percent": 40, + "metrics_available": True, "server": "x"}] + per_cluster = {"prod": {"nodes": [], "pod_status": {"Running": 10}, "errors": []}} + items = aggregate.detect_attention_items(overview, per_cluster) + self.assertEqual(items, []) + + def test_node_level_high_usage(self): + overview = [{"cluster": "prod", "cpu_percent": 50, "memory_percent": 50, + "metrics_available": True, "server": "x"}] + per_cluster = {"prod": { + "nodes": [{"name": "node-1", "cpu_used": 7.5, "cpu_total": 8.0, + "memory_used": 5.0, "memory_total": 32.0}], + "pod_status": {}, "errors": [], + }} + items = aggregate.detect_attention_items(overview, per_cluster) + self.assertTrue(any("node-1 CPU at 94%" in i for i in items)) + + +class TestFullPipeline(unittest.TestCase): + + def test_two_cluster_report(self): + input_data = { + "generated_at": "2026-03-03T14:30:00Z", + "clusters": { + "prod-us": { + "context": "prod-us", + "server": "https://api.prod-us.example.com:6443", + "nodes_top": [ + {"name": "node-1", "cpu_usage": "4000m", "memory_usage": "16Gi"}, + {"name": "node-2", "cpu_usage": "3000m", "memory_usage": "12Gi"}, + ], + "nodes_list": [ + { + "metadata": {"name": "node-1", + "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": {"allocatable": {"cpu": "8", "memory": "32Gi", + "nvidia.com/gpu": "2"}}, + }, + { + "metadata": {"name": "node-2", + "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": {"allocatable": {"cpu": "8", "memory": "32Gi"}}, + }, + ], + "projects": [{"name": f"proj-{i}"} for i in range(10)], + "pods": [ + {"metadata": {"namespace": "app"}, "status": {"phase": "Running"}} + for _ in range(8) + ] + [ + {"metadata": {"namespace": "app"}, "status": {"phase": "Failed"}} + for _ in range(2) + ], + "errors": [], + }, + "dev-eu": { + "context": "dev-eu", + "server": "https://api.dev-eu.example.com:6443", + "nodes_top": None, + "nodes_list": [ + { + "metadata": {"name": "dev-1", + "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": {"allocatable": {"cpu": "4", "memory": "16Gi"}}, + }, + ], + "projects": [{"name": f"ns-{i}"} for i in range(3)], + "pods": [ + {"metadata": {"namespace": "default"}, "status": {"phase": "Running"}} + for _ in range(5) + ], + "errors": [], + }, + }, + } + + script_path = Path(__file__).parent / "aggregate.py" + proc = subprocess.run( + [sys.executable, str(script_path)], + input=json.dumps(input_data), + capture_output=True, + text=True, + ) + self.assertEqual(proc.returncode, 0, f"Script failed: {proc.stderr}") + + output = json.loads(proc.stdout) + + self.assertEqual(output["generated_at"], "2026-03-03T14:30:00Z") + self.assertEqual(output["clusters_reported"], 2) + self.assertEqual(output["clusters_failed"], 0) + self.assertEqual(len(output["overview"]), 2) + + prod = next(o for o in output["overview"] if o["cluster"] == "prod-us") + self.assertEqual(prod["node_count"], 2) + self.assertEqual(prod["gpu_total"], 2) + self.assertTrue(prod["metrics_available"]) + self.assertEqual(prod["pods_running"], 8) + self.assertEqual(prod["pods_total"], 10) + + dev = next(o for o in output["overview"] if o["cluster"] == "dev-eu") + self.assertFalse(dev["metrics_available"]) + self.assertIsNone(dev["cpu_used_cores"]) + self.assertEqual(dev["node_count"], 1) + + self.assertTrue(any("Failed" in a for a in output["attention"])) + + self.assertEqual(output["totals"]["node_count"], 3) + self.assertEqual(output["totals"]["gpu_total"], 2) + + def test_malformed_input(self): + script_path = Path(__file__).parent / "aggregate.py" + proc = subprocess.run( + [sys.executable, str(script_path)], + input="not valid json{{{", + capture_output=True, + text=True, + ) + self.assertEqual(proc.returncode, 1) + output = json.loads(proc.stdout) + self.assertIn("error", output) + + def test_empty_clusters(self): + script_path = Path(__file__).parent / "aggregate.py" + proc = subprocess.run( + [sys.executable, str(script_path)], + input=json.dumps({"clusters": {}}), + capture_output=True, + text=True, + ) + self.assertEqual(proc.returncode, 1) + output = json.loads(proc.stdout) + self.assertIn("error", output) + + def test_tabular_input_pipeline(self): + input_data = { + "generated_at": "2026-03-03T15:00:00Z", + "clusters": { + "prod": { + "context": "prod", + "server": "https://api.prod.example.com:6443", + "nodes_top": ( + "NAME CPU(cores) CPU(%) MEMORY(bytes) MEMORY(%)\n" + "node-1 4000m 50% 16Gi 50%" + ), + "nodes_list": [ + { + "metadata": {"name": "node-1", + "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": {"allocatable": {"cpu": "8", "memory": "32Gi"}}, + } + ], + "projects": ( + "APIVERSION KIND NAME DISPLAY NAME STATUS LABELS\n" + "project.openshift.io/v1 Project proj-1 Active \n" + "project.openshift.io/v1 Project proj-2 Active " + ), + "pods": ( + "NAMESPACE NAME STATUS\n" + "default pod-1 Running\n" + "default pod-2 Running\n" + "kube-system pod-3 Pending" + ), + "namespaces": None, + "errors": [], + } + }, + } + script_path = Path(__file__).parent / "aggregate.py" + proc = subprocess.run( + [sys.executable, str(script_path)], + input=json.dumps(input_data), + capture_output=True, text=True, + ) + self.assertEqual(proc.returncode, 0, f"Script failed: {proc.stderr}") + output = json.loads(proc.stdout) + self.assertEqual(output["clusters_reported"], 1) + prod = output["overview"][0] + self.assertEqual(prod["pods_total"], 3) + self.assertEqual(prod["pods_running"], 2) + self.assertEqual(prod["project_count"], 2) + self.assertEqual(prod["cpu_used_cores"], 4.0) + self.assertEqual(prod["cpu_total_cores"], 8.0) + self.assertTrue(prod["metrics_available"]) + self.assertTrue(any("Pending" in a for a in output["attention"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/test_assemble.py b/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/test_assemble.py new file mode 100644 index 00000000..0cd3e9f8 --- /dev/null +++ b/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/test_assemble.py @@ -0,0 +1,490 @@ +#!/usr/bin/env python3 + +import json +import os +import stat +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) +import assemble + + +class TestUnwrapPersistedOutput(unittest.TestCase): + + def test_single_text_entry(self): + raw = json.dumps([{"type": "text", "text": "NAME STATUS\nnode-1 Ready"}]) + result = assemble.unwrap_persisted_output(raw) + self.assertEqual(result, "NAME STATUS\nnode-1 Ready") + + def test_multiple_text_entries(self): + raw = json.dumps([ + {"type": "text", "text": "part1"}, + {"type": "text", "text": "part2"}, + ]) + result = assemble.unwrap_persisted_output(raw) + self.assertEqual(result, "part1\npart2") + + def test_non_envelope_json_array(self): + data = [{"name": "proj-1"}, {"name": "proj-2"}] + raw = json.dumps(data) + result = assemble.unwrap_persisted_output(raw) + self.assertEqual(result, data) + + def test_envelope_with_no_text_type(self): + raw = json.dumps([{"type": "image", "data": "base64..."}]) + result = assemble.unwrap_persisted_output(raw) + self.assertIsNone(result) + + def test_mixed_types_in_envelope(self): + raw = json.dumps([ + {"type": "text", "text": "hello"}, + {"type": "image", "data": "..."}, + {"type": "text", "text": "world"}, + ]) + result = assemble.unwrap_persisted_output(raw) + self.assertEqual(result, "hello\nworld") + + def test_empty_list(self): + raw = "[]" + result = assemble.unwrap_persisted_output(raw) + self.assertEqual(result, []) + + def test_string_value(self): + raw = json.dumps("just a string") + result = assemble.unwrap_persisted_output(raw) + self.assertEqual(result, "just a string") + + def test_dict_value(self): + data = {"key": "value"} + raw = json.dumps(data) + result = assemble.unwrap_persisted_output(raw) + self.assertEqual(result, data) + + def test_non_json_returns_raw_string(self): + raw = "not valid json{{{" + result = assemble.unwrap_persisted_output(raw) + self.assertEqual(result, raw) + + def test_plain_text_tabular_passthrough(self): + raw = ( + "NAMESPACE APIVERSION KIND NAME READY STATUS\n" + "default v1 Pod web-1 1/1 Running\n" + "default v1 Pod web-2 0/1 Pending\n" + ) + result = assemble.unwrap_persisted_output(raw) + self.assertEqual(result, raw) + + def test_plain_text_oc_format_passthrough(self): + raw = "NAME STATUS ROLES AGE\nnode-1 Ready worker 5d\n" + result = assemble.unwrap_persisted_output(raw) + self.assertEqual(result, raw) + + def test_large_text_content(self): + big_text = "LINE\n" * 10000 + raw = json.dumps([{"type": "text", "text": big_text}]) + result = assemble.unwrap_persisted_output(raw) + self.assertEqual(result, big_text) + + +class TestResolveFileRef(unittest.TestCase): + + def test_valid_envelope_file(self): + content = json.dumps([{"type": "text", "text": "pod data here"}]) + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + f.write(content) + path = f.name + try: + result, error = assemble.resolve_file_ref(path) + self.assertIsNone(error) + self.assertEqual(result, "pod data here") + finally: + os.unlink(path) + + def test_valid_plain_json_file(self): + data = [{"name": "ns-1"}, {"name": "ns-2"}] + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(data, f) + path = f.name + try: + result, error = assemble.resolve_file_ref(path) + self.assertIsNone(error) + self.assertEqual(result, data) + finally: + os.unlink(path) + + def test_missing_file(self): + result, error = assemble.resolve_file_ref("/nonexistent/path/file.json") + self.assertIsNone(result) + self.assertIn("not found", error.lower()) + + def test_empty_file(self): + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + path = f.name + try: + result, error = assemble.resolve_file_ref(path) + self.assertIsNone(result) + self.assertIn("Empty", error) + finally: + os.unlink(path) + + def test_non_json_file_returns_content(self): + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: + f.write("not json{{{") + path = f.name + try: + result, error = assemble.resolve_file_ref(path) + self.assertIsNone(error) + self.assertEqual(result, "not json{{{") + finally: + os.unlink(path) + + def test_plain_text_tabular_file(self): + text = ( + "NAMESPACE APIVERSION KIND NAME READY STATUS RESTARTS AGE\n" + "default v1 Pod web-1 1/1 Running 0 1d\n" + ) + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: + f.write(text) + path = f.name + try: + result, error = assemble.resolve_file_ref(path) + self.assertIsNone(error) + self.assertEqual(result, text) + finally: + os.unlink(path) + + def test_envelope_with_no_text(self): + content = json.dumps([{"type": "image", "data": "..."}]) + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + f.write(content) + path = f.name + try: + result, error = assemble.resolve_file_ref(path) + self.assertIsNone(result) + self.assertIn("No text content", error) + finally: + os.unlink(path) + + @unittest.skipIf(os.getuid() == 0, "Cannot test permission denied as root") + def test_permission_denied(self): + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + f.write("data") + path = f.name + try: + os.chmod(path, 0o000) + result, error = assemble.resolve_file_ref(path) + self.assertIsNone(result) + self.assertIn("Permission denied", error) + finally: + os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) + os.unlink(path) + + +class TestResolveCluster(unittest.TestCase): + + def test_all_inline_passthrough(self): + cluster = { + "context": "test", + "server": "https://test:6443", + "nodes_top": "NAME CPU\nnode1 100m", + "nodes_list": "NAME STATUS\nnode1 Ready", + "projects": [{"name": "p1"}], + "namespaces": None, + "pods": "NS NAME STATUS\ndefault pod1 Running", + "errors": [], + } + original = json.loads(json.dumps(cluster)) + assemble.resolve_cluster(cluster) + self.assertEqual(cluster, original) + + def test_file_ref_resolved(self): + content = json.dumps([{"type": "text", "text": "pod data"}]) + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + f.write(content) + path = f.name + try: + cluster = { + "pods": {"$file": path}, + "nodes_top": None, + "nodes_list": None, + "projects": None, + "namespaces": None, + "errors": [], + } + assemble.resolve_cluster(cluster) + self.assertEqual(cluster["pods"], "pod data") + self.assertEqual(cluster["errors"], []) + finally: + os.unlink(path) + + def test_failed_file_ref_adds_error(self): + cluster = { + "pods": {"$file": "/nonexistent/file.json"}, + "nodes_top": None, + "nodes_list": None, + "projects": None, + "namespaces": None, + "errors": [], + } + assemble.resolve_cluster(cluster) + self.assertIsNone(cluster["pods"]) + self.assertEqual(len(cluster["errors"]), 1) + self.assertIn("not found", cluster["errors"][0].lower()) + + def test_preserves_existing_errors(self): + cluster = { + "pods": {"$file": "/nonexistent/file.json"}, + "nodes_top": None, + "nodes_list": None, + "projects": None, + "namespaces": None, + "errors": ["Metrics Server not available"], + } + assemble.resolve_cluster(cluster) + self.assertEqual(len(cluster["errors"]), 2) + self.assertEqual(cluster["errors"][0], "Metrics Server not available") + + def test_mixed_inline_file_null(self): + content = json.dumps([{"type": "text", "text": "node data"}]) + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + f.write(content) + path = f.name + try: + cluster = { + "nodes_top": None, + "nodes_list": {"$file": path}, + "projects": [{"name": "p1"}], + "namespaces": None, + "pods": "NS NAME STATUS\ndefault pod1 Running", + "errors": [], + } + assemble.resolve_cluster(cluster) + self.assertIsNone(cluster["nodes_top"]) + self.assertEqual(cluster["nodes_list"], "node data") + self.assertEqual(cluster["projects"], [{"name": "p1"}]) + self.assertEqual(cluster["pods"], "NS NAME STATUS\ndefault pod1 Running") + finally: + os.unlink(path) + + def test_non_data_fields_ignored(self): + cluster = { + "context": {"$file": "/should/not/resolve"}, + "nodes_top": None, + "nodes_list": None, + "projects": None, + "namespaces": None, + "pods": None, + "errors": [], + } + assemble.resolve_cluster(cluster) + self.assertEqual(cluster["context"], {"$file": "/should/not/resolve"}) + + +class TestFullPipeline(unittest.TestCase): + + SCRIPT = str(Path(__file__).parent / "assemble.py") + + def _run(self, input_data, extra_args=None): + cmd = [sys.executable, self.SCRIPT] + if extra_args: + cmd.extend(extra_args) + proc = subprocess.run( + cmd, + input=json.dumps(input_data), + capture_output=True, text=True, + ) + return proc + + def test_inline_passthrough(self): + manifest = { + "generated_at": "2026-01-01T00:00:00Z", + "clusters": { + "test": { + "context": "test", + "server": "https://test:6443", + "nodes_top": None, + "nodes_list": None, + "projects": [{"name": "default"}], + "namespaces": None, + "pods": "NS NAME STATUS\ndefault pod1 Running", + "errors": [], + } + }, + } + proc = self._run(manifest) + self.assertEqual(proc.returncode, 0, f"Failed: {proc.stderr}") + output = json.loads(proc.stdout) + cluster = output["clusters"]["test"] + self.assertEqual(cluster["pods"], "NS NAME STATUS\ndefault pod1 Running") + self.assertEqual(cluster["projects"], [{"name": "default"}]) + + def test_file_ref_resolution(self): + content = json.dumps([{"type": "text", "text": "resolved content"}]) + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + f.write(content) + path = f.name + try: + manifest = { + "generated_at": "2026-01-01T00:00:00Z", + "clusters": { + "test": { + "context": "test", + "server": "https://test:6443", + "nodes_top": None, + "nodes_list": None, + "projects": None, + "namespaces": None, + "pods": {"$file": path}, + "errors": [], + } + }, + } + proc = self._run(manifest) + self.assertEqual(proc.returncode, 0, f"Failed: {proc.stderr}") + output = json.loads(proc.stdout) + self.assertEqual(output["clusters"]["test"]["pods"], "resolved content") + finally: + os.unlink(path) + + def test_aggregate_flag(self): + manifest = { + "generated_at": "2026-01-01T00:00:00Z", + "clusters": { + "test": { + "context": "test", + "server": "https://test:6443", + "nodes_top": None, + "nodes_list": None, + "projects": [{"name": "default"}, {"name": "kube-system"}], + "namespaces": None, + "pods": [ + {"namespace": "default", "name": "pod1", "status": "Running"}, + {"namespace": "default", "name": "pod2", "status": "Pending"}, + ], + "errors": [], + } + }, + } + proc = self._run(manifest, extra_args=["--aggregate"]) + self.assertEqual(proc.returncode, 0, f"Failed: {proc.stderr}") + output = json.loads(proc.stdout) + self.assertIn("clusters_reported", output) + self.assertIn("overview", output) + self.assertEqual(output["clusters_reported"], 1) + self.assertEqual(output["overview"][0]["project_count"], 2) + self.assertEqual(output["overview"][0]["pods_total"], 2) + self.assertEqual(output["overview"][0]["pods_running"], 1) + + def test_malformed_manifest(self): + proc = subprocess.run( + [sys.executable, self.SCRIPT], + input="not valid json{{{", + capture_output=True, text=True, + ) + self.assertEqual(proc.returncode, 1) + output = json.loads(proc.stdout) + self.assertIn("error", output) + + def test_file_ref_error_in_pipeline(self): + manifest = { + "generated_at": "2026-01-01T00:00:00Z", + "clusters": { + "test": { + "context": "test", + "server": "https://test:6443", + "nodes_top": None, + "nodes_list": None, + "projects": None, + "namespaces": None, + "pods": {"$file": "/nonexistent/file.json"}, + "errors": [], + } + }, + } + proc = self._run(manifest) + self.assertEqual(proc.returncode, 0) + output = json.loads(proc.stdout) + cluster = output["clusters"]["test"] + self.assertIsNone(cluster["pods"]) + self.assertTrue(len(cluster["errors"]) > 0) + + def test_end_to_end_with_file_ref_and_aggregate(self): + pods_text = ( + "NAMESPACE APIVERSION KIND NAME READY STATUS RESTARTS AGE\n" + "default v1 Pod web-1 1/1 Running 0 1d\n" + "default v1 Pod web-2 1/1 Running 0 1d\n" + "kube-sys v1 Pod dns-1 0/1 Failed 3 2d\n" + ) + content = json.dumps([{"type": "text", "text": pods_text}]) + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + f.write(content) + path = f.name + try: + manifest = { + "generated_at": "2026-01-01T00:00:00Z", + "clusters": { + "prod": { + "context": "prod", + "server": "https://prod:6443", + "nodes_top": None, + "nodes_list": None, + "projects": [{"name": "default"}, {"name": "kube-sys"}], + "namespaces": None, + "pods": {"$file": path}, + "errors": [], + } + }, + } + proc = self._run(manifest, extra_args=["--aggregate"]) + self.assertEqual(proc.returncode, 0, f"Failed: {proc.stderr}") + output = json.loads(proc.stdout) + self.assertEqual(output["clusters_reported"], 1) + ov = output["overview"][0] + self.assertEqual(ov["pods_total"], 3) + self.assertEqual(ov["pods_running"], 2) + self.assertEqual(ov["project_count"], 2) + finally: + os.unlink(path) + + def test_plain_text_file_ref_with_aggregate(self): + pods_text = ( + "NAMESPACE APIVERSION KIND NAME READY STATUS RESTARTS AGE\n" + "default v1 Pod web-1 1/1 Running 0 1d\n" + "default v1 Pod web-2 0/1 Pending 0 1h\n" + ) + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: + f.write(pods_text) + path = f.name + try: + manifest = { + "generated_at": "2026-01-01T00:00:00Z", + "clusters": { + "prod": { + "context": "prod", + "server": "https://prod:6443", + "nodes_top": None, + "nodes_list": None, + "projects": [{"name": "default"}], + "namespaces": None, + "pods": {"$file": path}, + "errors": [], + } + }, + } + proc = self._run(manifest, extra_args=["--aggregate"]) + self.assertEqual(proc.returncode, 0, f"Failed: {proc.stderr}") + output = json.loads(proc.stdout) + self.assertEqual(output["clusters_reported"], 1) + ov = output["overview"][0] + self.assertEqual(ov["pods_total"], 2) + self.assertEqual(ov["pods_running"], 1) + finally: + os.unlink(path) + + +if __name__ == "__main__": + unittest.main() diff --git a/evaluation/with_skills/ocp-admin__cluster-report/environment/skills/cluster-report/SKILL.md b/evaluation/with_skills/ocp-admin__cluster-report/environment/skills/cluster-report/SKILL.md new file mode 100644 index 00000000..d9e6a429 --- /dev/null +++ b/evaluation/with_skills/ocp-admin__cluster-report/environment/skills/cluster-report/SKILL.md @@ -0,0 +1,387 @@ +--- +name: cluster-report +description: | + Generate a consolidated health report across multiple OpenShift clusters. + Verifies each kubeconfig context is a genuine OpenShift cluster before + reporting. Non-OpenShift contexts are skipped by default. + Collects node resources (CPU, memory, GPUs), namespace counts, and pod + status into a single comparison view. + Use when: + - "Show me a report across all clusters" + - "Compare cluster health" + - "Multi-cluster status overview" + - "How are my clusters doing?" + - "Include all clusters including non-OpenShift" (override default filter) + NOT for single-cluster deep-dives or troubleshooting specific pods. +model: inherit +color: cyan +metadata: + user_invocable: "true" +--- + +# Multi-Cluster Report Skill + +Generate a unified health and resource report across multiple OpenShift/Kubernetes clusters using the OpenShift MCP server's multi-cluster capabilities. + +## Prerequisites + +**Required MCP Servers**: `openshift` (configured in [.mcp.json](../../.mcp.json)) + +**Required MCP Tools** (all from `openshift` server): +- `configuration_contexts_list` — list all kubeconfig contexts and server URLs +- `resources_get` — get a single Kubernetes resource by apiVersion/kind/name +- `nodes_top` — node CPU and memory usage from Metrics Server +- `resources_list` — list Kubernetes resources by apiVersion/kind +- `namespaces_list` — list all namespaces in a cluster +- `projects_list` — list all OpenShift projects +- `pods_list` — list all pods across namespaces + +**Required Environment Variables**: `KUBECONFIG` — must contain at least one cluster context. Two or more recommended for comparison. + +**Multi-Cluster Setup**: For large-scale deployments using service account tokens instead of interactive `oc login`, see [multi-cluster-auth.md](../../docs/multi-cluster-auth.md) and the [build-kubeconfig.py](../../scripts/cluster-report/build-kubeconfig.py) helper script. + +**Helper Scripts** (Python 3, stdlib only — treat as black boxes): +- [`assemble.py`](../../scripts/cluster-report/assemble.py) — resolves `$file` references into complete raw data JSON +- [`aggregate.py`](../../scripts/cluster-report/aggregate.py) — aggregates raw data into structured report JSON + +**CRITICAL Script Rules**: +- **NEVER** read the source code of `aggregate.py` or `assemble.py` +- **NEVER** write ad-hoc Python to parse or transform MCP output +- **NEVER** manually reconstruct data already available in MCP output + +**Verification Steps:** +1. Confirm `openshift` MCP server is available in `.mcp.json` +2. Verify `KUBECONFIG` is set: `test -n "$KUBECONFIG"` (never expose path or contents) +3. If either check fails → Human Notification Protocol + +**Human Notification Protocol:** + +When prerequisites fail: +1. **Stop immediately** — do not make any MCP tool calls +2. **Report error:** + ``` + Cannot execute skill: [specific failure] + Setup: [instructions + link to .mcp.json or KUBECONFIG docs] + ``` +3. **Request decision:** "How to proceed? (setup/skip/abort)" +4. **Wait for user input** + +**Security:** Never display KUBECONFIG path, contents, or any credential values. + +## When to Use This Skill + +**Use when**: +- Comparing resource utilization across clusters +- Getting a fleet-wide health overview +- Preparing capacity planning reports + +**Do NOT use when**: +- Debugging a specific pod or workload (use `/debug-pod`) + +## Workflow + +### Step 0: Validate Environment + +Check that `KUBECONFIG` is set. **Never expose the path or contents** — only confirm it is set. If not set, stop and instruct the user to `export KUBECONFIG=/path/to/kubeconfig`. + +### Step 1: Discover and Verify Clusters + +#### Step 1a: List Contexts + +**MCP Tool**: `configuration_contexts_list` + +Collect all context names and server URLs. Do NOT present results to the user yet. + +**Expected Output**: List of context names with associated server URLs. + +**Error Handling**: +- If no contexts found: Stop and instruct user to verify KUBECONFIG points to a valid file with cluster contexts +- If tool call fails: Report MCP server connectivity issue, suggest checking `.mcp.json` configuration + +#### Step 1b: Verify OpenShift Clusters + +For **each** context discovered in Step 1a, probe for the OpenShift `ClusterVersion` resource: + +**MCP Tool**: `resources_get` + +| Parameter | Value | +|---|---| +| `apiVersion` | `config.openshift.io/v1` | +| `kind` | `ClusterVersion` | +| `name` | `version` | +| `context` | `` | + +**Classification rules**: + +| Probe Result | Classification | Default Behavior | +|---|---|---| +| Success (resource returned) | **OpenShift** — extract version from `.status.desired.version` | Include | +| 403 Forbidden | **OpenShift (unverified)** — API group exists, RBAC restricts access | Include (version shown as "unknown") | +| 404 / resource not found | **Non-OpenShift** (vanilla Kubernetes or other distribution) | Exclude | +| Timeout / connection refused / 401 | **Unreachable** | Always exclude | + +**Performance**: Issue all `resources_get` calls in parallel (one per context) since they are independent. + +#### Step 1c: Present Verification Results + +Present a categorized summary to the user: + +```markdown +## Cluster Discovery Results + +### OpenShift Clusters (will be included in report) + +| Context | Server | OpenShift Version | +|---------|--------|-------------------| +| prod-us | https://api.prod-us.example.com:6443 | 4.16.3 | +| staging | https://api.staging.example.com:6443 | 4.15.12 | + +### Non-OpenShift Clusters (excluded by default) + +| Context | Server | Reason | +|---------|--------|--------| +| dev-k8s | https://dev-k8s.example.com:6443 | No ClusterVersion resource (vanilla Kubernetes) | + +### Unreachable Clusters (excluded) + +| Context | Server | Error | +|---------|--------|-------| +| old-cluster | https://old.example.com:6443 | Connection refused | + +**Proceeding with 2 OpenShift clusters.** To include non-OpenShift clusters, say "include all clusters". +``` + +**Presentation rules**: +- Omit any section that has no entries (e.g., skip "Non-OpenShift" section if all contexts are OpenShift). +- If ALL contexts are OpenShift, simplify to: "All N contexts are verified OpenShift clusters." +- If ALL contexts are non-OpenShift, inform the user: "No OpenShift clusters found. To include non-OpenShift clusters, say 'include all clusters'." + +**User override handling**: + +If the user responds with "include all clusters", "include non-OpenShift", "report on all clusters", or any clear intent to include non-OpenShift contexts, add them back into the selected set. Unreachable clusters are always excluded. + +If the user's **original prompt** (before the skill started) already contains phrases like "all clusters", "including non-OpenShift", or "all contexts", pre-select the override and present verification results as: "Including all clusters as requested." + +Proceed automatically with the discovered OpenShift clusters unless the user explicitly requests changes. + +### Step 2: Collect Cluster Data + +For each selected cluster, pass `context=` to every tool call. Collect data using: + +| Manifest Key | MCP Tool | Extra Parameters | Fallback | +|---|---|---|---| +| `nodes_top` | `nodes_top` | — | Set null if Metrics Server unavailable | +| `nodes_list` | `resources_list` | `apiVersion=v1`, `kind=Node` | — | +| `projects` | `projects_list` | — | Use `namespaces_list` if fails | +| `pods` | `pods_list` | — | — | + +**Error policy**: Skip unreachable clusters. Set failed fields to `null` and append the error to the cluster's `errors` array. Never abort the entire report. + +#### Persist MCP Output to Files + +For each MCP tool call, **immediately save the output to a file** under `/tmp/cluster-report/`. +This ensures data is available for the assembly pipeline regardless of output size. + +**Naming convention**: `/tmp/cluster-report/-.txt` + +Use a sanitized short name for the context (e.g., `prod-us`, `dev-eu`). Create the directory first: + +```bash +mkdir -p /tmp/cluster-report +``` + +**How to save**: After each MCP tool call, use Bash to write the output to disk. `$file` references +accept **both plain text and JSON files** — no special formatting is required. + +If Claude Code auto-persisted the output to a file (shown as `persisted-output` in the tool result), +reference that file path directly. + +#### Assemble Manifest + +Write the manifest to `/tmp/cluster-report-manifest.json` with `$file` references to the saved files: + +```json +{ + "generated_at": "2026-03-03T14:30:00Z", + "clusters": { + "": { + "context": "", + "server": "", + "cluster_type": "openshift", + "openshift_version": "4.16.3", + "nodes_top": {"$file": "/tmp/cluster-report/-nodes_top.txt"} or null, + "nodes_list": {"$file": "/tmp/cluster-report/-nodes_list.txt"} or null, + "projects": {"$file": "/tmp/cluster-report/-projects.txt"} or null, + "namespaces": {"$file": "/tmp/cluster-report/-namespaces.txt"} or null, + "pods": {"$file": "/tmp/cluster-report/-pods.txt"} or null, + "errors": [""] + } + } +} +``` + +**Manifest fields from verification**: +- `cluster_type`: `"openshift"` or `"kubernetes"`. Determined during Step 1b verification. +- `openshift_version`: The OpenShift version string (e.g., `"4.16.3"`) or `null` for non-OpenShift clusters. + +Fields may also be inlined as raw text strings or set to `null` for failed/unavailable data. + +### Step 3: Aggregate Data + +Run the assembly and aggregation pipeline: + +```bash +python3 ocp-admin/scripts/cluster-report/assemble.py --aggregate < /tmp/cluster-report-manifest.json +``` + +If the pipeline exits with code 1, display the error JSON to the user and stop. + +### Step 4: Render Report + +Render the structured JSON output as markdown using this template: + +```markdown +# Multi-Cluster Report + +**Generated**: YYYY-MM-DDTHH:MM:SSZ +**Clusters**: clusters reporting + +--- + +## Cluster Overview + +| Cluster | Version | Nodes | CPU (used/total) | Memory (used/total) | GPUs | Projects | Pods (Running/Total) | +|---------|---------|-------|-------------------|---------------------|------|----------|---------------------| +| prod-us | OCP 4.16.3 | 12 | 48/96 cores (50%) | 192/384 GiB (50%) | 8 | 45 | 312/320 | +| dev-eu | OCP 4.15.12 | 4 | 8/32 cores (25%) | 32/128 GiB (25%) | 0 | 12 | 87/92 | +| **Total** | | **16** | **56/128 cores (44%)** | **224/512 GiB (44%)** | **8** | **57** | **399/412** | + +--- + +## Per-Cluster Details + +### () — OpenShift + +#### Node Resources + +| Node | Role | CPU Used | CPU Total | Memory Used | Memory Total | GPUs | +|------|------|----------|-----------|-------------|--------------|------| +| node-1 | worker | 4 cores | 8 cores | 16 GiB | 32 GiB | 2 | + +#### Pod Status + +| Status | Count | +|--------|-------| +| Running | 312 | +| Pending | 5 | +| Succeeded | 0 | +| Failed | 3 | +| Unknown | 0 | + +#### Top Namespaces (by pod count) + +| Namespace | Pods | Running | Pending | Failed | +|-----------|------|---------|---------|--------| +| openshift-monitoring | 24 | 24 | 0 | 0 | + +[Repeat for each cluster] + +--- + +## Attention Required + +[Render each item from the `attention` array] +``` + +### Step 5: Offer Next Steps + +```markdown +## Next Steps + +Would you like to: +1. **Drill down** into a specific cluster or namespace +2. **Check alerts** — query Prometheus/Alertmanager for active alerts +3. **Refresh** — re-run the report with updated data +``` + +## Dependencies + +### Required MCP Servers +- `openshift` — with multi-cluster support enabled + +### Required MCP Tools +- `configuration_contexts_list` (from openshift) — list all kubeconfig contexts and server URLs +- `resources_get` (from openshift) — get a single Kubernetes resource by apiVersion/kind/name + - Parameters: `apiVersion`, `kind`, `name`, `context` +- `nodes_top` (from openshift) — node CPU and memory usage from Metrics Server + - Parameters: `context` +- `resources_list` (from openshift) — list Kubernetes resources by apiVersion/kind + - Parameters: `apiVersion`, `kind`, `context` +- `namespaces_list` (from openshift) — list all namespaces in a cluster + - Parameters: `context` +- `projects_list` (from openshift) — list all OpenShift projects + - Parameters: `context` +- `pods_list` (from openshift) — list all pods across namespaces + - Parameters: `context` + +### Helper Scripts +- [`ocp-admin/scripts/cluster-report/assemble.py`](../../scripts/cluster-report/assemble.py) +- [`ocp-admin/scripts/cluster-report/aggregate.py`](../../scripts/cluster-report/aggregate.py) + +### Related Skills +- None currently + +### Reference Documentation +- [OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server) +- [Kubernetes MCP Server Tools](https://github.com/containers/kubernetes-mcp-server#tools) + +## Error Handling + +| Error | Behavior | +|---|---| +| ClusterVersion probe succeeds | Classify as OpenShift, include by default | +| ClusterVersion probe 404/not found | Classify as non-OpenShift, exclude by default | +| ClusterVersion probe 403 Forbidden | Classify as OpenShift (unverified), include by default with version "unknown" | +| ClusterVersion probe timeout/unreachable | Classify as unreachable, always exclude | +| All contexts are non-OpenShift | Inform user, suggest "include all clusters" override | +| User overrides to include non-OpenShift | Proceed normally; `projects_list` may fail (use `namespaces_list` fallback) | +| Cluster unreachable | Skip, continue with remaining clusters | +| Metrics Server missing | Set `nodes_top` to null, show N/A for CPU/memory usage | +| Auth expired (401) | Skip cluster, suggest: re-run `build-kubeconfig.py build --verify` or `oc login ` | +| No GPUs found | Display 0 (not an error) | +| Empty cluster | Report with all zeros (valid data) | + +## Example Usage + +### Multi-Cluster Report (Default: OpenShift Only) + +**User**: "Show me a report across all clusters" + +**Execution**: +1. Validate KUBECONFIG — OK +2. `configuration_contexts_list()` discovers: prod-us, dev-eu, dev-k8s +3. Verify each context with `resources_get(apiVersion="config.openshift.io/v1", kind="ClusterVersion", name="version", context=)` +4. Results: prod-us (OCP 4.16.3), dev-eu (OCP 4.15.12), dev-k8s (non-OpenShift) +5. Present: "2 OpenShift clusters found. dev-k8s excluded (non-OpenShift). Include all?" +6. User confirms default selection +7. Collect data for prod-us and dev-eu only +8. Write manifest with `cluster_type` and `openshift_version` fields +9. Run `assemble.py --aggregate` pipeline +10. Render report with OpenShift version column +11. Flag attention items + +### Multi-Cluster Report (Include All) + +**User**: "Report on all my clusters including non-OpenShift" + +**Execution**: +1. Validate KUBECONFIG — OK +2. `configuration_contexts_list()` discovers: prod-us, dev-eu, dev-k8s +3. Verify each context (same as above) +4. Results: prod-us (OCP 4.16.3), dev-eu (OCP 4.15.12), dev-k8s (non-OpenShift) +5. User's initial message indicates "include all" — present verification results and confirm +6. User confirms all clusters including dev-k8s +7. Collect data for all three clusters (`projects_list` fails on dev-k8s, falls back to `namespaces_list`) +8. Write manifest; dev-k8s has `cluster_type: "kubernetes"`, `openshift_version: null` +9. Run pipeline, render report +10. dev-k8s shown as "K8s" in version column diff --git a/evaluation/with_skills/ocp-admin__cluster-report/instruction.md b/evaluation/with_skills/ocp-admin__cluster-report/instruction.md new file mode 100644 index 00000000..b13ffc9a --- /dev/null +++ b/evaluation/with_skills/ocp-admin__cluster-report/instruction.md @@ -0,0 +1,17 @@ +# Cluster Report Task + +You are an OpenShift cluster administrator. Your operations lead has requested a comprehensive infrastructure health snapshot for the weekly review. Your environment has multiple cluster contexts configured. + +## Requirements +- Discover all available cluster contexts in your environment +- For each accessible OpenShift cluster, report: + - Cluster version and API server URL + - All nodes with their status (Ready/NotReady), roles, and resource utilization (CPU and memory usage vs capacity) + - All projects/namespaces with their status + - Workload counts: total pods, running vs failing, and any pods in error states +- Explicitly note any contexts that are not OpenShift clusters or could not be reached, and explain why +- Highlight any issues that need attention (unhealthy nodes, resource pressure, failing workloads) + +Use MCP tools to examine the clusters. Write the complete cluster report in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/ocp-admin__cluster-report/solution/solve.sh b/evaluation/with_skills/ocp-admin__cluster-report/solution/solve.sh new file mode 100644 index 00000000..62bd7e47 --- /dev/null +++ b/evaluation/with_skills/ocp-admin__cluster-report/solution/solve.sh @@ -0,0 +1,30 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Multi-Cluster Health Report + +## Cluster Discovery +Use configuration_contexts_list for kubeconfig contexts. Verify each with resources_get(apiVersion="config.openshift.io/v1", kind="ClusterVersion", name="version"). + +## Cluster Contexts +| Context | Type | Server | +|---------|------|--------| +| ocp-prod | OpenShift (ClusterVersion detected) | https://api.ocp-prod.example.com:6443 | + +### OpenShift Detection +Check for ClusterVersion resource: config.openshift.io/v1. Non-OpenShift contexts excluded by default. + +## Node Resources +| Node | CPU | Memory | GPUs | +|------|-----|--------|------| +| worker-01 | 16 cores (45% used) | 64Gi (60% used) | 2 | +| worker-02 | 16 cores (30% used) | 64Gi (40% used) | 0 | + +## Pod Status +| Namespace | Running | Pending | Failed | +|-----------|---------|---------|--------| +| default | 5 | 0 | 0 | +| openshift-operators | 12 | 0 | 1 | + +### Generated using assemble.py and aggregate.py helper scripts +Persist MCP output to /tmp/cluster-report/. Manifest with $file refs. Projects_list (fallback namespaces_list for non-OpenShift) +REPORT_EOF diff --git a/evaluation/with_skills/ocp-admin__cluster-report/task.toml b/evaluation/with_skills/ocp-admin__cluster-report/task.toml new file mode 100644 index 00000000..51a06299 --- /dev/null +++ b/evaluation/with_skills/ocp-admin__cluster-report/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "ocp-admin__cluster-report" +name = "ocp-admin Multi-Cluster Health Report Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["ocp-admin", "cluster-report", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/ocp-admin__cluster-report/tests/llm_judge.py b/evaluation/with_skills/ocp-admin__cluster-report/tests/llm_judge.py new file mode 100644 index 00000000..6c379f29 --- /dev/null +++ b/evaluation/with_skills/ocp-admin__cluster-report/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "clusterversion_verification", + "file": "/root/report.md", + "question": "Does the report explicitly mention probing or checking the ClusterVersion resource as the method used to determine whether each context is an OpenShift cluster?", + "reference": "A skilled report should mention 'ClusterVersion' as the API resource used to verify OpenShift status. It should explain that dev-k8s was classified as non-OpenShift because no ClusterVersion resource was found. Simply saying 'vanilla Kubernetes' or 'not OpenShift' without mentioning the ClusterVersion verification mechanism is insufficient." + }, + { + "id": "exclusion_methodology", + "file": "/root/report.md", + "question": "Does the report treat non-OpenShift clusters (like dev-k8s) as EXCLUDED from the detailed report — listing them briefly in an exclusion section — rather than including them as full sections with node/pod details?", + "reference": "A skilled report should have a separate 'Excluded Clusters' or 'Non-OpenShift' section where dev-k8s is listed briefly with the reason for exclusion. A report that includes dev-k8s as a full section with node details, namespaces, and pod data is NOT demonstrating the skill's exclusion methodology." + }, + { + "id": "aggregated_totals", + "file": "/root/report.md", + "question": "Does the report include aggregated totals across all OpenShift clusters — total nodes, total CPU, total memory, total GPUs — in a comparison or summary table?", + "reference": "A skilled report should have a comparison table with a 'Total' row showing aggregate counts (e.g., 13 nodes total, 148 CPU cores, 592 GiB memory, 8 GPUs). Reports that list each cluster's data without cross-cluster aggregation are insufficient." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/ocp-admin__cluster-report/tests/test.sh b/evaluation/with_skills/ocp-admin__cluster-report/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/ocp-admin__cluster-report/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/ocp-admin__cluster-report/tests/test_outputs.py b/evaluation/with_skills/ocp-admin__cluster-report/tests/test_outputs.py new file mode 100644 index 00000000..5c65747c --- /dev/null +++ b/evaluation/with_skills/ocp-admin__cluster-report/tests/test_outputs.py @@ -0,0 +1,105 @@ +""" +Tests for ocp-admin__cluster-report per-skill evaluation. +Baseline tests: any competent agent should pass. +Skill-dependent tests: based on empirical gaps between skilled and unskilled agent outputs. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_cluster(self): + content = read_report().lower() + assert any(t in content for t in ["cluster", "openshift", "node"]), ( + "report should mention cluster" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 200, "report should have substantial content" + + +class TestSkillDependent: + def test_clusterversion_resource(self): + """Skill teaches to probe the ClusterVersion resource to verify OpenShift. + Without skill, agents say 'vanilla Kubernetes' without mentioning the mechanism.""" + c = read_report().lower() + assert "clusterversion" in c or "cluster version resource" in c, ( + "should mention ClusterVersion resource as the OpenShift verification method" + ) + + def test_aggregated_cross_cluster_totals(self): + """Skill teaches a comparison table with aggregated totals across clusters. + Without skill, agents report each cluster separately without totals.""" + c = read_report().lower() + has_total_label = "total" in c or "aggregate" in c or "combined" in c + has_aggregate_context = any(t in c for t in [ + "total node", "total cpu", "total memory", "total gpu", + "across cluster", "combined resource", "aggregate", + ]) or (has_total_label and any(t in c for t in ["node", "cpu", "core", "memory", "gi"])) + assert has_total_label and has_aggregate_context, ( + "should include aggregated cross-cluster totals (total nodes, CPU, memory)" + ) + + def test_non_openshift_exclusion(self): + """Skill teaches to EXCLUDE non-OpenShift clusters from detailed reporting. + Without skill, agents include dev-k8s as a full section with nodes/pods/namespaces.""" + c = read_report().lower() + has_exclusion = any(t in c for t in [ + "excluded", "exclude", "excluded by default", "not included", + "omitted", "non-openshift", + ]) + assert has_exclusion and "dev-k8s" in c, ( + "should explicitly exclude non-OpenShift clusters from detailed data" + ) + + def test_unreachable_reporting(self): + """Both agents should mention unreachable clusters, but skill teaches categorization.""" + c = read_report().lower() + assert "legacy-dc" in c and any(t in c for t in [ + "unreachable", "connection refused", "offline", + ]), "should report legacy-dc as unreachable" + + def test_gpu_inventory(self): + """Skill template includes GPU column — moderate discriminator.""" + c = read_report().lower() + assert "gpu" in c, "should include GPU information" + + def test_version_numbers(self): + """Both agents get versions from MCP, but skill ensures all clusters are covered.""" + c = read_report() + versions = sum(1 for v in ["4.16.3", "4.15.12", "4.16.1"] if v in c) + assert versions >= 2, "should report exact version numbers for multiple clusters" + + def test_multi_cluster_tooling(self): + """Docs teach multi-cluster tooling/automation for consistent reporting. + Without docs, agents rely on manual kubectl context switching.""" + c = read_report().lower() + assert any(t in c for t in [ + "build-kubeconfig", "kubeconfig.py", "cluster-reporter", + "multi-cluster", "multiple context", "all contexts", + "setup script", "automation", + ]), "should reference multi-cluster tooling or automation approach" + + def test_rbac_for_reporting(self): + """Docs teach read-only RBAC (ClusterRole/ServiceAccount) for cluster reporting + instead of admin credentials.""" + c = read_report().lower() + assert any(t in c for t in [ + "cluster-reporter-readonly", "cluster-reporter-system", + "readonly", "read-only", "clusterrole", + "service account", "serviceaccount", "rbac", + "least privilege", "non-admin", + ]), "should reference read-only RBAC for cluster reporting" diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/Dockerfile b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/Dockerfile new file mode 100644 index 00000000..11301417 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/Dockerfile @@ -0,0 +1,78 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "rhoai": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-rhoai-mcp.py"] \ + }, \ + "observability": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-observability-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/common-issues.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/examples/model-deploy.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/examples/model-deploy.md new file mode 100644 index 00000000..dcd79e44 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/examples/model-deploy.md @@ -0,0 +1,160 @@ +# /model-deploy Examples + +Example interactions demonstrating the model-deploy skill workflow. + +## Example 1: Deploy Llama 3.1 8B with vLLM + +``` +User: "Deploy Llama 3.1 8B on my cluster" + +Agent: [Invokes /model-deploy skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand runtime capabilities. + +Step 1 of 10: Gathering deployment information. + +| Setting | Value | Source | +|---------|-------|--------| +| Model | Llama 3.1 8B Instruct | user input | +| Runtime | TBD | auto-detect | +| Namespace | ? | needs input | +| Model Source | ? | needs input | +| Deployment Mode | Serverless | default | + +What namespace should I deploy to, and where are the model weights stored? +(e.g., S3 bucket, OCI registry, PVC) + +User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" + +Agent: Step 2 of 10: Determining runtime. + +I consulted supported-runtimes.md to understand runtime capabilities. + +Based on Llama 3.1 8B (open-source HuggingFace model): + +Recommended runtime: vLLM +Reason: Default runtime for open-source LLMs. Broad compatibility, + OpenAI-compatible API, no additional setup required. + +Proceed with vLLM? (yes / no / suggest alternative) + +User: "yes" + +Agent: Step 3 of 10: Looking up hardware profile. + +I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. + +| Requirement | Value | +|-------------|-------| +| GPUs | 1x A10G/L4/A100/H100 | +| VRAM | 16GB | +| Key Args | --max-model-len=4096 --tool-call-parser hermes | + +Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). +Ensure your cluster has at least 1 GPU with 16GB VRAM. + +Step 5 of 10: Skipped (NIM platform check not needed for vLLM). + +Step 6-7 of 10: Generating InferenceService YAML. + +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: llama-3-1-8b + namespace: ai-models + annotations: + serving.kserve.io/deploymentMode: Serverless +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: vllm-runtime + storageUri: s3://my-bucket/llama-3.1-8b + resources: + limits: + nvidia.com/gpu: "1" + requests: + cpu: "4" + memory: "16Gi" + args: + - --max-model-len=4096 + - --tool-call-parser=hermes + +| Setting | Value | +|---------|-------| +| Model | llama-3-1-8b | +| Runtime | vllm-runtime | +| GPUs | 1 | +| Namespace | ai-models | +| Mode | Serverless | + +Proceed with creating this InferenceService? (yes/no/modify) + +User: "yes" + +Agent: Step 8 of 10: Creating InferenceService... + [Creates via resources_create_or_update] + + InferenceService created. Monitoring rollout... + +Step 9 of 10: Monitoring rollout. + +| Stage | Status | +|-------|--------| +| Pod Scheduled | Complete | +| Image Pulled | Complete | +| Container Started | Complete | +| Model Loaded | Complete | +| Ready | Complete | + +Step 10 of 10: Deployment complete! + +Model Deployment Successful + +Model: llama-3-1-8b +Runtime: vLLM +Namespace: ai-models +GPUs: 1 + +Inference Endpoint: + URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com + API: OpenAI-compatible REST + +Quick Test: + curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' + +Next Steps: + Monitor performance: "/ai-observability llama-3-1-8b" +``` + +## Example 2: Deploy a Model with NIM (NIM Not Set Up) + +``` +User: "Deploy Llama 3.1 70B with NIM" + +Agent: [Invokes /model-deploy skill] + ... + Step 2: User requested NIM runtime. + + Step 5: Checking NIM platform... + +NIM platform is not set up in namespace ai-models. + +The NIM runtime requires a one-time platform setup. Would you like to: +1. Run /nim-setup now to configure the NIM platform +2. Switch to vLLM runtime instead (no setup required) +3. Abort deployment + +User: "1" + +Agent: [Invokes /nim-setup skill — walks through NGC credential setup] + ... + NIM Platform Setup Complete. + + Resuming model deployment... + [Continues from Step 6 with NIM runtime] +``` diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/examples/nim-setup.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/examples/nim-setup.md new file mode 100644 index 00000000..ccfc1a5d --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/examples/nim-setup.md @@ -0,0 +1,115 @@ +# /nim-setup Examples + +Example interactions demonstrating the nim-setup skill workflow. + +## Example 1: First-Time NIM Setup + +``` +User: "Set up NIM on my cluster" + +Agent: [Invokes /nim-setup skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand NIM platform requirements. + +Let me verify the prerequisites for NIM setup. + +Step 1 of 7: Checking GPU Operator and NFD Operator... + +GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) +NFD Operator: nfd.v4.19.0 (Succeeded) + +Both operators are installed and healthy. + +Step 2 of 7: I need your NGC credentials. + +To set up NIM, I need your NVIDIA NGC API key. +You can generate one at: https://ngc.nvidia.com/setup/api-key + +Please provide: +1. Your NGC API key +2. The target namespace for NIM resources + +User: "Here's my key: [user provides NGC API key], namespace: ai-models" + +Agent: Step 3 of 7: Creating NGC image pull secret... + +I'll create a docker-registry secret for pulling NIM images from nvcr.io: + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-image-pull-secret + namespace: ai-models + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] + +Should I create this image pull secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + Image pull secret created successfully. + +Step 4 of 7: Creating NGC API key secret... + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-api-key + namespace: ai-models + type: Opaque + stringData: + NGC_API_KEY: "[REDACTED]" + +Should I create this API key secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + API key secret created successfully. + +Step 5 of 7: Creating NIM Account CR... + + apiVersion: nim.opendatahub.io/v1 + kind: Account + metadata: + name: nim-account + namespace: ai-models + spec: + apiKeySecret: + name: ngc-api-key + imagePullSecret: + name: ngc-image-pull-secret + +Should I create this NIM Account CR in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates Account CR via resources_create_or_update] + Account CR created successfully. + +Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) + +User: "no" + +Agent: Step 7 of 7: Validating NIM platform readiness... + +NIM Platform Setup Complete + +Account CR: `nim-account` (namespace: ai-models) + Status: Ready + +NGC Credentials: + Image Pull Secret: ngc-image-pull-secret [created] + API Key Secret: ngc-api-key [created] + +NIM ServingRuntimes Available: + - nim-llama-3-1-8b-instruct + - nim-llama-3-1-70b-instruct + - nim-mistral-7b-instruct + +Next Steps: + To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" +``` diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/references/known-model-profiles.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/references/known-model-profiles.md new file mode 100644 index 00000000..40148981 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/references/known-model-profiles.md @@ -0,0 +1,83 @@ +--- +title: Known Model Hardware Profiles +category: references +tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] +semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] +use_cases: [model-deploy, debug-inference, ai-observability] +last_updated: 2026-03-10 +--- + +# Known Model Hardware Profiles + +Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). + +**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. + +## Model Source Conventions + +Each model lists a recommended `storageUri` with its authentication requirements: +- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** +- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). +- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. + +When the user does not specify a model source, use the `hf://` URI listed in the profile below. + +## Llama 3.x (Meta) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | +| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | +| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | +| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | + +- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) +- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly + +## Granite 3.x (IBM/Red Hat) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | +| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | + +- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) +- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser granite --chat-template` +- Red Hat-supported model family on RHOAI + +## Mixtral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | +| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) +- Mixture-of-Experts architecture: only ~13B/45B parameters active per token + +## Mistral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | +| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) + +## When a Model Is Not Listed + +If the requested model is not in this file, the agent MUST use the live doc lookup protocol: + +1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure +2. Fetch hardware specs from the appropriate source +3. Report findings to the user before proceeding with deployment + +Common cases requiring live lookup: +- Newly released models (after this file's last update) +- Domain-specific fine-tuned models +- Models with custom quantization +- NIM-specific optimized profiles diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/references/supported-runtimes.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/references/supported-runtimes.md new file mode 100644 index 00000000..9d018651 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/references/supported-runtimes.md @@ -0,0 +1,104 @@ +--- +title: Supported Serving Runtimes +category: references +tags: [runtimes, vllm, nim, caikit, tgis, serving] +semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] +use_cases: [model-deploy, serving-runtime-config, nim-setup] +last_updated: 2026-02-26 +--- + +# Supported Serving Runtimes + +This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. + +## Runtime Comparison + +| Runtime | API Type | Model Format | GPU Support | License | Setup Required | +|---------|----------|-------------|-------------|---------|----------------| +| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | +| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | +| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | +| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | + +## vLLM + +**Default runtime for most open-source models.** + +- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) +- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized +- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) +- **Model source**: S3-compatible storage, OCI registry, PVC, URI +- **Key features**: + - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) + - PagedAttention for efficient memory management + - Tensor parallelism for multi-GPU inference + - Tool/function calling support (`--tool-call-parser`) + - Continuous batching for high throughput +- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name + +## NVIDIA NIM + +**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** + +- **API**: OpenAI-compatible REST +- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog +- **GPU support**: NVIDIA only (requires specific GPU models per model profile) +- **Model source**: NGC model catalog (pulled automatically via NGC credentials) +- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) +- **Key features**: + - TensorRT-LLM optimization for lower latency + - Pre-compiled model engines (no compilation on first load) + - Optimized and generic profiles per GPU type + - Automatic model download from NGC +- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. +- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM +- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. +- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state + +## Caikit+TGIS + +**Red Hat's Caikit format with Text Generation Inference Server.** + +- **API**: gRPC (not REST) +- **Model formats**: Caikit format (requires conversion from HuggingFace) +- **GPU support**: NVIDIA +- **Model source**: S3-compatible storage +- **Key features**: + - Red Hat-supported runtime + - gRPC API for streaming inference + - Integrated with RHOAI model serving platform +- **When to choose**: When the model is already in Caikit format or when gRPC API is required. +- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name + +## Custom Runtimes + +**User-provided ServingRuntime CRs for unsupported frameworks.** + +- **API**: Defined by the custom runtime +- **Model formats**: Defined by the custom runtime +- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. +- **How to create**: Use `/serving-runtime-config` skill +- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility + +## Runtime Selection Decision Tree + +``` +Is the user's preferred runtime explicitly stated? +├── Yes → Use that runtime +└── No → Continue + +Is the model available in the NGC NIM catalog? +├── Yes → Suggest NIM (with vLLM as fallback) +│ Note: Requires /nim-setup and NVIDIA GPUs +└── No → Continue + +Is the model in Caikit format? +├── Yes → Caikit+TGIS +└── No → Continue + +Is the model a standard open-source LLM (HuggingFace-compatible)? +├── Yes → vLLM (default) +└── No → Custom runtime via /serving-runtime-config +``` diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-observability-mcp.py b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-observability-mcp.py new file mode 100644 index 00000000..f150dcff --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-observability-mcp.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +"""Mock Observability MCP server for SkillsBench rh-ai-engineer__ai-observability task. + +Simulates Prometheus/Grafana-style metrics for inference services: latency, +throughput, error rates, GPU utilization, resource usage, and alerts. + +Scenario (aligned with rhoai/openshift mocks): +- ml-production namespace: + - text-gen-legacy (Mistral 7B on vLLM): OOMKilled; before crash: 22GB/24GB GPU, + p99=2800ms, throughput=3 req/s, error rate=15% + - nim-llama-prod (Llama 3.1 8B on NIM): not running, no metrics (empty/error) + - sentiment-classifier: running well, 4GB/24GB GPU, p99=45ms, throughput=150 req/s, + error rate=0.1% +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("observability") + +# ── Mock metrics data ────────────────────────────────────────────────────── + +# text-gen-legacy: OOMKilled, metrics from before crash +MODEL_METRICS = { + "ml-production": { + "text-gen-legacy": { + "status": "OOMKilled", + "latency_ms": {"p50": 1200, "p95": 2100, "p99": 2800}, + "throughput_req_per_sec": 3.0, + "error_rate_percent": 15.0, + "input_tokens_per_sec": 45, + "output_tokens_per_sec": 12, + "total_requests_24h": 259200, # 3 * 86400 + }, + "nim-llama-prod": None, # not running, no metrics + "sentiment-classifier": { + "status": "Running", + "latency_ms": {"p50": 18, "p95": 38, "p99": 45}, + "throughput_req_per_sec": 150.0, + "error_rate_percent": 0.1, + "input_tokens_per_sec": 1200, + "output_tokens_per_sec": 50, + "total_requests_24h": 12960000, + }, + }, +} + +GPU_UTILIZATION = { + "ml-production": [ + { + "pod": "text-gen-legacy-predictor-00001-abc12", + "model": "text-gen-legacy", + "gpu_memory_used_gb": 22.0, + "gpu_memory_total_gb": 24.0, + "gpu_memory_utilization_percent": 91.7, + "gpu_compute_utilization_percent": 35.0, + "status": "OOMKilled", + }, + { + "pod": "sentiment-classifier-predictor-00001-xyz99", + "model": "sentiment-classifier", + "gpu_memory_used_gb": 4.0, + "gpu_memory_total_gb": 24.0, + "gpu_memory_utilization_percent": 16.7, + "gpu_compute_utilization_percent": 42.0, + "status": "Running", + }, + # nim-llama-prod: no pod + ], +} + +RESOURCE_USAGE = { + "ml-production": [ + { + "pod": "text-gen-legacy-predictor-00001-abc12", + "model": "text-gen-legacy", + "cpu_request": "4", + "cpu_limit": "4", + "memory_request": "16Gi", + "memory_limit": "16Gi", + "cpu_actual_usage": "3.2", + "memory_actual_usage_mib": 16384, + "status": "CrashLoopBackOff", + }, + { + "pod": "sentiment-classifier-predictor-00001-xyz99", + "model": "sentiment-classifier", + "cpu_request": "2", + "cpu_limit": "4", + "memory_request": "8Gi", + "memory_limit": "16Gi", + "cpu_actual_usage": "1.1", + "memory_actual_usage_mib": 4096, + "status": "Running", + }, + ], +} + +PROMETHEUS_ALERTS = { + "ml-production": [ + { + "name": "InferenceServiceOOMKilled", + "severity": "critical", + "state": "firing", + "summary": "text-gen-legacy predictor pod OOMKilled", + "description": "Container kserve-container was OOMKilled (exit code 137). " + "GPU memory exhausted during KV cache allocation.", + "labels": { + "inference_service": "text-gen-legacy", + "namespace": "ml-production", + }, + }, + { + "name": "HighInferenceLatency", + "severity": "warning", + "state": "firing", + "summary": "text-gen-legacy p99 latency > 2000ms", + "description": "Inference latency p99 is 2800ms, exceeding threshold of 2000ms.", + "labels": { + "inference_service": "text-gen-legacy", + "namespace": "ml-production", + }, + }, + { + "name": "HighErrorRate", + "severity": "warning", + "state": "firing", + "summary": "text-gen-legacy error rate 15%", + "description": "Inference error rate is 15%, exceeding threshold of 5%.", + "labels": { + "inference_service": "text-gen-legacy", + "namespace": "ml-production", + }, + }, + ], +} + + +# ── Tools ────────────────────────────────────────────────────────────────── + + +@mcp.tool() +def query_model_metrics( + model_name: str, + namespace: str, + metric_type: str = "all", +) -> str: + """Query inference metrics for a model. Returns latency (p50/p95/p99), throughput + (requests/sec), error rates, and token counts. + + metric_type: 'all', 'latency', 'throughput', 'errors', or 'tokens' + """ + ns_data = MODEL_METRICS.get(namespace) + if not ns_data: + return json.dumps({"error": f"Namespace '{namespace}' not found"}, indent=2) + + metrics = ns_data.get(model_name) + if metrics is None: + return json.dumps({ + "error": f"No metrics for model '{model_name}' in namespace '{namespace}'. " + "Model may not be running (e.g., nim-llama-prod has no pods).", + "model_name": model_name, + "namespace": namespace, + }, indent=2) + + result = { + "model_name": model_name, + "namespace": namespace, + "status": metrics["status"], + } + + if metric_type in ("all", "latency"): + result["latency_ms"] = metrics["latency_ms"] + if metric_type in ("all", "throughput"): + result["throughput_req_per_sec"] = metrics["throughput_req_per_sec"] + result["total_requests_24h"] = metrics.get("total_requests_24h") + if metric_type in ("all", "errors"): + result["error_rate_percent"] = metrics["error_rate_percent"] + if metric_type in ("all", "tokens"): + result["input_tokens_per_sec"] = metrics["input_tokens_per_sec"] + result["output_tokens_per_sec"] = metrics["output_tokens_per_sec"] + + return json.dumps(result, indent=2) + + +@mcp.tool() +def query_gpu_utilization(namespace: str) -> str: + """Query GPU memory used/total and compute utilization per inference pod.""" + pods = GPU_UTILIZATION.get(namespace, []) + if not pods: + return json.dumps({ + "namespace": namespace, + "pods": [], + "message": "No GPU-backed inference pods found in namespace.", + }, indent=2) + return json.dumps({ + "namespace": namespace, + "pods": pods, + }, indent=2) + + +@mcp.tool() +def query_resource_usage(namespace: str) -> str: + """Query actual CPU/memory usage vs requests/limits for inference pods.""" + pods = RESOURCE_USAGE.get(namespace, []) + if not pods: + return json.dumps({ + "namespace": namespace, + "pods": [], + "message": "No inference pods found in namespace.", + }, indent=2) + return json.dumps({ + "namespace": namespace, + "pods": pods, + }, indent=2) + + +@mcp.tool() +def list_prometheus_alerts(namespace: str) -> str: + """List firing Prometheus alerts related to inference services in the namespace.""" + alerts = PROMETHEUS_ALERTS.get(namespace, []) + return json.dumps({ + "namespace": namespace, + "alerts": alerts, + "firing_count": len(alerts), + }, indent=2) + + +@mcp.tool() +def get_model_performance_summary(namespace: str) -> str: + """Get aggregated performance data across all models in the namespace.""" + ns_data = MODEL_METRICS.get(namespace) + if not ns_data: + return json.dumps({"error": f"Namespace '{namespace}' not found"}, indent=2) + + models = [] + for name, metrics in ns_data.items(): + if metrics is None: + models.append({ + "model_name": name, + "status": "NotRunning", + "error": "No metrics available (pod not created or not running)", + }) + else: + models.append({ + "model_name": name, + "status": metrics["status"], + "latency_p99_ms": metrics["latency_ms"]["p99"], + "throughput_req_per_sec": metrics["throughput_req_per_sec"], + "error_rate_percent": metrics["error_rate_percent"], + }) + + return json.dumps({ + "namespace": namespace, + "models": models, + }, indent=2) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..e7a4d11c --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +"""Mock OpenShift MCP server for SkillsBench rh-ai-engineer task. + +Simulates Kubernetes resource CRUD, pod management, logs, and events. + +Key scenario elements: +- LimitRange in namespaces: min CPU=100m, min memory=128Mi + (conflicts with KServe sidecar containers hardcoded at 10m CPU/15Mi memory) +- GPU node with custom taint ai-workload=true:NoSchedule +- NIM Account CR in ml-production: not ready (NGC credentials invalid) +- text-gen-legacy pods: OOMKilled (max-model-len=32768 on A10G) +- nim-llama-prod: no pods created (Account CR not ready) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + +# ── Cluster state ──────────────────────────────────────────────────────── + +GPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "gpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + "nvidia.com/gpu.present": "true", + "nvidia.com/gpu.product": "NVIDIA-A10G", + }, + }, + "spec": { + "taints": [ + { + "key": "ai-workload", + "value": "true", + "effect": "NoSchedule", + }, + ], + }, + "status": { + "allocatable": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "capacity": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "conditions": [ + {"type": "Ready", "status": "True"}, + ], + }, +} + +CPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "cpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + }, + }, + "spec": {"taints": []}, + "status": { + "allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "capacity": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +MASTER_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "master-1", + "labels": { + "node-role.kubernetes.io/master": "", + "node-role.kubernetes.io/control-plane": "", + }, + }, + "spec": { + "taints": [ + {"key": "node-role.kubernetes.io/master", "effect": "NoSchedule"}, + ], + }, + "status": { + "allocatable": {"cpu": "8", "memory": "32Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +ALL_NODES = [GPU_NODE, CPU_NODE, MASTER_NODE] + +# LimitRange applied by cluster policy to all DS project namespaces +NAMESPACE_LIMITRANGE = { + "apiVersion": "v1", + "kind": "LimitRange", + "metadata": { + "name": "default-limits", + }, + "spec": { + "limits": [ + { + "type": "Container", + "default": { + "cpu": "2", + "memory": "4Gi", + }, + "defaultRequest": { + "cpu": "500m", + "memory": "256Mi", + }, + "min": { + "cpu": "100m", + "memory": "128Mi", + }, + "max": { + "cpu": "32", + "memory": "128Gi", + }, + }, + ], + }, +} + +NIM_ACCOUNT_CR = { + "apiVersion": "nim.opendatahub.io/v1", + "kind": "Account", + "metadata": { + "name": "nim-account", + "namespace": "ml-production", + }, + "spec": { + "apiKeySecret": { + "name": "ngc-api-key", + }, + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "NGCCredentialsInvalid", + "message": "NGC API key validation failed: 401 Unauthorized. " + "The API key in secret 'ngc-api-key' is expired or invalid. " + "Re-create the secret with a valid NGC API key from " + "https://ngc.nvidia.com/setup/api-key and restart the " + "Account reconciliation.", + "lastTransitionTime": "2026-03-14T12:00:00Z", + }, + ], + "nimPullSecretStatus": "Failed", + "nimConfigStatus": "Pending", + }, +} + +SERVING_RUNTIME_VLLM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "vllm-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "vLLM", "version": "1", "autoSelect": True}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "quay.io/modh/vllm:rhoai-2.16", + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + }, + ], + }, +} + +SERVING_RUNTIME_NIM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "nim-serving-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "NIM", "version": "1"}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "nvcr.io/nim/meta/llama-3.1-8b-instruct:latest", + "ports": [{"containerPort": 8000, "protocol": "TCP"}], + "env": [ + {"name": "NGC_API_KEY", "valueFrom": { + "secretKeyRef": {"name": "ngc-api-key", "key": "api_key"}, + }}, + ], + }, + ], + }, +} + +PODS_BY_NAMESPACE = { + "ml-production": [ + { + "name": "text-gen-legacy-predictor-00001-abc12", + "namespace": "ml-production", + "status": "CrashLoopBackOff", + "restarts": 5, + "node": "gpu-worker-1", + "containers": [ + { + "name": "kserve-container", + "state": "waiting", + "reason": "CrashLoopBackOff", + "last_termination_reason": "OOMKilled", + "last_termination_exit_code": 137, + }, + ], + "labels": { + "serving.kserve.io/inferenceservice": "text-gen-legacy", + }, + "gpu": "1", + }, + # nim-llama-prod: NO pods created (Account CR not ready) + ], +} + +POD_LOGS = { + "text-gen-legacy-predictor-00001-abc12": ( + "INFO 2026-03-01 10:00:00 vllm_engine.py:125] vLLM engine starting...\n" + "INFO 2026-03-01 10:00:01 config.py:89] Model: mistralai/Mistral-7B-Instruct-v0.3\n" + "INFO 2026-03-01 10:00:01 config.py:92] max_model_len = 32768\n" + "INFO 2026-03-01 10:00:02 gpu_executor.py:45] GPU 0: NVIDIA A10G (24576 MiB)\n" + "INFO 2026-03-01 10:00:03 model_runner.py:88] Loading model weights...\n" + "INFO 2026-03-01 10:00:15 model_runner.py:112] Model weights loaded: 13.5 GiB\n" + "INFO 2026-03-01 10:00:15 worker.py:201] Allocating KV cache...\n" + "ERROR 2026-03-01 10:00:16 worker.py:215] torch.cuda.OutOfMemoryError: " + "CUDA out of memory. Tried to allocate 28.5 GiB for KV cache but only " + "10.1 GiB available after loading model weights (13.5 GiB).\n" + "ERROR 2026-03-01 10:00:16 vllm_engine.py:178] Engine failed to start\n" + "Traceback (most recent call last):\n" + " File \"/opt/vllm/vllm/engine/engine.py\", line 175, in start\n" + " self._init_kv_cache()\n" + " File \"/opt/vllm/vllm/worker/worker.py\", line 215, in _init_kv_cache\n" + " raise torch.cuda.OutOfMemoryError(msg)\n" + "torch.cuda.OutOfMemoryError: CUDA out of memory\n" + ), +} + +EVENTS_BY_NAMESPACE = { + "ml-production": [ + { + "type": "Warning", + "reason": "BackOff", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Back-off restarting failed container kserve-container in pod " + "text-gen-legacy-predictor-00001-abc12", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Warning", + "reason": "OOMKilled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Container kserve-container was OOMKilled (exit code 137). " + "GPU memory exhausted during KV cache allocation.", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Normal", + "reason": "Scheduled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Successfully assigned ml-production/" + "text-gen-legacy-predictor-00001-abc12 to gpu-worker-1", + "count": 1, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-02-28T08:00:00Z", + }, + { + "type": "Warning", + "reason": "NIMAccountNotReady", + "object": "InferenceService/nim-llama-prod", + "message": "NIM Account 'nim-account' in namespace 'ml-production' " + "is not ready", + "count": 12, + "first_timestamp": "2026-03-14T12:00:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + { + "type": "Warning", + "reason": "ImagePullBackOff", + "object": "InferenceService/nim-llama-prod", + "message": "Failed to pull image 'nvcr.io/nim/meta/llama-3.1-8b-instruct:" + "latest': unauthorized: authentication required", + "count": 8, + "first_timestamp": "2026-03-14T12:05:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + ], +} + + +# ── Resource tools ─────────────────────────────────────────────────────── + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: str = "", +) -> str: + """Get a single Kubernetes resource by apiVersion, kind, and name.""" + if kind == "Node": + for node in ALL_NODES: + if node["metadata"]["name"] == name: + return json.dumps(node, indent=2) + raise ValueError(f"Node '{name}' not found") + + if kind == "ServingRuntime": + if name == "vllm-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_VLLM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + if name == "nim-serving-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_NIM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + raise ValueError(f"ServingRuntime '{name}' not found in namespace '{namespace}'") + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps(lr, indent=2) + + if kind == "Account" and "nim" in apiVersion.lower(): + if namespace == "ml-production" and name == "nim-account": + return json.dumps(NIM_ACCOUNT_CR, indent=2) + raise ValueError( + f"Account '{name}' not found in namespace '{namespace}'" + ) + + if kind == "ClusterVersion" and apiVersion == "config.openshift.io/v1": + return json.dumps({ + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterVersion", + "metadata": {"name": "version"}, + "status": {"desired": {"version": "4.16.3"}}, + }) + + raise ValueError(f"Resource {apiVersion}/{kind}/{name} not found") + + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: str = "", + labelSelector: str = "", +) -> str: + """List Kubernetes resources by apiVersion and kind.""" + if kind == "Node": + nodes = ALL_NODES + if labelSelector: + parts = labelSelector.split("=", 1) + key = parts[0] + value = parts[1] if len(parts) > 1 else "" + nodes = [ + n for n in nodes + if n["metadata"]["labels"].get(key) == value + ] + return json.dumps(nodes, indent=2) + + if kind == "Service" and apiVersion == "serving.knative.dev/v1": + return json.dumps({ + "kind": "ServiceList", + "apiVersion": "serving.knative.dev/v1", + "items": [], + "metadata": {}, + }) + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps({ + "kind": "LimitRangeList", + "items": [lr], + }) + + if kind == "InferenceService": + return json.dumps({ + "kind": "InferenceServiceList", + "items": [], + }) + + raise ValueError(f"Unsupported list: {apiVersion}/{kind}") + + +@mcp.tool() +def pods_list( + namespace: str, + labelSelector: str = "", +) -> str: + """List pods in a namespace with optional label selector.""" + pods = PODS_BY_NAMESPACE.get(namespace, []) + + if labelSelector: + key, _, value = labelSelector.partition("=") + pods = [p for p in pods if p.get("labels", {}).get(key) == value] + + results = [] + for pod in pods: + results.append({ + "name": pod["name"], + "namespace": pod["namespace"], + "status": pod["status"], + "restarts": pod.get("restarts", 0), + "node": pod.get("node", ""), + "containers": pod.get("containers", []), + "gpu": pod.get("gpu", "0"), + }) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def pods_log( + namespace: str, + name: str, + container: str = "", +) -> str: + """Get logs from a pod container.""" + logs = POD_LOGS.get(name) + if logs is None: + raise ValueError(f"Pod '{name}' not found in namespace '{namespace}'") + return logs + + +@mcp.tool() +def events_list(namespace: str) -> str: + """List events in a namespace.""" + events = EVENTS_BY_NAMESPACE.get(namespace, []) + return json.dumps(events, indent=2) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-rhoai-mcp.py b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-rhoai-mcp.py new file mode 100644 index 00000000..0ae9e4cb --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-rhoai-mcp.py @@ -0,0 +1,780 @@ +#!/usr/bin/env python3 +"""Mock RHOAI MCP server for SkillsBench rh-ai-engineer task. + +Simulates Red Hat OpenShift AI operations: Data Science Projects, +model serving, data connections, serving runtimes, inference services. + +Scenario: +- ml-production: existing project with two broken deployments + - text-gen-legacy: vLLM OOMKilled (max-model-len=32768 on A10G) + - nim-llama-prod: NIM failing (Account CR not ready, NGC creds invalid) +- fraud-detection: does not exist yet (agent creates it) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("rhoai") + +# ── In-memory state ────────────────────────────────────────────────────── + +PROJECTS = { + "ml-production": { + "name": "ml-production", + "display_name": "ML Production", + "description": "Production ML workloads", + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": "single", + "pipeline_server": True, + }, +} + +DATA_CONNECTIONS = { + "ml-production": [ + { + "name": "prod-model-store", + "type": "S3", + "bucket": "ml-models-prod", + "endpoint": "https://s3.us-east-1.amazonaws.com", + "region": "us-east-1", + }, + ], +} + +SERVING_RUNTIMES = { + "__platform_templates__": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "REST", + "supported_model_formats": [ + {"name": "vLLM", "version": "1", "autoSelect": True} + ], + }, + { + "name": "caikit-tgis-runtime", + "display_name": "Caikit+TGIS ServingRuntime", + "model_formats": ["caikit"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "gRPC", + }, + ], + "ml-production": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + { + "name": "nim-serving-runtime", + "display_name": "NVIDIA NIM ServingRuntime", + "model_formats": ["NIM"], + "requires_instantiation": False, + "source": "nim-account", + "api_protocol": "REST", + }, + { + "name": "ovms-1", + "display_name": "OpenVINO Model Server", + "model_formats": ["openvino_ir", "onnx"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + ], +} + +INFERENCE_SERVICES = { + "ml-production": { + "text-gen-legacy": { + "name": "text-gen-legacy", + "namespace": "ml-production", + "runtime": "vllm-runtime", + "model_format": "vLLM", + "storage_uri": "hf://mistralai/Mistral-7B-Instruct-v0.3", + "display_name": "Mistral 7B Legacy", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "16Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "PredictorFailed", + "message": "Predictor pod is not ready", + }, + { + "type": "PredictorReady", + "status": "False", + "reason": "ContainerCrashLoop", + "message": "Container kserve-container terminated: " + "OOMKilled (exit code 137). 5 restarts.", + }, + { + "type": "IngressReady", + "status": "True", + "reason": "IngressReady", + "message": "Ingress is ready", + }, + ], + "age": "3d", + }, + "nim-llama-prod": { + "name": "nim-llama-prod", + "namespace": "ml-production", + "runtime": "nim-serving-runtime", + "model_format": "NIM", + "storage_uri": "nim://meta/llama-3.1-8b-instruct", + "display_name": "Llama 3.1 8B (NIM)", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "32Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "RuntimeNotReady", + "message": "ServingRuntime 'nim-serving-runtime' " + "is not in ready state", + }, + { + "type": "PredictorReady", + "status": "Unknown", + "reason": "PodNotCreated", + "message": "Predictor pod has not been created. " + "Waiting for ServingRuntime to become ready.", + }, + { + "type": "IngressReady", + "status": "Unknown", + "reason": "PredictorNotReady", + "message": "Waiting for predictor to become ready", + }, + ], + "age": "1d", + }, + }, +} + +DEPLOYED_MODELS = {} + +WORKBENCHES = { + "ml-production": [ + { + "name": "data-exploration-nb", + "display_name": "Data Exploration", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Running", + "cpu_request": "1", + "memory_request": "8Gi", + "gpu_count": 0, + "pvc_name": "data-exploration-nb-pvc", + "pvc_size": "20Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-10T09:00:00Z", + }, + { + "name": "model-training-nb", + "display_name": "Model Training", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Stopped", + "cpu_request": "4", + "memory_request": "16Gi", + "gpu_count": 1, + "pvc_name": "model-training-nb-pvc", + "pvc_size": "50Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-15T14:00:00Z", + }, + ], +} + +PIPELINE_SERVERS = { + "ml-production": { + "configured": True, + "data_connection": "prod-model-store", + "status": "Ready", + "database": "MariaDB", + }, +} + +NOTEBOOK_IMAGES = [ + {"name": "jupyter-pytorch-ubi9-python-3.9-2024.1", "display_name": "PyTorch 2024.1", "packages": ["torch", "transformers"]}, + {"name": "jupyter-tensorflow-ubi9-python-3.9-2024.1", "display_name": "TensorFlow 2024.1", "packages": ["tensorflow"]}, + {"name": "jupyter-datascience-ubi9-python-3.9-2024.1", "display_name": "Standard Data Science", "packages": ["pandas", "scikit-learn"]}, + {"name": "jupyter-minimal-ubi9-python-3.9-2024.1", "display_name": "Minimal Python", "packages": []}, +] + + +# ── Project tools ──────────────────────────────────────────────────────── + + +@mcp.tool() +def list_data_science_projects() -> str: + """List all RHOAI Data Science Projects on the cluster.""" + projects = [] + for name, proj in PROJECTS.items(): + projects.append({ + "name": name, + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + }) + return json.dumps(projects, indent=2) + + +@mcp.tool() +def create_data_science_project( + name: str, + display_name: str, + description: str = "", +) -> str: + """Create a new RHOAI Data Science Project (namespace with dashboard labels).""" + if name in PROJECTS: + raise ValueError( + f"Project '{name}' already exists. Choose a different name " + "or configure the existing project." + ) + if not name.replace("-", "").replace("_", "").isalnum() or len(name) > 63: + raise ValueError( + f"Invalid project name '{name}'. Must be DNS-compatible: " + "lowercase alphanumeric and hyphens, max 63 chars." + ) + + PROJECTS[name] = { + "name": name, + "display_name": display_name, + "description": description, + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": None, + "pipeline_server": False, + } + DATA_CONNECTIONS[name] = [] + SERVING_RUNTIMES[name] = [] + INFERENCE_SERVICES[name] = {} + + return json.dumps({ + "status": "created", + "name": name, + "display_name": display_name, + "namespace": name, + "labels": {"opendatahub.io/dashboard": "true"}, + }) + + +@mcp.tool() +def get_project_details(name: str) -> str: + """Get detailed information about an RHOAI Data Science Project.""" + if name not in PROJECTS: + raise ValueError(f"Project '{name}' not found") + proj = PROJECTS[name] + dc_count = len(DATA_CONNECTIONS.get(name, [])) + isvc_count = len(INFERENCE_SERVICES.get(name, {})) + return json.dumps({ + "name": proj["name"], + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "labels": proj["labels"], + "data_connections": dc_count, + "inference_services": isvc_count, + "model_serving_mode": proj.get("model_serving_mode"), + "pipeline_server": proj.get("pipeline_server", False), + }) + + +@mcp.tool() +def get_project_status(namespace: str) -> str: + """Get comprehensive status of an RHOAI Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Project '{namespace}' not found") + proj = PROJECTS[namespace] + dcs = DATA_CONNECTIONS.get(namespace, []) + isvcs = INFERENCE_SERVICES.get(namespace, {}) + return json.dumps({ + "namespace": namespace, + "display_name": proj["display_name"], + "status": "Active", + "components": { + "data_connections": len(dcs), + "inference_services": len(isvcs), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + "pipeline_server": "configured" if proj.get("pipeline_server") else "not configured", + }, + }) + + +# ── Data connection tools ──────────────────────────────────────────────── + + +@mcp.tool() +def create_s3_data_connection( + namespace: str, + name: str, + bucket: str, + endpoint: str, + access_key: str, + secret_key: str, + region: str = "", +) -> str: + """Create an S3-compatible data connection in an RHOAI project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + existing = DATA_CONNECTIONS.get(namespace, []) + if any(dc["name"] == name for dc in existing): + raise ValueError( + f"Data connection '{name}' already exists in namespace '{namespace}'" + ) + + dc = { + "name": name, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + "region": region, + } + DATA_CONNECTIONS.setdefault(namespace, []).append(dc) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + }) + + +@mcp.tool() +def list_data_connections(namespace: str) -> str: + """List data connections in an RHOAI project namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + return json.dumps(dcs, indent=2) + + +# ── Model serving tools ───────────────────────────────────────────────── + + +@mcp.tool() +def set_model_serving_mode(namespace: str, mode: str) -> str: + """Enable model serving on a Data Science Project (single or multi mode).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + if mode not in ("single", "multi"): + raise ValueError(f"Invalid mode '{mode}'. Must be 'single' or 'multi'.") + + PROJECTS[namespace]["model_serving_mode"] = mode + + if not SERVING_RUNTIMES.get(namespace): + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + SERVING_RUNTIMES[namespace] = [ + {**t, "requires_instantiation": False, "source": "existing"} + for t in templates + ] + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "mode": mode, + }) + + +@mcp.tool() +def list_serving_runtimes( + namespace: str, + include_templates: bool = False, +) -> str: + """List available ServingRuntimes in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + runtimes = list(SERVING_RUNTIMES.get(namespace, [])) + if include_templates: + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + existing_names = {r["name"] for r in runtimes} + for t in templates: + if t["name"] not in existing_names: + runtimes.append(t) + + return json.dumps(runtimes, indent=2) + + +# ── Inference service tools ────────────────────────────────────────────── + + +@mcp.tool() +def deploy_model( + name: str, + namespace: str, + runtime: str, + model_format: str, + storage_uri: str, + display_name: str = "", + min_replicas: int = 1, + max_replicas: int = 1, + cpu_request: str = "1", + cpu_limit: str = "2", + memory_request: str = "4Gi", + memory_limit: str = "8Gi", + gpu_count: int = 0, +) -> str: + """Deploy an AI/ML model as a KServe InferenceService.""" + if namespace not in PROJECTS: + raise ValueError( + f"Namespace '{namespace}' is not a Data Science Project. " + "Create one via create_data_science_project first." + ) + + ns_runtimes = SERVING_RUNTIMES.get(namespace, []) + runtime_names = [r["name"] for r in ns_runtimes] + if runtime not in runtime_names: + available = ", ".join(runtime_names) or "none" + raise ValueError( + f"ServingRuntime '{runtime}' not found in namespace '{namespace}'. " + f"Available runtimes: {available}" + ) + + endpoint = f"https://{name}-{namespace}.apps.ocp-cluster.example.com" + isvc = { + "name": name, + "namespace": namespace, + "runtime": runtime, + "model_format": model_format, + "storage_uri": storage_uri, + "display_name": display_name or name, + "gpu_count": gpu_count, + "cpu_request": cpu_request, + "memory_request": memory_request, + "min_replicas": min_replicas, + "max_replicas": max_replicas, + "ready": True, + "url": endpoint, + "conditions": [ + {"type": "Ready", "status": "True", "reason": "Ready", "message": ""}, + {"type": "PredictorReady", "status": "True", "reason": "PodReady", "message": ""}, + {"type": "IngressReady", "status": "True", "reason": "IngressReady", "message": ""}, + ], + "age": "0s", + } + + INFERENCE_SERVICES.setdefault(namespace, {})[name] = isvc + DEPLOYED_MODELS[f"{namespace}/{name}"] = isvc + + return json.dumps({ + "status": "deployed", + "name": name, + "namespace": namespace, + "runtime": runtime, + "endpoint": endpoint, + "ready": True, + }) + + +@mcp.tool() +def list_inference_services( + namespace: str, + verbosity: str = "standard", +) -> str: + """List deployed InferenceServices in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + isvcs = INFERENCE_SERVICES.get(namespace, {}) + results = [] + for isvc_name, isvc in isvcs.items(): + entry = { + "name": isvc["name"], + "runtime": isvc["runtime"], + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "age": isvc.get("age", ""), + } + if verbosity == "full": + entry["conditions"] = isvc.get("conditions", []) + entry["storage_uri"] = isvc.get("storage_uri", "") + entry["gpu_count"] = isvc.get("gpu_count", 0) + results.append(entry) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def get_inference_service( + name: str, + namespace: str, + verbosity: str = "standard", +) -> str: + """Get detailed status of a specific InferenceService.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + + isvc = isvcs[name] + result = { + "name": isvc["name"], + "namespace": isvc["namespace"], + "runtime": isvc["runtime"], + "model_format": isvc.get("model_format", ""), + "storage_uri": isvc.get("storage_uri", ""), + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "conditions": isvc.get("conditions", []), + "gpu_count": isvc.get("gpu_count", 0), + "replicas": {"min": isvc.get("min_replicas", 1), "max": isvc.get("max_replicas", 1)}, + "resources": { + "cpu_request": isvc.get("cpu_request", "1"), + "memory_request": isvc.get("memory_request", "4Gi"), + "memory_limit": isvc.get("memory_limit", "8Gi"), + }, + "age": isvc.get("age", ""), + } + return json.dumps(result, indent=2) + + +@mcp.tool() +def get_model_endpoint(name: str, namespace: str) -> str: + """Get the inference endpoint URL for a deployed model.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + isvc = isvcs[name] + if not isvc["ready"]: + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": "", + "error": "InferenceService is not ready. Check conditions for details.", + }) + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": isvc["url"], + }) + + +# ── Workbench tools ────────────────────────────────────────────────────── + + +@mcp.tool() +def list_workbenches(namespace: str) -> str: + """List workbenches (Jupyter notebooks) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + return json.dumps(wbs, indent=2) + + +@mcp.tool() +def create_workbench( + namespace: str, + name: str, + display_name: str = "", + image: str = "jupyter-datascience-ubi9-python-3.9-2024.1", + cpu_request: str = "1", + memory_request: str = "4Gi", + gpu_count: int = 0, + pvc_size: str = "20Gi", +) -> str: + """Create a new workbench (Jupyter notebook) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + valid_images = [img["name"] for img in NOTEBOOK_IMAGES] + if image not in valid_images: + raise ValueError( + f"Image '{image}' not found. Available: {', '.join(valid_images)}" + ) + + wb = { + "name": name, + "display_name": display_name or name, + "image": image, + "status": "Running", + "cpu_request": cpu_request, + "memory_request": memory_request, + "gpu_count": gpu_count, + "pvc_name": f"{name}-pvc", + "pvc_size": pvc_size, + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-03-02T12:00:00Z", + } + WORKBENCHES.setdefault(namespace, []).append(wb) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "image": image, + "pvc": f"{name}-pvc", + }) + + +@mcp.tool() +def stop_workbench(namespace: str, name: str) -> str: + """Stop a running workbench (preserves data).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Stopped" + return json.dumps({"status": "stopped", "name": name, "namespace": namespace}) + + +@mcp.tool() +def start_workbench(namespace: str, name: str) -> str: + """Start a stopped workbench.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Running" + return json.dumps({"status": "running", "name": name, "namespace": namespace}) + + +@mcp.tool() +def delete_workbench(namespace: str, name: str) -> str: + """Delete a workbench. WARNING: PVC data may be lost if not backed up.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wbs.remove(wb) + return json.dumps({ + "status": "deleted", + "name": name, + "namespace": namespace, + "warning": "Associated PVC data has been deleted", + }) + + +@mcp.tool() +def list_notebook_images() -> str: + """List available notebook images for workbench creation.""" + return json.dumps(NOTEBOOK_IMAGES, indent=2) + + +# ── Pipeline server tools ─────────────────────────────────────────────── + + +@mcp.tool() +def configure_pipeline_server( + namespace: str, + data_connection: str, + database: str = "MariaDB", +) -> str: + """Configure a pipeline server for a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + if not any(dc["name"] == data_connection for dc in dcs): + available = [dc["name"] for dc in dcs] + raise ValueError( + f"Data connection '{data_connection}' not found. Available: {available}" + ) + + PIPELINE_SERVERS[namespace] = { + "configured": True, + "data_connection": data_connection, + "status": "Ready", + "database": database, + } + PROJECTS[namespace]["pipeline_server"] = True + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "data_connection": data_connection, + "database": database, + }) + + +@mcp.tool() +def get_pipeline_server_status(namespace: str) -> str: + """Get the status of the pipeline server in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + ps = PIPELINE_SERVERS.get(namespace) + if not ps: + return json.dumps({"namespace": namespace, "configured": False}) + return json.dumps({ + "namespace": namespace, + "configured": ps["configured"], + "data_connection": ps["data_connection"], + "status": ps["status"], + "database": ps["database"], + }) + + +# ── Serving runtime creation ──────────────────────────────────────────── + + +@mcp.tool() +def create_serving_runtime( + namespace: str, + name: str, + display_name: str = "", + model_formats: list = None, + container_image: str = "", + container_port: int = 8080, + multi_model: bool = False, + api_protocol: str = "REST", +) -> str: + """Create a custom ServingRuntime in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + if not model_formats: + raise ValueError("model_formats must specify at least one model format") + + runtime = { + "name": name, + "display_name": display_name or name, + "model_formats": model_formats, + "requires_instantiation": False, + "source": "custom", + "api_protocol": api_protocol, + "container_image": container_image, + "container_port": container_port, + "multi_model": multi_model, + } + SERVING_RUNTIMES.setdefault(namespace, []).append(runtime) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "model_formats": model_formats, + }) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/ai-observability/SKILL.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/ai-observability/SKILL.md new file mode 100644 index 00000000..d93861ec --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/ai-observability/SKILL.md @@ -0,0 +1,420 @@ +--- +name: ai-observability +description: | + Analyze AI model performance, GPU utilization, and cluster health on OpenShift AI. + + Use when: + - "How is my model performing?" + - "What GPUs are available in the cluster?" + - "Show me inference latency for Llama" + - "Check OpenShift cluster health metrics" + - "Trace a slow inference request" + - "Correlate errors across my inference stack" + + Query-driven, read-only analysis. Routes to the appropriate observability domain based on user intent. + + NOT for deploying models (use /model-deploy). + NOT for debugging failed deployments (use /debug-inference). +model: inherit +color: cyan +--- + +# /ai-observability Skill + +Analyze AI model inference performance, GPU utilization, OpenShift cluster health, and distributed traces on Red Hat OpenShift AI. This is a query-driven, read-only skill: the user asks a question, and the skill routes to the appropriate observability domain (vLLM metrics, OpenShift health, Tempo traces, or cross-domain correlation via Korrel8r). + +## Prerequisites + +**Required MCP Server**: `ai-observability` ([AI Observability MCP](https://github.com/rh-ai-quickstart/ai-observability-summarizer)) + +**Required MCP Tools**: +- `list_models` (from ai-observability) - Discover served models +- `list_vllm_namespaces` (from ai-observability) - List monitored namespaces +- `get_gpu_info` (from ai-observability) - GPU inventory and utilization +- `get_deployment_info` (from ai-observability) - Deployment health status +- `analyze_vllm` (from ai-observability) - Model performance analysis +- `chat_vllm` (from ai-observability) - Conversational follow-up on vLLM metrics +- `analyze_openshift` (from ai-observability) - Cluster/namespace health metrics +- `list_openshift_metric_groups` (from ai-observability) - Available metric categories +- `list_openshift_namespaces` (from ai-observability) - Namespaces in Prometheus +- `query_tempo_tool` (from ai-observability) - Distributed trace queries +- `get_trace_details_tool` (from ai-observability) - Trace span details +- `search_metrics` (from ai-observability) - Metric discovery by pattern +- `execute_promql` (from ai-observability) - Custom PromQL queries +- `korrel8r_get_correlated` (from ai-observability) - Cross-domain signal correlation + +**Optional MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) + +**Optional MCP Tools** (from rhoai): +- `list_data_science_projects` - Discover RHOAI projects for scope selection +- `list_inference_services` - List deployed models with structured status for context +- `get_inference_service` - Get InferenceService status for context + +**Optional MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Optional MCP Tools** (from openshift): +- `resources_get` (from openshift) - Get raw resource details for context +- `pods_list` (from openshift) - List predictor pods for correlation context + +**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). + +**Additional environment variables**: +- `AI_OBSERVABILITY_MCP_URL` - URL for the AI Observability MCP server (e.g., `http://aiobs-mcp.apps.cluster.example.com`) + +**Additional cluster requirements**: +- AI Observability MCP server deployed on-cluster (from `quay.io/ecosystem-appeng/aiobs-mcp-server`) +- Prometheus configured with vLLM and OpenShift metrics +- Tempo configured for distributed tracing (optional, for trace analysis) +- Korrel8r deployed (optional, for cross-domain correlation) + +## When to Use This Skill + +**Use this skill when you need to:** +- Check model inference performance (latency, throughput, error rates) +- View GPU inventory and utilization across the cluster +- Analyze OpenShift cluster health metrics by category +- Trace slow inference requests with distributed tracing (Tempo) +- Correlate signals across logs, metrics, traces, and alerts (Korrel8r) +- Run custom PromQL queries against cluster Prometheus + +**Do NOT use this skill when:** +- You need to troubleshoot a failed deployment (use `/debug-inference`) +- You want to deploy or redeploy a model (use `/model-deploy`) +- You need to create or modify a ServingRuntime (use `/serving-runtime-config`) + +## Workflow + +### Step 1: Verify MCP and Triage Intent + +**Verify ai-observability MCP server is reachable.** If any tool call fails with a connection error: + +``` +Cannot execute /ai-observability: ai-observability MCP server is not reachable. + +Setup Instructions: +1. Deploy the server on your cluster from quay.io/ecosystem-appeng/aiobs-mcp-server + See: https://github.com/rh-ai-quickstart/ai-observability-summarizer +2. Set AI_OBSERVABILITY_MCP_URL to the server route URL +3. Restart Claude Code to reload MCP servers + +Options: setup (show deployment guide) / abort +``` + +Proceed automatically without waiting for user input. + +**Classify user query** into one of these domains: + +| Domain | Trigger Phrases | Primary Tool(s) | +|--------|----------------|-----------------| +| Model Discovery | "what models", "list models", "what's deployed" | `list_models`, `list_vllm_namespaces` | +| GPU Inventory | "GPU", "GPU utilization", "what GPUs", "available hardware" | `get_gpu_info` | +| vLLM Performance | "latency", "throughput", "performance", "how is [model]", "slow" | `analyze_vllm` | +| OpenShift Health | "cluster health", "namespace metrics", "node health", "pods" | `analyze_openshift` | +| Tracing | "trace", "latency trace", "slow request", "spans" | `query_tempo_tool` | +| Correlation | "correlate", "root cause", "what's related to" | `korrel8r_get_correlated` | +| Custom PromQL | "PromQL", "custom query", "specific metric" | `execute_promql` | + +If the intent is ambiguous, present the domain options and ask the user to choose. + +If the user specifies a model name, use `list_models` first to verify it exists and get the correct identifier. If the user does not specify a namespace, use `list_vllm_namespaces` or `list_openshift_namespaces` to discover available namespaces and present them. + +**Project context** (if `rhoai` MCP available): For "what's running" or "what's deployed" queries, use `list_data_science_projects` (from rhoai) to provide project-level overview. Use `list_inference_services` (from rhoai) per project to show deployed models with status. + +Proceed automatically without waiting for user input. + +### Step 2: Execute Analysis + +Branch based on the domain determined in Step 1. + +#### Step 2a: Model Discovery + +**MCP Tool**: `list_models` (from ai-observability) + +**Parameters**: None + +**MCP Tool**: `list_vllm_namespaces` (from ai-observability) + +**Parameters**: None + +Present results: + +| Model Name | Namespace | +|------------|-----------| +| [model] | [namespace] | + +**Offer**: "Would you like to analyze performance for a specific model, or check GPU inventory?" + +Proceed automatically without waiting for user input. + +#### Step 2b: GPU Inventory + +**MCP Tool**: `get_gpu_info` (from ai-observability) + +**Parameters**: None + +Present results: + +| Node | GPU Type | Count | Temperature | Power Usage | +|------|----------|-------|-------------|-------------| +| [node] | [type] | [count] | [temp] | [watts] | + +If GPUs are near capacity, note: "Some GPUs are heavily utilized. Check model performance or consider scaling." + +**Offer**: "Would you like to check which models are using these GPUs, or analyze a specific model's performance?" + +Proceed automatically without waiting for user input. + +#### Step 2c: vLLM Performance Analysis + +Requires: model name (from user or discovered via `list_models` in Step 1). + +**MCP Tool**: `get_deployment_info` (from ai-observability) + +**Parameters**: +- `namespace`: model's namespace - REQUIRED +- `model`: model name - REQUIRED + +Report deployment status (is_new_deployment, deployment_date). + +**MCP Tool**: `analyze_vllm` (from ai-observability) + +**Parameters**: +- `model_name`: vLLM model identifier - REQUIRED +- `summarize_model_id`: LLM for analysis (use server default if not specified) - REQUIRED +- `time_range`: natural language time range, e.g., `"15m"`, `"1h"`, `"24h"` - OPTIONAL (default: `"15m"`) +- `start_datetime`: ISO datetime string - OPTIONAL (alternative to time_range) +- `end_datetime`: ISO datetime string - OPTIONAL (alternative to time_range) + +Present the LLM-generated analysis covering: latency (p50/p95/p99), throughput (requests/sec), token rates (input/output tokens/sec), error rate, queue depth. + +**Offer**: +``` +Would you like to: +1. Ask a follow-up question about these metrics +2. Trace a slow inference request +3. Correlate with other signals (logs, alerts) +4. Check a different time range +5. Exit analysis +``` + +Proceed automatically without waiting for user input. + +If user asks a follow-up question: + +**MCP Tool**: `chat_vllm` (from ai-observability) + +**Parameters**: +- `model_name`: same model name - REQUIRED +- `prompt_summary`: the analysis output from `analyze_vllm` - REQUIRED +- `question`: the user's follow-up question - REQUIRED +- `summarize_model_id`: LLM for response - REQUIRED + +#### Step 2d: OpenShift Health Analysis + +**MCP Tool**: `list_openshift_metric_groups` (from ai-observability) + +**Parameters**: None + +Present available metric categories to user if they did not specify one. + +Proceed automatically without waiting for user input. + +**MCP Tool**: `analyze_openshift` (from ai-observability) + +**Parameters**: +- `metric_category`: the selected category (e.g., `"Fleet Overview"`, `"GPU & Accelerators"`, `"Workloads & Pods"`, `"Storage & Networking"`) - REQUIRED +- `scope`: `"cluster_wide"` or `"namespace_scoped"` - OPTIONAL (default: `"cluster_wide"`) +- `namespace`: required when scope is `"namespace_scoped"` - CONDITIONAL +- `time_range`: natural language time range - OPTIONAL +- `start_datetime`: ISO datetime string - OPTIONAL +- `end_datetime`: ISO datetime string - OPTIONAL + +Present the health assessment and key metrics. + +**Offer**: "Would you like to check another metric category, drill into a specific namespace, or exit?" + +Proceed automatically without waiting for user input. + +#### Step 2e: Distributed Tracing + +Requires: service name or operation name, and time range. + +**MCP Tool**: `query_tempo_tool` (from ai-observability) + +**Parameters**: +- `query`: TraceQL query string (e.g., `"{resource.service.name=\"[service]\"}"`) - REQUIRED +- `start_time`: ISO datetime string (e.g., `"2024-01-01T00:00:00Z"`) - REQUIRED +- `end_time`: ISO datetime string - REQUIRED +- `limit`: max traces to return - OPTIONAL (default: 10) + +Present traces: + +| Trace ID | Duration (ms) | Root Service | Span Count | Start Time | +|----------|--------------|--------------|------------|------------| +| [id] | [duration] | [service] | [spans] | [time] | + +**Ask**: "Would you like to drill into a specific trace? Enter a Trace ID." + +Proceed automatically without waiting for user input. + +If user selects a trace: + +**MCP Tool**: `get_trace_details_tool` (from ai-observability) + +**Parameters**: +- `trace_id`: the trace ID string - REQUIRED + +Present span waterfall: + +| Span | Service | Operation | Duration (ms) | Status | +|------|---------|-----------|---------------|--------| +| [span-id] | [service] | [operation] | [duration] | [ok/error] | + +**Offer**: "Would you like to view another trace, correlate this trace with logs/metrics, or exit?" + +Proceed automatically without waiting for user input. + +#### Step 2f: Cross-Domain Correlation (Korrel8r) + +Requires: a starting point (pod name and namespace, or other Korrel8r domain query). + +**MCP Tool**: `korrel8r_get_correlated` (from ai-observability) + +**Parameters**: +- `query`: Korrel8r domain query string - REQUIRED + - Example: `k8s:Pod:{"namespace":"llm-serving","name":"vllm-predictor-abc"}` +- `goals`: array of target domain class names - REQUIRED + - Example: `["log:application", "metric:metric", "trace:span", "alert:alert"]` + +Present correlated signals grouped by domain: + +**Related Logs**: [count] log entries found +**Related Metrics**: [count] metric series +**Related Traces**: [count] trace spans +**Related Alerts**: [count] active alerts + +**Offer**: "Would you like to drill into any of these correlated signals?" + +Proceed automatically without waiting for user input. + +#### Step 2g: Custom PromQL Query + +For advanced users who want to run specific PromQL. + +**MCP Tool**: `search_metrics` (from ai-observability) + +**Parameters**: +- `pattern`: search string (e.g., `"vllm latency"`) - OPTIONAL (default: `""`) +- `limit`: max results, 1-1000 - OPTIONAL (default: 50) + +Present matching metrics with their descriptions. Let user select or compose a query. + +**MCP Tool**: `execute_promql` (from ai-observability) + +**Parameters**: +- `query`: PromQL query string - REQUIRED +- `time_range`: relative time range (e.g., `"5m"`, `"1h"`) - OPTIONAL +- `start_datetime`: ISO datetime string - OPTIONAL +- `end_datetime`: ISO datetime string - OPTIONAL + +Present query results. + +**Offer**: "Would you like to run another query, or exit?" + +Proceed automatically without waiting for user input. + +### Step 3: Follow-Up and Drill-Down + +After presenting initial results, offer domain-appropriate follow-up options: + +- For vLLM analysis: use `chat_vllm` for conversational follow-up +- For traces: allow drilling into specific trace IDs via `get_trace_details_tool` +- For correlation: allow drilling into correlated signals +- For any domain: offer to switch to a different analysis domain + +Present options and Proceed automatically without waiting for user input. + +### Step 4: Summary and Next Steps + +When the user chooses to exit: + +Summarize key findings from the analysis session. + +**If issues were found**, suggest: +- `/debug-inference` for deployment or pod-level problems +- `/model-deploy` to redeploy with different configuration +- Custom PromQL queries for ongoing monitoring + +**If everything looks healthy**, confirm: "All monitored metrics are within normal ranges." + +## Common Issues + +### Issue 1: AI Observability MCP Server Not Deployed + +**Error**: Connection refused or timeout when reaching `AI_OBSERVABILITY_MCP_URL` + +**Cause**: The AI Observability MCP server is not deployed on the cluster, or the route/service is not accessible. + +**Solution:** +1. Deploy the server from `quay.io/ecosystem-appeng/aiobs-mcp-server` -- see https://github.com/rh-ai-quickstart/ai-observability-summarizer +2. Verify the route is accessible: `oc get route -n [namespace] aiobs-mcp` +3. Set `AI_OBSERVABILITY_MCP_URL` to the route URL +4. Restart Claude Code to reload MCP servers + +### Issue 2: No Models Found in Monitoring + +**Error**: `list_models` returns empty results + +**Cause**: vLLM metrics are not being scraped by Prometheus, or no InferenceServices are deployed. + +**Solution:** +1. Verify InferenceServices exist: use `resources_list` from `openshift` MCP +2. Check that Prometheus ServiceMonitor is configured for vLLM metrics +3. Verify the vLLM serving container exposes `/metrics` endpoint + +### Issue 3: Tempo Traces Not Available + +**Error**: `query_tempo_tool` returns empty or connection error + +**Cause**: Tempo is not deployed, or distributed tracing is not configured for the inference stack. + +**Solution:** +1. Verify Tempo is deployed in the cluster +2. Check OpenTelemetry instrumentation on the inference endpoints +3. Verify Tempo datasource is configured in the MCP server + +### Issue 4: Korrel8r Correlation Returns No Results + +**Error**: `korrel8r_get_correlated` returns empty correlation + +**Cause**: Korrel8r is not deployed, or the query format is incorrect. + +**Solution:** +1. Verify Korrel8r is deployed and accessible +2. Check the query format matches Korrel8r domain syntax (e.g., `k8s:Pod:{"namespace":"[ns]","name":"[pod]"}`) +3. Ensure the target pod/namespace exists and has generated observability signals + +## Dependencies + +### MCP Tools +See [Prerequisites](#prerequisites) for the complete list of required and optional MCP tools. + +### Related Skills +- `/debug-inference` - Troubleshoot deployment issues found during analysis +- `/model-deploy` - Redeploy models with different configuration based on findings +- `/serving-runtime-config` - Adjust runtime parameters if performance issues are runtime-related + +### Reference Documentation +- [known-model-profiles.md](../../docs/references/known-model-profiles.md) - Expected performance baselines for common models +- [supported-runtimes.md](../../docs/references/supported-runtimes.md) - Runtime capabilities and known limitations + +## Critical: Human-in-the-Loop Requirements + +See [skill-conventions.md](../references/skill-conventions.md) for general HITL and security conventions. + +**Skill-specific checkpoints:** +- After triage (Step 1): confirm analysis scope (model, namespace, time range) before running queries +- After initial analysis (Step 2): present follow-up options, wait for user choice +- After correlation (Step 2f): confirm before drilling into correlated signals +- **NEVER** expose raw Prometheus/Tempo credentials or internal cluster endpoints in output +- **NEVER** execute unbounded PromQL queries (no time limit, extremely wide label selectors) without confirming with the user diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/common-issues.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/instruction.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/instruction.md new file mode 100644 index 00000000..f76c1829 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/instruction.md @@ -0,0 +1,13 @@ +# AI Observability Task + +You are an AI engineer on Red Hat OpenShift AI. Your team has deployed several inference services, but has no visibility into how they are performing or whether resources are sized correctly. + +## Requirements +- Assess the current state of deployed inference services and their resource consumption +- Define a metrics strategy covering: inference latency, throughput, error rates, and GPU memory utilization +- Identify any models that appear over-provisioned or under-provisioned based on current usage +- Recommend specific resource adjustments (CPU, memory, GPU, replicas) with justification + +Document your observability strategy and resource recommendations in `/root/report.md`. + +Use MCP tools to interact with the platform. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/solution/solve.sh b/evaluation/with_skills/rh-ai-engineer__ai-observability/solution/solve.sh new file mode 100644 index 00000000..d319c204 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/solution/solve.sh @@ -0,0 +1,23 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# AI Observability Report + +## Model: fraud-detection + +### GPU Utilization (last 15m) +- GPU memory: 65% utilized +- GPU compute: 45% utilized +- Recommendation: GPU is underutilized, consider right-sizing + +### Model Latency +- P50: 120ms +- P99: 450ms + +### Right-Sizing +- Current: 1x A100 80GB +- Recommended: 1x A100 40GB (sufficient for workload) + +### Advanced Observability +- execute_promql for custom metrics (e.g., vllm:request_success:ratio) +- query_tempo_tool for trace latency on slow requests +REPORT_EOF diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/task.toml b/evaluation/with_skills/rh-ai-engineer__ai-observability/task.toml new file mode 100644 index 00000000..130dece1 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-ai-engineer__ai-observability" +name = "rh-ai-engineer AI Observability Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-ai-engineer", "ai-observability", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/tests/llm_judge.py b/evaluation/with_skills/rh-ai-engineer__ai-observability/tests/llm_judge.py new file mode 100644 index 00000000..28b8d6a0 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "observability_stack_awareness", + "file": "/root/report.md", + "question": "Does the report recommend specific observability tools from the OpenShift AI ecosystem — specifically Tempo for distributed tracing AND Korrel8r for cross-domain correlation — as part of the observability strategy?", + "reference": "A skilled report should mention BOTH Tempo (for distributed tracing of inference requests) and Korrel8r (for correlating signals across logs, metrics, traces, and alerts). These are specific tools the skill teaches. A report that only recommends generic Prometheus/Grafana monitoring without Tempo or Korrel8r is insufficient." + }, + { + "id": "gpu_metric_specificity", + "file": "/root/report.md", + "question": "Does the report reference DCGM (Data Center GPU Manager) metric names (like DCGM_FI_DEV_FB_USED or DCGM_FI_DEV_GPU_UTIL) for GPU monitoring, rather than generic nvidia_gpu_memory metric names?", + "reference": "A skilled report uses DCGM-specific metric names (DCGM_FI_DEV_*) which are the actual metrics exposed by the GPU operator on OpenShift. Using generic names like nvidia_gpu_memory_used_bytes suggests the agent doesn't know the specific metric naming convention." + }, + { + "id": "vllm_tuning_specificity", + "file": "/root/report.md", + "question": "Does the report recommend specific vLLM configuration parameters (like --max-model-len, --gpu-memory-utilization, or tensor parallelism) for resolving GPU memory issues, rather than only recommending generic resource increases?", + "reference": "A skilled report should mention vLLM-specific tuning args like --max-model-len to limit KV cache size, --gpu-memory-utilization to control memory allocation, or tensor parallelism for multi-GPU distribution. Only recommending 'increase memory to 32Gi' without vLLM-specific configuration is insufficient." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/tests/test.sh b/evaluation/with_skills/rh-ai-engineer__ai-observability/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/tests/test_outputs.py b/evaluation/with_skills/rh-ai-engineer__ai-observability/tests/test_outputs.py new file mode 100644 index 00000000..eb3755b2 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ai-observability/tests/test_outputs.py @@ -0,0 +1,91 @@ +""" +Tests for rh-ai-engineer__ai-observability per-skill evaluation. +Baseline tests: any competent agent should pass. +Skill-dependent tests: based on empirical gaps between skilled and unskilled agent outputs. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ["monitor", "metric", "observ", "inference"]), ( + "report should mention monitoring or observability" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 200, "report should have substantial content" + + +class TestSkillDependent: + def test_tempo_distributed_tracing(self): + """Skill teaches Tempo for distributed tracing of inference requests. + Without skill, agents don't mention Tempo at all.""" + c = read_report().lower() + assert any(t in c for t in ["tempo", "distributed trac"]), ( + "should recommend Tempo for distributed tracing" + ) + + def test_korrel8r_correlation(self): + """Skill teaches Korrel8r for cross-domain signal correlation. + Without skill, agents don't know about Korrel8r.""" + c = read_report().lower() + assert any(t in c for t in ["korrel8r", "cross-domain correlation"]), ( + "should mention Korrel8r for cross-domain correlation" + ) + + def test_dcgm_gpu_metric_names(self): + """Skill teaches DCGM-specific GPU metric names (DCGM_FI_DEV_*). + Without skill, agents use generic nvidia_gpu_memory_* names.""" + c = read_report() + assert any(t in c for t in ["DCGM_FI_DEV", "dcgm_fi_dev", "DCGM"]), ( + "should reference DCGM GPU metric names (not generic nvidia_gpu_*)" + ) + + def test_opentelemetry_instrumentation(self): + """Skill teaches OpenTelemetry for trace instrumentation on inference endpoints. + Without skill, agents don't mention OpenTelemetry.""" + c = read_report().lower() + assert any(t in c for t in ["opentelemetry", "otel"]), ( + "should recommend OpenTelemetry instrumentation" + ) + + def test_vllm_tuning_args(self): + """Skill teaches vLLM CLI args for memory management. + Without skill, agents recommend generic resource increases but not vLLM-specific tuning.""" + c = read_report().lower() + assert any(t in c for t in [ + "max-model-len", "max_model_len", "gpu-memory-utilization", + "gpu_memory_utilization", "tensor parallel", "tensor_parallel", + ]), "should mention vLLM-specific configuration args for resource tuning" + + def test_latency_percentiles(self): + """Both agents should report latency percentiles (easy test).""" + c = read_report().lower() + assert any(t in c for t in ["p50", "p95", "p99"]), ( + "should report latency with percentiles" + ) + + def test_tensor_parallel_size_tuning(self): + """Docs teach reducing --tensor-parallel-size as GPU scheduling triage step, + and OOM mitigation via --max-model-len and quantized models (AWQ/GPTQ/FP8). + Without docs, agents don't know these vLLM tuning parameters.""" + c = read_report().lower() + assert any(t in c for t in [ + "tensor-parallel-size", "tensor_parallel_size", "tensor parallel", + "awq", "gptq", "fp8", "quantiz", + ]), "should address tensor-parallel-size and quantization for GPU tuning" diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/Dockerfile b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/Dockerfile new file mode 100644 index 00000000..d4978abe --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/Dockerfile @@ -0,0 +1,74 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "rhoai": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-rhoai-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/common-issues.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/examples/model-deploy.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/examples/model-deploy.md new file mode 100644 index 00000000..dcd79e44 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/examples/model-deploy.md @@ -0,0 +1,160 @@ +# /model-deploy Examples + +Example interactions demonstrating the model-deploy skill workflow. + +## Example 1: Deploy Llama 3.1 8B with vLLM + +``` +User: "Deploy Llama 3.1 8B on my cluster" + +Agent: [Invokes /model-deploy skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand runtime capabilities. + +Step 1 of 10: Gathering deployment information. + +| Setting | Value | Source | +|---------|-------|--------| +| Model | Llama 3.1 8B Instruct | user input | +| Runtime | TBD | auto-detect | +| Namespace | ? | needs input | +| Model Source | ? | needs input | +| Deployment Mode | Serverless | default | + +What namespace should I deploy to, and where are the model weights stored? +(e.g., S3 bucket, OCI registry, PVC) + +User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" + +Agent: Step 2 of 10: Determining runtime. + +I consulted supported-runtimes.md to understand runtime capabilities. + +Based on Llama 3.1 8B (open-source HuggingFace model): + +Recommended runtime: vLLM +Reason: Default runtime for open-source LLMs. Broad compatibility, + OpenAI-compatible API, no additional setup required. + +Proceed with vLLM? (yes / no / suggest alternative) + +User: "yes" + +Agent: Step 3 of 10: Looking up hardware profile. + +I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. + +| Requirement | Value | +|-------------|-------| +| GPUs | 1x A10G/L4/A100/H100 | +| VRAM | 16GB | +| Key Args | --max-model-len=4096 --tool-call-parser hermes | + +Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). +Ensure your cluster has at least 1 GPU with 16GB VRAM. + +Step 5 of 10: Skipped (NIM platform check not needed for vLLM). + +Step 6-7 of 10: Generating InferenceService YAML. + +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: llama-3-1-8b + namespace: ai-models + annotations: + serving.kserve.io/deploymentMode: Serverless +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: vllm-runtime + storageUri: s3://my-bucket/llama-3.1-8b + resources: + limits: + nvidia.com/gpu: "1" + requests: + cpu: "4" + memory: "16Gi" + args: + - --max-model-len=4096 + - --tool-call-parser=hermes + +| Setting | Value | +|---------|-------| +| Model | llama-3-1-8b | +| Runtime | vllm-runtime | +| GPUs | 1 | +| Namespace | ai-models | +| Mode | Serverless | + +Proceed with creating this InferenceService? (yes/no/modify) + +User: "yes" + +Agent: Step 8 of 10: Creating InferenceService... + [Creates via resources_create_or_update] + + InferenceService created. Monitoring rollout... + +Step 9 of 10: Monitoring rollout. + +| Stage | Status | +|-------|--------| +| Pod Scheduled | Complete | +| Image Pulled | Complete | +| Container Started | Complete | +| Model Loaded | Complete | +| Ready | Complete | + +Step 10 of 10: Deployment complete! + +Model Deployment Successful + +Model: llama-3-1-8b +Runtime: vLLM +Namespace: ai-models +GPUs: 1 + +Inference Endpoint: + URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com + API: OpenAI-compatible REST + +Quick Test: + curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' + +Next Steps: + Monitor performance: "/ai-observability llama-3-1-8b" +``` + +## Example 2: Deploy a Model with NIM (NIM Not Set Up) + +``` +User: "Deploy Llama 3.1 70B with NIM" + +Agent: [Invokes /model-deploy skill] + ... + Step 2: User requested NIM runtime. + + Step 5: Checking NIM platform... + +NIM platform is not set up in namespace ai-models. + +The NIM runtime requires a one-time platform setup. Would you like to: +1. Run /nim-setup now to configure the NIM platform +2. Switch to vLLM runtime instead (no setup required) +3. Abort deployment + +User: "1" + +Agent: [Invokes /nim-setup skill — walks through NGC credential setup] + ... + NIM Platform Setup Complete. + + Resuming model deployment... + [Continues from Step 6 with NIM runtime] +``` diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/examples/nim-setup.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/examples/nim-setup.md new file mode 100644 index 00000000..ccfc1a5d --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/examples/nim-setup.md @@ -0,0 +1,115 @@ +# /nim-setup Examples + +Example interactions demonstrating the nim-setup skill workflow. + +## Example 1: First-Time NIM Setup + +``` +User: "Set up NIM on my cluster" + +Agent: [Invokes /nim-setup skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand NIM platform requirements. + +Let me verify the prerequisites for NIM setup. + +Step 1 of 7: Checking GPU Operator and NFD Operator... + +GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) +NFD Operator: nfd.v4.19.0 (Succeeded) + +Both operators are installed and healthy. + +Step 2 of 7: I need your NGC credentials. + +To set up NIM, I need your NVIDIA NGC API key. +You can generate one at: https://ngc.nvidia.com/setup/api-key + +Please provide: +1. Your NGC API key +2. The target namespace for NIM resources + +User: "Here's my key: [user provides NGC API key], namespace: ai-models" + +Agent: Step 3 of 7: Creating NGC image pull secret... + +I'll create a docker-registry secret for pulling NIM images from nvcr.io: + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-image-pull-secret + namespace: ai-models + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] + +Should I create this image pull secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + Image pull secret created successfully. + +Step 4 of 7: Creating NGC API key secret... + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-api-key + namespace: ai-models + type: Opaque + stringData: + NGC_API_KEY: "[REDACTED]" + +Should I create this API key secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + API key secret created successfully. + +Step 5 of 7: Creating NIM Account CR... + + apiVersion: nim.opendatahub.io/v1 + kind: Account + metadata: + name: nim-account + namespace: ai-models + spec: + apiKeySecret: + name: ngc-api-key + imagePullSecret: + name: ngc-image-pull-secret + +Should I create this NIM Account CR in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates Account CR via resources_create_or_update] + Account CR created successfully. + +Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) + +User: "no" + +Agent: Step 7 of 7: Validating NIM platform readiness... + +NIM Platform Setup Complete + +Account CR: `nim-account` (namespace: ai-models) + Status: Ready + +NGC Credentials: + Image Pull Secret: ngc-image-pull-secret [created] + API Key Secret: ngc-api-key [created] + +NIM ServingRuntimes Available: + - nim-llama-3-1-8b-instruct + - nim-llama-3-1-70b-instruct + - nim-mistral-7b-instruct + +Next Steps: + To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" +``` diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/references/known-model-profiles.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/references/known-model-profiles.md new file mode 100644 index 00000000..40148981 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/references/known-model-profiles.md @@ -0,0 +1,83 @@ +--- +title: Known Model Hardware Profiles +category: references +tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] +semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] +use_cases: [model-deploy, debug-inference, ai-observability] +last_updated: 2026-03-10 +--- + +# Known Model Hardware Profiles + +Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). + +**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. + +## Model Source Conventions + +Each model lists a recommended `storageUri` with its authentication requirements: +- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** +- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). +- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. + +When the user does not specify a model source, use the `hf://` URI listed in the profile below. + +## Llama 3.x (Meta) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | +| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | +| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | +| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | + +- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) +- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly + +## Granite 3.x (IBM/Red Hat) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | +| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | + +- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) +- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser granite --chat-template` +- Red Hat-supported model family on RHOAI + +## Mixtral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | +| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) +- Mixture-of-Experts architecture: only ~13B/45B parameters active per token + +## Mistral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | +| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) + +## When a Model Is Not Listed + +If the requested model is not in this file, the agent MUST use the live doc lookup protocol: + +1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure +2. Fetch hardware specs from the appropriate source +3. Report findings to the user before proceeding with deployment + +Common cases requiring live lookup: +- Newly released models (after this file's last update) +- Domain-specific fine-tuned models +- Models with custom quantization +- NIM-specific optimized profiles diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/references/supported-runtimes.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/references/supported-runtimes.md new file mode 100644 index 00000000..9d018651 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/references/supported-runtimes.md @@ -0,0 +1,104 @@ +--- +title: Supported Serving Runtimes +category: references +tags: [runtimes, vllm, nim, caikit, tgis, serving] +semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] +use_cases: [model-deploy, serving-runtime-config, nim-setup] +last_updated: 2026-02-26 +--- + +# Supported Serving Runtimes + +This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. + +## Runtime Comparison + +| Runtime | API Type | Model Format | GPU Support | License | Setup Required | +|---------|----------|-------------|-------------|---------|----------------| +| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | +| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | +| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | +| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | + +## vLLM + +**Default runtime for most open-source models.** + +- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) +- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized +- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) +- **Model source**: S3-compatible storage, OCI registry, PVC, URI +- **Key features**: + - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) + - PagedAttention for efficient memory management + - Tensor parallelism for multi-GPU inference + - Tool/function calling support (`--tool-call-parser`) + - Continuous batching for high throughput +- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name + +## NVIDIA NIM + +**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** + +- **API**: OpenAI-compatible REST +- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog +- **GPU support**: NVIDIA only (requires specific GPU models per model profile) +- **Model source**: NGC model catalog (pulled automatically via NGC credentials) +- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) +- **Key features**: + - TensorRT-LLM optimization for lower latency + - Pre-compiled model engines (no compilation on first load) + - Optimized and generic profiles per GPU type + - Automatic model download from NGC +- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. +- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM +- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. +- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state + +## Caikit+TGIS + +**Red Hat's Caikit format with Text Generation Inference Server.** + +- **API**: gRPC (not REST) +- **Model formats**: Caikit format (requires conversion from HuggingFace) +- **GPU support**: NVIDIA +- **Model source**: S3-compatible storage +- **Key features**: + - Red Hat-supported runtime + - gRPC API for streaming inference + - Integrated with RHOAI model serving platform +- **When to choose**: When the model is already in Caikit format or when gRPC API is required. +- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name + +## Custom Runtimes + +**User-provided ServingRuntime CRs for unsupported frameworks.** + +- **API**: Defined by the custom runtime +- **Model formats**: Defined by the custom runtime +- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. +- **How to create**: Use `/serving-runtime-config` skill +- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility + +## Runtime Selection Decision Tree + +``` +Is the user's preferred runtime explicitly stated? +├── Yes → Use that runtime +└── No → Continue + +Is the model available in the NGC NIM catalog? +├── Yes → Suggest NIM (with vLLM as fallback) +│ Note: Requires /nim-setup and NVIDIA GPUs +└── No → Continue + +Is the model in Caikit format? +├── Yes → Caikit+TGIS +└── No → Continue + +Is the model a standard open-source LLM (HuggingFace-compatible)? +├── Yes → vLLM (default) +└── No → Custom runtime via /serving-runtime-config +``` diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..e7a4d11c --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +"""Mock OpenShift MCP server for SkillsBench rh-ai-engineer task. + +Simulates Kubernetes resource CRUD, pod management, logs, and events. + +Key scenario elements: +- LimitRange in namespaces: min CPU=100m, min memory=128Mi + (conflicts with KServe sidecar containers hardcoded at 10m CPU/15Mi memory) +- GPU node with custom taint ai-workload=true:NoSchedule +- NIM Account CR in ml-production: not ready (NGC credentials invalid) +- text-gen-legacy pods: OOMKilled (max-model-len=32768 on A10G) +- nim-llama-prod: no pods created (Account CR not ready) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + +# ── Cluster state ──────────────────────────────────────────────────────── + +GPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "gpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + "nvidia.com/gpu.present": "true", + "nvidia.com/gpu.product": "NVIDIA-A10G", + }, + }, + "spec": { + "taints": [ + { + "key": "ai-workload", + "value": "true", + "effect": "NoSchedule", + }, + ], + }, + "status": { + "allocatable": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "capacity": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "conditions": [ + {"type": "Ready", "status": "True"}, + ], + }, +} + +CPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "cpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + }, + }, + "spec": {"taints": []}, + "status": { + "allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "capacity": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +MASTER_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "master-1", + "labels": { + "node-role.kubernetes.io/master": "", + "node-role.kubernetes.io/control-plane": "", + }, + }, + "spec": { + "taints": [ + {"key": "node-role.kubernetes.io/master", "effect": "NoSchedule"}, + ], + }, + "status": { + "allocatable": {"cpu": "8", "memory": "32Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +ALL_NODES = [GPU_NODE, CPU_NODE, MASTER_NODE] + +# LimitRange applied by cluster policy to all DS project namespaces +NAMESPACE_LIMITRANGE = { + "apiVersion": "v1", + "kind": "LimitRange", + "metadata": { + "name": "default-limits", + }, + "spec": { + "limits": [ + { + "type": "Container", + "default": { + "cpu": "2", + "memory": "4Gi", + }, + "defaultRequest": { + "cpu": "500m", + "memory": "256Mi", + }, + "min": { + "cpu": "100m", + "memory": "128Mi", + }, + "max": { + "cpu": "32", + "memory": "128Gi", + }, + }, + ], + }, +} + +NIM_ACCOUNT_CR = { + "apiVersion": "nim.opendatahub.io/v1", + "kind": "Account", + "metadata": { + "name": "nim-account", + "namespace": "ml-production", + }, + "spec": { + "apiKeySecret": { + "name": "ngc-api-key", + }, + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "NGCCredentialsInvalid", + "message": "NGC API key validation failed: 401 Unauthorized. " + "The API key in secret 'ngc-api-key' is expired or invalid. " + "Re-create the secret with a valid NGC API key from " + "https://ngc.nvidia.com/setup/api-key and restart the " + "Account reconciliation.", + "lastTransitionTime": "2026-03-14T12:00:00Z", + }, + ], + "nimPullSecretStatus": "Failed", + "nimConfigStatus": "Pending", + }, +} + +SERVING_RUNTIME_VLLM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "vllm-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "vLLM", "version": "1", "autoSelect": True}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "quay.io/modh/vllm:rhoai-2.16", + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + }, + ], + }, +} + +SERVING_RUNTIME_NIM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "nim-serving-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "NIM", "version": "1"}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "nvcr.io/nim/meta/llama-3.1-8b-instruct:latest", + "ports": [{"containerPort": 8000, "protocol": "TCP"}], + "env": [ + {"name": "NGC_API_KEY", "valueFrom": { + "secretKeyRef": {"name": "ngc-api-key", "key": "api_key"}, + }}, + ], + }, + ], + }, +} + +PODS_BY_NAMESPACE = { + "ml-production": [ + { + "name": "text-gen-legacy-predictor-00001-abc12", + "namespace": "ml-production", + "status": "CrashLoopBackOff", + "restarts": 5, + "node": "gpu-worker-1", + "containers": [ + { + "name": "kserve-container", + "state": "waiting", + "reason": "CrashLoopBackOff", + "last_termination_reason": "OOMKilled", + "last_termination_exit_code": 137, + }, + ], + "labels": { + "serving.kserve.io/inferenceservice": "text-gen-legacy", + }, + "gpu": "1", + }, + # nim-llama-prod: NO pods created (Account CR not ready) + ], +} + +POD_LOGS = { + "text-gen-legacy-predictor-00001-abc12": ( + "INFO 2026-03-01 10:00:00 vllm_engine.py:125] vLLM engine starting...\n" + "INFO 2026-03-01 10:00:01 config.py:89] Model: mistralai/Mistral-7B-Instruct-v0.3\n" + "INFO 2026-03-01 10:00:01 config.py:92] max_model_len = 32768\n" + "INFO 2026-03-01 10:00:02 gpu_executor.py:45] GPU 0: NVIDIA A10G (24576 MiB)\n" + "INFO 2026-03-01 10:00:03 model_runner.py:88] Loading model weights...\n" + "INFO 2026-03-01 10:00:15 model_runner.py:112] Model weights loaded: 13.5 GiB\n" + "INFO 2026-03-01 10:00:15 worker.py:201] Allocating KV cache...\n" + "ERROR 2026-03-01 10:00:16 worker.py:215] torch.cuda.OutOfMemoryError: " + "CUDA out of memory. Tried to allocate 28.5 GiB for KV cache but only " + "10.1 GiB available after loading model weights (13.5 GiB).\n" + "ERROR 2026-03-01 10:00:16 vllm_engine.py:178] Engine failed to start\n" + "Traceback (most recent call last):\n" + " File \"/opt/vllm/vllm/engine/engine.py\", line 175, in start\n" + " self._init_kv_cache()\n" + " File \"/opt/vllm/vllm/worker/worker.py\", line 215, in _init_kv_cache\n" + " raise torch.cuda.OutOfMemoryError(msg)\n" + "torch.cuda.OutOfMemoryError: CUDA out of memory\n" + ), +} + +EVENTS_BY_NAMESPACE = { + "ml-production": [ + { + "type": "Warning", + "reason": "BackOff", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Back-off restarting failed container kserve-container in pod " + "text-gen-legacy-predictor-00001-abc12", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Warning", + "reason": "OOMKilled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Container kserve-container was OOMKilled (exit code 137). " + "GPU memory exhausted during KV cache allocation.", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Normal", + "reason": "Scheduled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Successfully assigned ml-production/" + "text-gen-legacy-predictor-00001-abc12 to gpu-worker-1", + "count": 1, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-02-28T08:00:00Z", + }, + { + "type": "Warning", + "reason": "NIMAccountNotReady", + "object": "InferenceService/nim-llama-prod", + "message": "NIM Account 'nim-account' in namespace 'ml-production' " + "is not ready", + "count": 12, + "first_timestamp": "2026-03-14T12:00:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + { + "type": "Warning", + "reason": "ImagePullBackOff", + "object": "InferenceService/nim-llama-prod", + "message": "Failed to pull image 'nvcr.io/nim/meta/llama-3.1-8b-instruct:" + "latest': unauthorized: authentication required", + "count": 8, + "first_timestamp": "2026-03-14T12:05:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + ], +} + + +# ── Resource tools ─────────────────────────────────────────────────────── + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: str = "", +) -> str: + """Get a single Kubernetes resource by apiVersion, kind, and name.""" + if kind == "Node": + for node in ALL_NODES: + if node["metadata"]["name"] == name: + return json.dumps(node, indent=2) + raise ValueError(f"Node '{name}' not found") + + if kind == "ServingRuntime": + if name == "vllm-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_VLLM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + if name == "nim-serving-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_NIM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + raise ValueError(f"ServingRuntime '{name}' not found in namespace '{namespace}'") + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps(lr, indent=2) + + if kind == "Account" and "nim" in apiVersion.lower(): + if namespace == "ml-production" and name == "nim-account": + return json.dumps(NIM_ACCOUNT_CR, indent=2) + raise ValueError( + f"Account '{name}' not found in namespace '{namespace}'" + ) + + if kind == "ClusterVersion" and apiVersion == "config.openshift.io/v1": + return json.dumps({ + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterVersion", + "metadata": {"name": "version"}, + "status": {"desired": {"version": "4.16.3"}}, + }) + + raise ValueError(f"Resource {apiVersion}/{kind}/{name} not found") + + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: str = "", + labelSelector: str = "", +) -> str: + """List Kubernetes resources by apiVersion and kind.""" + if kind == "Node": + nodes = ALL_NODES + if labelSelector: + parts = labelSelector.split("=", 1) + key = parts[0] + value = parts[1] if len(parts) > 1 else "" + nodes = [ + n for n in nodes + if n["metadata"]["labels"].get(key) == value + ] + return json.dumps(nodes, indent=2) + + if kind == "Service" and apiVersion == "serving.knative.dev/v1": + return json.dumps({ + "kind": "ServiceList", + "apiVersion": "serving.knative.dev/v1", + "items": [], + "metadata": {}, + }) + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps({ + "kind": "LimitRangeList", + "items": [lr], + }) + + if kind == "InferenceService": + return json.dumps({ + "kind": "InferenceServiceList", + "items": [], + }) + + raise ValueError(f"Unsupported list: {apiVersion}/{kind}") + + +@mcp.tool() +def pods_list( + namespace: str, + labelSelector: str = "", +) -> str: + """List pods in a namespace with optional label selector.""" + pods = PODS_BY_NAMESPACE.get(namespace, []) + + if labelSelector: + key, _, value = labelSelector.partition("=") + pods = [p for p in pods if p.get("labels", {}).get(key) == value] + + results = [] + for pod in pods: + results.append({ + "name": pod["name"], + "namespace": pod["namespace"], + "status": pod["status"], + "restarts": pod.get("restarts", 0), + "node": pod.get("node", ""), + "containers": pod.get("containers", []), + "gpu": pod.get("gpu", "0"), + }) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def pods_log( + namespace: str, + name: str, + container: str = "", +) -> str: + """Get logs from a pod container.""" + logs = POD_LOGS.get(name) + if logs is None: + raise ValueError(f"Pod '{name}' not found in namespace '{namespace}'") + return logs + + +@mcp.tool() +def events_list(namespace: str) -> str: + """List events in a namespace.""" + events = EVENTS_BY_NAMESPACE.get(namespace, []) + return json.dumps(events, indent=2) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/mcp-servers/mock-rhoai-mcp.py b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/mcp-servers/mock-rhoai-mcp.py new file mode 100644 index 00000000..0ae9e4cb --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/mcp-servers/mock-rhoai-mcp.py @@ -0,0 +1,780 @@ +#!/usr/bin/env python3 +"""Mock RHOAI MCP server for SkillsBench rh-ai-engineer task. + +Simulates Red Hat OpenShift AI operations: Data Science Projects, +model serving, data connections, serving runtimes, inference services. + +Scenario: +- ml-production: existing project with two broken deployments + - text-gen-legacy: vLLM OOMKilled (max-model-len=32768 on A10G) + - nim-llama-prod: NIM failing (Account CR not ready, NGC creds invalid) +- fraud-detection: does not exist yet (agent creates it) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("rhoai") + +# ── In-memory state ────────────────────────────────────────────────────── + +PROJECTS = { + "ml-production": { + "name": "ml-production", + "display_name": "ML Production", + "description": "Production ML workloads", + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": "single", + "pipeline_server": True, + }, +} + +DATA_CONNECTIONS = { + "ml-production": [ + { + "name": "prod-model-store", + "type": "S3", + "bucket": "ml-models-prod", + "endpoint": "https://s3.us-east-1.amazonaws.com", + "region": "us-east-1", + }, + ], +} + +SERVING_RUNTIMES = { + "__platform_templates__": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "REST", + "supported_model_formats": [ + {"name": "vLLM", "version": "1", "autoSelect": True} + ], + }, + { + "name": "caikit-tgis-runtime", + "display_name": "Caikit+TGIS ServingRuntime", + "model_formats": ["caikit"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "gRPC", + }, + ], + "ml-production": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + { + "name": "nim-serving-runtime", + "display_name": "NVIDIA NIM ServingRuntime", + "model_formats": ["NIM"], + "requires_instantiation": False, + "source": "nim-account", + "api_protocol": "REST", + }, + { + "name": "ovms-1", + "display_name": "OpenVINO Model Server", + "model_formats": ["openvino_ir", "onnx"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + ], +} + +INFERENCE_SERVICES = { + "ml-production": { + "text-gen-legacy": { + "name": "text-gen-legacy", + "namespace": "ml-production", + "runtime": "vllm-runtime", + "model_format": "vLLM", + "storage_uri": "hf://mistralai/Mistral-7B-Instruct-v0.3", + "display_name": "Mistral 7B Legacy", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "16Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "PredictorFailed", + "message": "Predictor pod is not ready", + }, + { + "type": "PredictorReady", + "status": "False", + "reason": "ContainerCrashLoop", + "message": "Container kserve-container terminated: " + "OOMKilled (exit code 137). 5 restarts.", + }, + { + "type": "IngressReady", + "status": "True", + "reason": "IngressReady", + "message": "Ingress is ready", + }, + ], + "age": "3d", + }, + "nim-llama-prod": { + "name": "nim-llama-prod", + "namespace": "ml-production", + "runtime": "nim-serving-runtime", + "model_format": "NIM", + "storage_uri": "nim://meta/llama-3.1-8b-instruct", + "display_name": "Llama 3.1 8B (NIM)", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "32Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "RuntimeNotReady", + "message": "ServingRuntime 'nim-serving-runtime' " + "is not in ready state", + }, + { + "type": "PredictorReady", + "status": "Unknown", + "reason": "PodNotCreated", + "message": "Predictor pod has not been created. " + "Waiting for ServingRuntime to become ready.", + }, + { + "type": "IngressReady", + "status": "Unknown", + "reason": "PredictorNotReady", + "message": "Waiting for predictor to become ready", + }, + ], + "age": "1d", + }, + }, +} + +DEPLOYED_MODELS = {} + +WORKBENCHES = { + "ml-production": [ + { + "name": "data-exploration-nb", + "display_name": "Data Exploration", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Running", + "cpu_request": "1", + "memory_request": "8Gi", + "gpu_count": 0, + "pvc_name": "data-exploration-nb-pvc", + "pvc_size": "20Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-10T09:00:00Z", + }, + { + "name": "model-training-nb", + "display_name": "Model Training", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Stopped", + "cpu_request": "4", + "memory_request": "16Gi", + "gpu_count": 1, + "pvc_name": "model-training-nb-pvc", + "pvc_size": "50Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-15T14:00:00Z", + }, + ], +} + +PIPELINE_SERVERS = { + "ml-production": { + "configured": True, + "data_connection": "prod-model-store", + "status": "Ready", + "database": "MariaDB", + }, +} + +NOTEBOOK_IMAGES = [ + {"name": "jupyter-pytorch-ubi9-python-3.9-2024.1", "display_name": "PyTorch 2024.1", "packages": ["torch", "transformers"]}, + {"name": "jupyter-tensorflow-ubi9-python-3.9-2024.1", "display_name": "TensorFlow 2024.1", "packages": ["tensorflow"]}, + {"name": "jupyter-datascience-ubi9-python-3.9-2024.1", "display_name": "Standard Data Science", "packages": ["pandas", "scikit-learn"]}, + {"name": "jupyter-minimal-ubi9-python-3.9-2024.1", "display_name": "Minimal Python", "packages": []}, +] + + +# ── Project tools ──────────────────────────────────────────────────────── + + +@mcp.tool() +def list_data_science_projects() -> str: + """List all RHOAI Data Science Projects on the cluster.""" + projects = [] + for name, proj in PROJECTS.items(): + projects.append({ + "name": name, + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + }) + return json.dumps(projects, indent=2) + + +@mcp.tool() +def create_data_science_project( + name: str, + display_name: str, + description: str = "", +) -> str: + """Create a new RHOAI Data Science Project (namespace with dashboard labels).""" + if name in PROJECTS: + raise ValueError( + f"Project '{name}' already exists. Choose a different name " + "or configure the existing project." + ) + if not name.replace("-", "").replace("_", "").isalnum() or len(name) > 63: + raise ValueError( + f"Invalid project name '{name}'. Must be DNS-compatible: " + "lowercase alphanumeric and hyphens, max 63 chars." + ) + + PROJECTS[name] = { + "name": name, + "display_name": display_name, + "description": description, + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": None, + "pipeline_server": False, + } + DATA_CONNECTIONS[name] = [] + SERVING_RUNTIMES[name] = [] + INFERENCE_SERVICES[name] = {} + + return json.dumps({ + "status": "created", + "name": name, + "display_name": display_name, + "namespace": name, + "labels": {"opendatahub.io/dashboard": "true"}, + }) + + +@mcp.tool() +def get_project_details(name: str) -> str: + """Get detailed information about an RHOAI Data Science Project.""" + if name not in PROJECTS: + raise ValueError(f"Project '{name}' not found") + proj = PROJECTS[name] + dc_count = len(DATA_CONNECTIONS.get(name, [])) + isvc_count = len(INFERENCE_SERVICES.get(name, {})) + return json.dumps({ + "name": proj["name"], + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "labels": proj["labels"], + "data_connections": dc_count, + "inference_services": isvc_count, + "model_serving_mode": proj.get("model_serving_mode"), + "pipeline_server": proj.get("pipeline_server", False), + }) + + +@mcp.tool() +def get_project_status(namespace: str) -> str: + """Get comprehensive status of an RHOAI Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Project '{namespace}' not found") + proj = PROJECTS[namespace] + dcs = DATA_CONNECTIONS.get(namespace, []) + isvcs = INFERENCE_SERVICES.get(namespace, {}) + return json.dumps({ + "namespace": namespace, + "display_name": proj["display_name"], + "status": "Active", + "components": { + "data_connections": len(dcs), + "inference_services": len(isvcs), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + "pipeline_server": "configured" if proj.get("pipeline_server") else "not configured", + }, + }) + + +# ── Data connection tools ──────────────────────────────────────────────── + + +@mcp.tool() +def create_s3_data_connection( + namespace: str, + name: str, + bucket: str, + endpoint: str, + access_key: str, + secret_key: str, + region: str = "", +) -> str: + """Create an S3-compatible data connection in an RHOAI project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + existing = DATA_CONNECTIONS.get(namespace, []) + if any(dc["name"] == name for dc in existing): + raise ValueError( + f"Data connection '{name}' already exists in namespace '{namespace}'" + ) + + dc = { + "name": name, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + "region": region, + } + DATA_CONNECTIONS.setdefault(namespace, []).append(dc) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + }) + + +@mcp.tool() +def list_data_connections(namespace: str) -> str: + """List data connections in an RHOAI project namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + return json.dumps(dcs, indent=2) + + +# ── Model serving tools ───────────────────────────────────────────────── + + +@mcp.tool() +def set_model_serving_mode(namespace: str, mode: str) -> str: + """Enable model serving on a Data Science Project (single or multi mode).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + if mode not in ("single", "multi"): + raise ValueError(f"Invalid mode '{mode}'. Must be 'single' or 'multi'.") + + PROJECTS[namespace]["model_serving_mode"] = mode + + if not SERVING_RUNTIMES.get(namespace): + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + SERVING_RUNTIMES[namespace] = [ + {**t, "requires_instantiation": False, "source": "existing"} + for t in templates + ] + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "mode": mode, + }) + + +@mcp.tool() +def list_serving_runtimes( + namespace: str, + include_templates: bool = False, +) -> str: + """List available ServingRuntimes in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + runtimes = list(SERVING_RUNTIMES.get(namespace, [])) + if include_templates: + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + existing_names = {r["name"] for r in runtimes} + for t in templates: + if t["name"] not in existing_names: + runtimes.append(t) + + return json.dumps(runtimes, indent=2) + + +# ── Inference service tools ────────────────────────────────────────────── + + +@mcp.tool() +def deploy_model( + name: str, + namespace: str, + runtime: str, + model_format: str, + storage_uri: str, + display_name: str = "", + min_replicas: int = 1, + max_replicas: int = 1, + cpu_request: str = "1", + cpu_limit: str = "2", + memory_request: str = "4Gi", + memory_limit: str = "8Gi", + gpu_count: int = 0, +) -> str: + """Deploy an AI/ML model as a KServe InferenceService.""" + if namespace not in PROJECTS: + raise ValueError( + f"Namespace '{namespace}' is not a Data Science Project. " + "Create one via create_data_science_project first." + ) + + ns_runtimes = SERVING_RUNTIMES.get(namespace, []) + runtime_names = [r["name"] for r in ns_runtimes] + if runtime not in runtime_names: + available = ", ".join(runtime_names) or "none" + raise ValueError( + f"ServingRuntime '{runtime}' not found in namespace '{namespace}'. " + f"Available runtimes: {available}" + ) + + endpoint = f"https://{name}-{namespace}.apps.ocp-cluster.example.com" + isvc = { + "name": name, + "namespace": namespace, + "runtime": runtime, + "model_format": model_format, + "storage_uri": storage_uri, + "display_name": display_name or name, + "gpu_count": gpu_count, + "cpu_request": cpu_request, + "memory_request": memory_request, + "min_replicas": min_replicas, + "max_replicas": max_replicas, + "ready": True, + "url": endpoint, + "conditions": [ + {"type": "Ready", "status": "True", "reason": "Ready", "message": ""}, + {"type": "PredictorReady", "status": "True", "reason": "PodReady", "message": ""}, + {"type": "IngressReady", "status": "True", "reason": "IngressReady", "message": ""}, + ], + "age": "0s", + } + + INFERENCE_SERVICES.setdefault(namespace, {})[name] = isvc + DEPLOYED_MODELS[f"{namespace}/{name}"] = isvc + + return json.dumps({ + "status": "deployed", + "name": name, + "namespace": namespace, + "runtime": runtime, + "endpoint": endpoint, + "ready": True, + }) + + +@mcp.tool() +def list_inference_services( + namespace: str, + verbosity: str = "standard", +) -> str: + """List deployed InferenceServices in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + isvcs = INFERENCE_SERVICES.get(namespace, {}) + results = [] + for isvc_name, isvc in isvcs.items(): + entry = { + "name": isvc["name"], + "runtime": isvc["runtime"], + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "age": isvc.get("age", ""), + } + if verbosity == "full": + entry["conditions"] = isvc.get("conditions", []) + entry["storage_uri"] = isvc.get("storage_uri", "") + entry["gpu_count"] = isvc.get("gpu_count", 0) + results.append(entry) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def get_inference_service( + name: str, + namespace: str, + verbosity: str = "standard", +) -> str: + """Get detailed status of a specific InferenceService.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + + isvc = isvcs[name] + result = { + "name": isvc["name"], + "namespace": isvc["namespace"], + "runtime": isvc["runtime"], + "model_format": isvc.get("model_format", ""), + "storage_uri": isvc.get("storage_uri", ""), + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "conditions": isvc.get("conditions", []), + "gpu_count": isvc.get("gpu_count", 0), + "replicas": {"min": isvc.get("min_replicas", 1), "max": isvc.get("max_replicas", 1)}, + "resources": { + "cpu_request": isvc.get("cpu_request", "1"), + "memory_request": isvc.get("memory_request", "4Gi"), + "memory_limit": isvc.get("memory_limit", "8Gi"), + }, + "age": isvc.get("age", ""), + } + return json.dumps(result, indent=2) + + +@mcp.tool() +def get_model_endpoint(name: str, namespace: str) -> str: + """Get the inference endpoint URL for a deployed model.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + isvc = isvcs[name] + if not isvc["ready"]: + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": "", + "error": "InferenceService is not ready. Check conditions for details.", + }) + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": isvc["url"], + }) + + +# ── Workbench tools ────────────────────────────────────────────────────── + + +@mcp.tool() +def list_workbenches(namespace: str) -> str: + """List workbenches (Jupyter notebooks) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + return json.dumps(wbs, indent=2) + + +@mcp.tool() +def create_workbench( + namespace: str, + name: str, + display_name: str = "", + image: str = "jupyter-datascience-ubi9-python-3.9-2024.1", + cpu_request: str = "1", + memory_request: str = "4Gi", + gpu_count: int = 0, + pvc_size: str = "20Gi", +) -> str: + """Create a new workbench (Jupyter notebook) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + valid_images = [img["name"] for img in NOTEBOOK_IMAGES] + if image not in valid_images: + raise ValueError( + f"Image '{image}' not found. Available: {', '.join(valid_images)}" + ) + + wb = { + "name": name, + "display_name": display_name or name, + "image": image, + "status": "Running", + "cpu_request": cpu_request, + "memory_request": memory_request, + "gpu_count": gpu_count, + "pvc_name": f"{name}-pvc", + "pvc_size": pvc_size, + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-03-02T12:00:00Z", + } + WORKBENCHES.setdefault(namespace, []).append(wb) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "image": image, + "pvc": f"{name}-pvc", + }) + + +@mcp.tool() +def stop_workbench(namespace: str, name: str) -> str: + """Stop a running workbench (preserves data).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Stopped" + return json.dumps({"status": "stopped", "name": name, "namespace": namespace}) + + +@mcp.tool() +def start_workbench(namespace: str, name: str) -> str: + """Start a stopped workbench.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Running" + return json.dumps({"status": "running", "name": name, "namespace": namespace}) + + +@mcp.tool() +def delete_workbench(namespace: str, name: str) -> str: + """Delete a workbench. WARNING: PVC data may be lost if not backed up.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wbs.remove(wb) + return json.dumps({ + "status": "deleted", + "name": name, + "namespace": namespace, + "warning": "Associated PVC data has been deleted", + }) + + +@mcp.tool() +def list_notebook_images() -> str: + """List available notebook images for workbench creation.""" + return json.dumps(NOTEBOOK_IMAGES, indent=2) + + +# ── Pipeline server tools ─────────────────────────────────────────────── + + +@mcp.tool() +def configure_pipeline_server( + namespace: str, + data_connection: str, + database: str = "MariaDB", +) -> str: + """Configure a pipeline server for a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + if not any(dc["name"] == data_connection for dc in dcs): + available = [dc["name"] for dc in dcs] + raise ValueError( + f"Data connection '{data_connection}' not found. Available: {available}" + ) + + PIPELINE_SERVERS[namespace] = { + "configured": True, + "data_connection": data_connection, + "status": "Ready", + "database": database, + } + PROJECTS[namespace]["pipeline_server"] = True + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "data_connection": data_connection, + "database": database, + }) + + +@mcp.tool() +def get_pipeline_server_status(namespace: str) -> str: + """Get the status of the pipeline server in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + ps = PIPELINE_SERVERS.get(namespace) + if not ps: + return json.dumps({"namespace": namespace, "configured": False}) + return json.dumps({ + "namespace": namespace, + "configured": ps["configured"], + "data_connection": ps["data_connection"], + "status": ps["status"], + "database": ps["database"], + }) + + +# ── Serving runtime creation ──────────────────────────────────────────── + + +@mcp.tool() +def create_serving_runtime( + namespace: str, + name: str, + display_name: str = "", + model_formats: list = None, + container_image: str = "", + container_port: int = 8080, + multi_model: bool = False, + api_protocol: str = "REST", +) -> str: + """Create a custom ServingRuntime in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + if not model_formats: + raise ValueError("model_formats must specify at least one model format") + + runtime = { + "name": name, + "display_name": display_name or name, + "model_formats": model_formats, + "requires_instantiation": False, + "source": "custom", + "api_protocol": api_protocol, + "container_image": container_image, + "container_port": container_port, + "multi_model": multi_model, + } + SERVING_RUNTIMES.setdefault(namespace, []).append(runtime) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "model_formats": model_formats, + }) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/debug-inference/SKILL.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/debug-inference/SKILL.md new file mode 100644 index 00000000..6a9a2d2a --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/debug-inference/SKILL.md @@ -0,0 +1,348 @@ +--- +name: debug-inference +description: | + Troubleshoot failed or slow InferenceService deployments on OpenShift AI. + + Use when: + - "My InferenceService won't start" + - "Model deployment is stuck" + - "Inference endpoint returns errors" + - "Model is slow / high latency" + - "GPU scheduling failed for my model" + + Progressive diagnosis: status conditions, events, pod logs, GPU health, and observability analysis. + + NOT for deploying models (use /model-deploy). + NOT for creating runtimes (use /serving-runtime-config). +model: inherit +color: yellow +--- + +# /debug-inference Skill + +Troubleshoot failed, stuck, or slow InferenceService deployments on Red Hat OpenShift AI. Performs progressive diagnosis through status conditions, events, pod logs, related resources, and optional observability analysis. Follows a 6-step diagnosis pattern with human-in-the-loop confirmation at each step. + +## Prerequisites + +**Required MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) + +**Required MCP Tools** (from rhoai): +- `list_inference_services` - List deployed models with structured status data +- `get_inference_service` - Get detailed deployment status (conditions, endpoint, ready state) +- `get_model_endpoint` - Quick check if endpoint is available (early diagnostic) + +**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools** (from openshift): +- `resources_get` (from openshift) - Get ServingRuntime, NIM Account CR details +- `pods_list` (from openshift) - Find predictor/transformer pods +- `pods_log` (from openshift) - Retrieve container logs +- `events_list` (from openshift) - Check events for errors + +**Optional MCP Server**: `ai-observability` ([AI Observability MCP](https://github.com/rh-ai-quickstart/ai-observability-summarizer)) + +**Optional MCP Tools** (from ai-observability): +- `get_deployment_info` - Check model initialization status +- `analyze_vllm` - Analyze vLLM performance bottlenecks (latency, throughput, errors, token rates) +- `chat_vllm` - Conversational follow-up on vLLM metrics during diagnosis +- `get_gpu_info` - GPU inventory and utilization +- `analyze_openshift` - Check GPU health with "GPU & Accelerators" category +- `query_tempo_tool` - Trace request latency by service/operation/time range +- `get_trace_details_tool` - Get detailed span-level info for a specific trace ID +- `execute_promql` - Run custom PromQL queries for metrics not covered by standard analysis +- `korrel8r_get_correlated` - Correlate signals (logs, traces, metrics, alerts) across a pod/namespace for root cause analysis + +**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, KServe, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). + +**Additional cluster requirements**: +- An existing InferenceService deployment to debug + +## When to Use This Skill + +**Use this skill when you need to:** +- Troubleshoot an InferenceService that won't start, is stuck, or shows errors +- Diagnose slow inference latency or high error rates +- Investigate GPU scheduling failures or OOMKilled pods +- Perform root cause analysis on model deployment issues + +**Do NOT use this skill when:** +- You want to deploy a new model (use `/model-deploy`) +- You want to analyze ongoing model performance (use `/ai-observability`) +- You need to create or fix a ServingRuntime (use `/serving-runtime-config`) +- You need to set up NIM credentials (use `/nim-setup`) + +## Workflow + +### Step 1: Identify Target InferenceService + +**Ask the user:** +- Which InferenceService is having issues? (name or "list all") +- What namespace is it in? +- What is the symptom? (won't start / slow / errors / other) + +If user says "list all" or is unsure: + +**MCP Tool**: `list_inference_services` (from rhoai) + +**Parameters**: +- `namespace`: user-specified namespace - REQUIRED +- `verbosity`: `"standard"` - OPTIONAL + +Present InferenceServices with their status: + +| Name | Runtime | Ready | URL | Age | +|------|---------|-------|-----|-----| +| [name] | [runtime] | [True/False/Unknown] | [url or "N/A"] | [age] | + +Proceed automatically without waiting for user input. + +### Step 2: Status Overview + +**MCP Tool**: `get_inference_service` (from rhoai) + +**Parameters**: +- `name`: the InferenceService name - REQUIRED +- `namespace`: user-specified namespace - REQUIRED +- `verbosity`: `"full"` - REQUIRED + +**Early endpoint check:** + +**MCP Tool**: `get_model_endpoint` (from rhoai) +- `name`: the InferenceService name, `namespace`: user-specified namespace + +An empty or error URL indicates deployment issues. Report endpoint availability status. + +Present status conditions: + +| Condition | Status | Reason | Message | +|-----------|--------|--------|---------| +| Ready | [True/False/Unknown] | [reason] | [message] | +| PredictorReady | [True/False/Unknown] | [reason] | [message] | +| IngressReady | [True/False/Unknown] | [reason] | [message] | + +**Quick Assessment**: Based on conditions, provide initial assessment (e.g., "PredictorReady is False -- the model container is not running. Likely a pod-level issue.") + +**Ask**: "Continue with deep analysis of events and pods? (yes/no)" + +Proceed automatically without waiting for user input. + +### Step 3: Events and Pod Analysis + +**MCP Tool**: `events_list` (from openshift) + +**Parameters**: +- `namespace`: user-specified namespace - REQUIRED + +Filter events related to the InferenceService name. + +**MCP Tool**: `pods_list` (from openshift) + +**Parameters**: +- `namespace`: user-specified namespace - REQUIRED +- `labelSelector`: `"serving.kserve.io/inferenceservice=[isvc-name]"` - REQUIRED + +Present findings: + +**Events:** + +| Time | Type | Reason | Message | +|------|------|--------|---------| +| [time] | [Normal/Warning] | [reason] | [message] | + +**Predictor Pods:** + +| Pod | Status | Restarts | Node | GPU | +|-----|--------|----------|------|-----| +| [pod-name] | [status] | [count] | [node] | [gpu-count] | + +**Issues Found:** +- [Issue from events or pod status] + +**Ask**: "Continue to view pod logs? (yes/no)" + +Proceed automatically without waiting for user input. + +### Step 4: Pod Logs Review + +**MCP Tool**: `pods_log` (from openshift) + +**Parameters**: +- `namespace`: user-specified namespace - REQUIRED +- `name`: predictor pod name from Step 3 - REQUIRED +- `container`: `"kserve-container"` - REQUIRED (main serving container) + +If the container has restarted, also retrieve previous logs. + +Present log analysis: + +**Log Analysis:** +- [Error pattern identified, e.g., "CUDA out of memory", "S3 access denied", "Model not found"] +- [Relevant log line with explanation] + +**For NIM-specific deployments**, also check: +- NGC authentication errors in logs +- TensorRT engine compilation status +- GPU compatibility messages + +**If the error is unrecognized -> Trigger live doc lookup:** +1. **Action**: Read [live-doc-lookup.md](../references/live-doc-lookup.md) using the Read tool +2. Use **WebFetch** to look up the error message in RHOAI documentation +3. **Output to user**: "I looked up this error on [source]: [explanation and fix]" + +**Ask**: "Continue to check related resources and observability? (yes/no)" + +Proceed automatically without waiting for user input. + +### Step 5: Related Resources and Observability + +**Check ServingRuntime:** + +**MCP Tool**: `resources_get` (from openshift) + +**Parameters**: +- `apiVersion`: `"serving.kserve.io/v1alpha1"` - REQUIRED +- `kind`: `"ServingRuntime"` - REQUIRED +- `namespace`: user-specified namespace - REQUIRED +- `name`: runtime name from the InferenceService spec - REQUIRED + +Verify the runtime exists and its model format matches the InferenceService. + +**For NIM deployments -- Check Account CR:** + +**MCP Tool**: `resources_get` (from openshift) + +**Parameters**: +- `apiVersion`: `"nim.opendatahub.io/v1alpha1"` - REQUIRED +- `kind`: `"Account"` - REQUIRED +- `namespace`: user-specified namespace - REQUIRED +- `name`: `"nim-account"` - REQUIRED + +**If ai-observability MCP is available:** + +- `get_deployment_info`: Check if the model appears in monitoring and its initialization status +- `analyze_vllm`: Analyze performance metrics for slow inference (latency, throughput, errors, token rates) +- `chat_vllm`: Ask follow-up questions about analyzed metrics (e.g., "Why is latency spiking?") +- `analyze_openshift` with category `"GPU & Accelerators"`: Check GPU health and utilization +- `query_tempo_tool`: Trace request latency if the symptom is slow responses +- `get_trace_details_tool`: Drill into a specific trace ID to see span-level timing +- `execute_promql`: Run custom PromQL queries for deeper metric investigation (e.g., `vllm:request_success:ratio`, GPU memory utilization) +- `korrel8r_get_correlated`: Correlate signals across the inference stack -- find related logs, traces, metrics, and alerts for the failing pod/namespace (query example: `k8s:Pod:{"namespace":"[ns]","name":"[pod-name]"}`, goals: `["log:application", "metric:metric", "trace:span"]`) + +**If ai-observability not available**: Skip with note: "Observability analysis skipped (ai-observability MCP not configured)." + +**Present findings:** +- ServingRuntime status and compatibility +- NIM Account CR status (if applicable) +- Observability insights (if available) + +**Ask**: "Continue to diagnosis summary? (yes/no)" + +Proceed automatically without waiting for user input. + +### Step 6: Diagnosis Summary + +Present a structured diagnosis: + +``` +## Diagnosis Summary: [isvc-name] + +### Root Cause + +**Primary Issue:** [Categorized root cause] + +| Category | Status | Details | +|----------|--------|---------| +| ServingRuntime | [OK/FAIL] | [details] | +| Pod Scheduling | [OK/FAIL] | [details] | +| Container Start | [OK/FAIL] | [details] | +| Model Loading | [OK/FAIL] | [details] | +| GPU Access | [OK/FAIL] | [details] | +| Endpoint Health | [OK/FAIL] | [details] | + +### Evidence + +- [Evidence 1 from events/logs/status] +- [Evidence 2] + +### Recommended Actions + +1. **[Action 1]** - [description] +2. **[Action 2]** - [description] +3. **[Action 3]** - [description] + +### Verification Steps + +After applying fixes: +1. Check InferenceService status: `resources_get` for the InferenceService +2. Verify pod is running: `pods_list` with label selector +3. Test endpoint: curl command to the inference URL +``` + +**End with options:** + +``` +Would you like me to: +1. Execute a recommended fix +2. Dig deeper into a specific area +3. Debug a related resource (ServingRuntime, pod, NIM Account) +4. Invoke /serving-runtime-config to fix runtime issues +5. Exit debugging +``` + +Proceed automatically without waiting for user input. + +## Common Issues + +For common issues (GPU scheduling, OOMKilled, image pull errors, RBAC), see [common-issues.md](../references/common-issues.md). + +### Issue 1: S3 Storage Access Denied + +**Error**: Pod logs show "Access Denied" or "NoSuchBucket" when loading model weights + +**Cause**: S3 credentials are missing, expired, or the bucket/path is incorrect. + +**Solution:** +1. Verify the `storageUri` in the InferenceService spec +2. Check that the S3 credential Secret exists in the namespace +3. Verify the Secret is referenced by the ServiceAccount or data connection +4. Test S3 access independently to confirm credentials are valid + +### Issue 2: NIM Authentication / GPU Incompatibility + +**Error**: NIM pod logs show NGC authentication failure, or TensorRT engine fails to compile for the available GPU + +**Cause**: NGC API key is invalid/expired, or the GPU type is not supported by the NIM model profile. + +**Solution:** +1. Check Account CR status for credential errors: `resources_get` for `accounts.nim.opendatahub.io` +2. Verify NGC API key is valid at https://ngc.nvidia.com +3. Check NIM supported GPU matrix via live doc lookup against [NVIDIA NIM supported models](https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html) +4. Re-run `/nim-setup` to refresh credentials if expired + +## Dependencies + +### MCP Tools +See [Prerequisites](#prerequisites) for the complete list of required and optional MCP tools. + +### Related Skills +- `/model-deploy` - Redeploy or modify the InferenceService after fixing issues +- `/serving-runtime-config` - Fix or create ServingRuntime if runtime is the issue +- `/nim-setup` - Re-run NIM platform setup if NIM credentials are the issue + +### Reference Documentation +- [known-model-profiles.md](../../docs/references/known-model-profiles.md) - Correct resource sizing for common models +- [supported-runtimes.md](../../docs/references/supported-runtimes.md) - Runtime capabilities and known limitations +- [live-doc-lookup.md](../references/live-doc-lookup.md) - Protocol for looking up unrecognized errors + +## Critical: Human-in-the-Loop Requirements + +See [skill-conventions.md](../references/skill-conventions.md) for general HITL and security conventions. + +**Skill-specific checkpoints:** +- After identifying target (Step 1): confirm which InferenceService to debug +- After status overview (Step 2): confirm before deep analysis +- After events/pod analysis (Step 3): confirm before viewing logs +- After log review (Step 4): confirm before checking related resources +- After diagnosis summary (Step 6): present options, wait for user decision +- **NEVER** auto-delete or auto-modify InferenceService resources without user confirmation +- **NEVER** execute remediation actions without presenting the plan and getting explicit approval diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/common-issues.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/instruction.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/instruction.md new file mode 100644 index 00000000..11b9268d --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/instruction.md @@ -0,0 +1,13 @@ +# Inference Debugging Task + +You are an AI engineer on Red Hat OpenShift AI. There are failing model inference deployments in the `ml-production` namespace that need debugging. + +## Requirements +- List all InferenceServices in the `ml-production` namespace and identify which ones are not ready +- For each failing InferenceService, diagnose the root cause: check status conditions, pod state, container logs, events, and related resources (ServingRuntime, Account CRs) +- Recommend a specific fix for each failing deployment +- Document your methodology and the diagnostic steps you followed + +Use MCP tools to interact with the platform. Write your complete findings and recommendations in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/solution/solve.sh b/evaluation/with_skills/rh-ai-engineer__debug-inference/solution/solve.sh new file mode 100644 index 00000000..6b94e02f --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/solution/solve.sh @@ -0,0 +1,36 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Inference Debug Report + +## Diagnosis Categories (get_inference_service verbosity full) + +### 1. ServingRuntime ✓ +ServingRuntime CR exists and is valid + +### 2. Pod Scheduling ✗ +Pod cannot be scheduled — check kserve-container logs (pods_log container=kserve-container) +Label selector: serving.kserve.io/inferenceservice + +### 3. Container Start +KServe sidecar containers may conflict with LimitRange + +### 4. Model Loading +Check model download and initialization + +### 5. GPU Access +Verify GPU allocation and CUDA compatibility + +### 6. Endpoint Health +Check InferenceService URL and readiness (PredictorReady, IngressReady conditions) + +## Events +events_list filtered by namespace for pod/InferenceService events + +## NIM Deployments +For NIM: Check Account CR (nim.opendatahub.io) for NGC credential errors + +## Observability (optional) +- korrel8r_get_correlated for cross-domain signals +- query_tempo_tool for trace latency +- execute_promql for custom metrics +REPORT_EOF diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/task.toml b/evaluation/with_skills/rh-ai-engineer__debug-inference/task.toml new file mode 100644 index 00000000..44c5ea2a --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-ai-engineer__debug-inference" +name = "rh-ai-engineer Inference Debugging Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-ai-engineer", "debug-inference", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/tests/llm_judge.py b/evaluation/with_skills/rh-ai-engineer__debug-inference/tests/llm_judge.py new file mode 100644 index 00000000..748256ca --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/tests/llm_judge.py @@ -0,0 +1,114 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "kserve_conditions_awareness", + "file": "/root/report.md", + "question": "Does the report explicitly name the KServe InferenceService status conditions — specifically PredictorReady and IngressReady — and present them in a structured conditions table with Status/Reason/Message columns?", + "reference": "A skilled report should present a conditions table showing PredictorReady and IngressReady as distinct conditions with their status (True/False), reason, and message. Simply reporting 'CrashLoopBackOff' or 'pod failing' without naming the specific KServe conditions is insufficient." + }, + { + "id": "kserve_container_specificity", + "file": "/root/report.md", + "question": "Does the report mention 'kserve-container' by name as the specific container to inspect for logs, and reference the serving.kserve.io/inferenceservice label selector as the method for discovering predictor pods?", + "reference": "A skilled report should mention 'kserve-container' as the container name for log inspection and reference the serving.kserve.io/inferenceservice label selector for pod discovery. Generically saying 'check pod logs' or 'look at the container' without these specific KServe identifiers is insufficient." + }, + { + "id": "nim_account_cr_pattern", + "file": "/root/report.md", + "question": "Does the report prescribe creating a NIM Account custom resource (kind: Account) as the credential management mechanism for NVIDIA NIM, rather than only manually creating docker-registry secrets and patching service accounts?", + "reference": "A skilled report creates a NIM Account CR (kind: Account, apiVersion: nvidia.com/v1alpha1) with ngcSecret reference and imagePullSecret auto-creation. An unskilled report manually creates docker-registry secrets and patches service accounts without using the Account CR pattern." + }, + { + "id": "ngc_credential_expiry", + "file": "/root/report.md", + "question": "Does the report identify NGC API key or pull-secret expiry as a possible root cause for image pull failures in NIM deployments, and recommend checking the secret's expiration date?", + "reference": "A skilled report checks whether the NGC pull-secret has expired as a diagnosis step for ImagePullBackOff. An unskilled report treats image pull failures generically without considering credential expiry." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/tests/test.sh b/evaluation/with_skills/rh-ai-engineer__debug-inference/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/tests/test_outputs.py b/evaluation/with_skills/rh-ai-engineer__debug-inference/tests/test_outputs.py new file mode 100644 index 00000000..60f73901 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__debug-inference/tests/test_outputs.py @@ -0,0 +1,98 @@ +""" +Tests for rh-ai-engineer__debug-inference per-skill evaluation. +Baseline tests: any competent agent should pass. +Skill-dependent tests: based on empirical gaps between skilled and unskilled agent outputs. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ["inference", "model", "serving", "deploy"]), ( + "report should mention inference" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 200, "report should have substantial content" + + +class TestSkillDependent: + def test_kserve_status_conditions(self): + """Skill teaches presenting PredictorReady and IngressReady as distinct KServe conditions. + Without skill, agents report generic pod status (CrashLoopBackOff) without naming these conditions.""" + c = read_report().lower() + assert any(t in c for t in [ + "predictorready", "predictor ready", "predictor_ready", + "ingressready", "ingress ready", "ingress_ready", + ]), "should name KServe status conditions (PredictorReady, IngressReady)" + + def test_kserve_container_name(self): + """Skill teaches 'kserve-container' as the specific container for log inspection. + Without skill, agents check logs generically without naming this container.""" + c = read_report().lower() + assert "kserve-container" in c or "kserve container" in c, ( + "should mention kserve-container by name as the container to inspect" + ) + + def test_label_selector_methodology(self): + """Skill teaches using serving.kserve.io/inferenceservice label to find predictor pods. + Without skill, agents discover pods through generic namespace listing.""" + c = read_report().lower() + assert any(t in c for t in [ + "serving.kserve.io", "kserve.io/inferenceservice", + ]), "should reference the KServe label selector for predictor pod discovery" + + def test_account_cr_awareness(self): + """Skill teaches NIM Account CR as the credential management mechanism. + Without skill, agents manually create docker-registry secrets and + patch service accounts instead of using the Account custom resource.""" + c = read_report() + assert any(t in c for t in [ + "Account CR", "kind: Account", "Account resource", + "Account custom resource", + ]) or "account cr" in c.lower(), ( + "should reference NIM Account CR as credential management mechanism" + ) + + def test_nim_api_version(self): + """Skill teaches the nvidia.com API group for NIM Account and ngcSecret + field for NGC credential binding. Without skill, agents create + generic secrets without the Account CR pattern.""" + c = read_report().lower() + assert any(t in c for t in [ + "nvidia.com/v1alpha1", "ngcsecret", "ngc_api_key", + ]) or ("account" in c and "api" in c and "nvidia" in c), ( + "should reference NIM Account API version or NGC secret binding" + ) + + def test_root_cause_with_remediation(self): + """Both agents should link diagnosis to fix — easy test.""" + c = read_report().lower() + has_diagnosis = any(t in c for t in ["oom", "memory", "crash", "fail"]) + has_fix = any(t in c for t in ["fix", "recommend", "solution", "increase", "reduce"]) + assert has_diagnosis and has_fix, "should link diagnosis to recommended fix" + + def test_ngc_pull_secret_expiry(self): + """Docs teach NGC pull-secret expiry as a common issue, and + 'Insufficient nvidia.com/gpu' as GPU scheduling error signature. + Without docs, agents miss these specific failure patterns.""" + c = read_report().lower() + assert any(t in c for t in [ + "ngc", "pull-secret", "pull secret", "expir", + "insufficient nvidia.com/gpu", "nvidia.com/gpu", + ]), "should address NGC pull-secret expiry or GPU scheduling errors" diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/Dockerfile b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/Dockerfile new file mode 100644 index 00000000..d4978abe --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/Dockerfile @@ -0,0 +1,74 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "rhoai": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-rhoai-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/common-issues.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/model-deploy.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/model-deploy.md new file mode 100644 index 00000000..dcd79e44 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/model-deploy.md @@ -0,0 +1,160 @@ +# /model-deploy Examples + +Example interactions demonstrating the model-deploy skill workflow. + +## Example 1: Deploy Llama 3.1 8B with vLLM + +``` +User: "Deploy Llama 3.1 8B on my cluster" + +Agent: [Invokes /model-deploy skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand runtime capabilities. + +Step 1 of 10: Gathering deployment information. + +| Setting | Value | Source | +|---------|-------|--------| +| Model | Llama 3.1 8B Instruct | user input | +| Runtime | TBD | auto-detect | +| Namespace | ? | needs input | +| Model Source | ? | needs input | +| Deployment Mode | Serverless | default | + +What namespace should I deploy to, and where are the model weights stored? +(e.g., S3 bucket, OCI registry, PVC) + +User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" + +Agent: Step 2 of 10: Determining runtime. + +I consulted supported-runtimes.md to understand runtime capabilities. + +Based on Llama 3.1 8B (open-source HuggingFace model): + +Recommended runtime: vLLM +Reason: Default runtime for open-source LLMs. Broad compatibility, + OpenAI-compatible API, no additional setup required. + +Proceed with vLLM? (yes / no / suggest alternative) + +User: "yes" + +Agent: Step 3 of 10: Looking up hardware profile. + +I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. + +| Requirement | Value | +|-------------|-------| +| GPUs | 1x A10G/L4/A100/H100 | +| VRAM | 16GB | +| Key Args | --max-model-len=4096 --tool-call-parser hermes | + +Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). +Ensure your cluster has at least 1 GPU with 16GB VRAM. + +Step 5 of 10: Skipped (NIM platform check not needed for vLLM). + +Step 6-7 of 10: Generating InferenceService YAML. + +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: llama-3-1-8b + namespace: ai-models + annotations: + serving.kserve.io/deploymentMode: Serverless +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: vllm-runtime + storageUri: s3://my-bucket/llama-3.1-8b + resources: + limits: + nvidia.com/gpu: "1" + requests: + cpu: "4" + memory: "16Gi" + args: + - --max-model-len=4096 + - --tool-call-parser=hermes + +| Setting | Value | +|---------|-------| +| Model | llama-3-1-8b | +| Runtime | vllm-runtime | +| GPUs | 1 | +| Namespace | ai-models | +| Mode | Serverless | + +Proceed with creating this InferenceService? (yes/no/modify) + +User: "yes" + +Agent: Step 8 of 10: Creating InferenceService... + [Creates via resources_create_or_update] + + InferenceService created. Monitoring rollout... + +Step 9 of 10: Monitoring rollout. + +| Stage | Status | +|-------|--------| +| Pod Scheduled | Complete | +| Image Pulled | Complete | +| Container Started | Complete | +| Model Loaded | Complete | +| Ready | Complete | + +Step 10 of 10: Deployment complete! + +Model Deployment Successful + +Model: llama-3-1-8b +Runtime: vLLM +Namespace: ai-models +GPUs: 1 + +Inference Endpoint: + URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com + API: OpenAI-compatible REST + +Quick Test: + curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' + +Next Steps: + Monitor performance: "/ai-observability llama-3-1-8b" +``` + +## Example 2: Deploy a Model with NIM (NIM Not Set Up) + +``` +User: "Deploy Llama 3.1 70B with NIM" + +Agent: [Invokes /model-deploy skill] + ... + Step 2: User requested NIM runtime. + + Step 5: Checking NIM platform... + +NIM platform is not set up in namespace ai-models. + +The NIM runtime requires a one-time platform setup. Would you like to: +1. Run /nim-setup now to configure the NIM platform +2. Switch to vLLM runtime instead (no setup required) +3. Abort deployment + +User: "1" + +Agent: [Invokes /nim-setup skill — walks through NGC credential setup] + ... + NIM Platform Setup Complete. + + Resuming model deployment... + [Continues from Step 6 with NIM runtime] +``` diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/nim-setup.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/nim-setup.md new file mode 100644 index 00000000..ccfc1a5d --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/nim-setup.md @@ -0,0 +1,115 @@ +# /nim-setup Examples + +Example interactions demonstrating the nim-setup skill workflow. + +## Example 1: First-Time NIM Setup + +``` +User: "Set up NIM on my cluster" + +Agent: [Invokes /nim-setup skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand NIM platform requirements. + +Let me verify the prerequisites for NIM setup. + +Step 1 of 7: Checking GPU Operator and NFD Operator... + +GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) +NFD Operator: nfd.v4.19.0 (Succeeded) + +Both operators are installed and healthy. + +Step 2 of 7: I need your NGC credentials. + +To set up NIM, I need your NVIDIA NGC API key. +You can generate one at: https://ngc.nvidia.com/setup/api-key + +Please provide: +1. Your NGC API key +2. The target namespace for NIM resources + +User: "Here's my key: [user provides NGC API key], namespace: ai-models" + +Agent: Step 3 of 7: Creating NGC image pull secret... + +I'll create a docker-registry secret for pulling NIM images from nvcr.io: + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-image-pull-secret + namespace: ai-models + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] + +Should I create this image pull secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + Image pull secret created successfully. + +Step 4 of 7: Creating NGC API key secret... + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-api-key + namespace: ai-models + type: Opaque + stringData: + NGC_API_KEY: "[REDACTED]" + +Should I create this API key secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + API key secret created successfully. + +Step 5 of 7: Creating NIM Account CR... + + apiVersion: nim.opendatahub.io/v1 + kind: Account + metadata: + name: nim-account + namespace: ai-models + spec: + apiKeySecret: + name: ngc-api-key + imagePullSecret: + name: ngc-image-pull-secret + +Should I create this NIM Account CR in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates Account CR via resources_create_or_update] + Account CR created successfully. + +Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) + +User: "no" + +Agent: Step 7 of 7: Validating NIM platform readiness... + +NIM Platform Setup Complete + +Account CR: `nim-account` (namespace: ai-models) + Status: Ready + +NGC Credentials: + Image Pull Secret: ngc-image-pull-secret [created] + API Key Secret: ngc-api-key [created] + +NIM ServingRuntimes Available: + - nim-llama-3-1-8b-instruct + - nim-llama-3-1-70b-instruct + - nim-mistral-7b-instruct + +Next Steps: + To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" +``` diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/known-model-profiles.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/known-model-profiles.md new file mode 100644 index 00000000..40148981 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/known-model-profiles.md @@ -0,0 +1,83 @@ +--- +title: Known Model Hardware Profiles +category: references +tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] +semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] +use_cases: [model-deploy, debug-inference, ai-observability] +last_updated: 2026-03-10 +--- + +# Known Model Hardware Profiles + +Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). + +**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. + +## Model Source Conventions + +Each model lists a recommended `storageUri` with its authentication requirements: +- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** +- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). +- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. + +When the user does not specify a model source, use the `hf://` URI listed in the profile below. + +## Llama 3.x (Meta) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | +| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | +| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | +| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | + +- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) +- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly + +## Granite 3.x (IBM/Red Hat) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | +| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | + +- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) +- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser granite --chat-template` +- Red Hat-supported model family on RHOAI + +## Mixtral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | +| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) +- Mixture-of-Experts architecture: only ~13B/45B parameters active per token + +## Mistral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | +| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) + +## When a Model Is Not Listed + +If the requested model is not in this file, the agent MUST use the live doc lookup protocol: + +1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure +2. Fetch hardware specs from the appropriate source +3. Report findings to the user before proceeding with deployment + +Common cases requiring live lookup: +- Newly released models (after this file's last update) +- Domain-specific fine-tuned models +- Models with custom quantization +- NIM-specific optimized profiles diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/supported-runtimes.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/supported-runtimes.md new file mode 100644 index 00000000..9d018651 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/supported-runtimes.md @@ -0,0 +1,104 @@ +--- +title: Supported Serving Runtimes +category: references +tags: [runtimes, vllm, nim, caikit, tgis, serving] +semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] +use_cases: [model-deploy, serving-runtime-config, nim-setup] +last_updated: 2026-02-26 +--- + +# Supported Serving Runtimes + +This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. + +## Runtime Comparison + +| Runtime | API Type | Model Format | GPU Support | License | Setup Required | +|---------|----------|-------------|-------------|---------|----------------| +| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | +| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | +| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | +| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | + +## vLLM + +**Default runtime for most open-source models.** + +- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) +- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized +- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) +- **Model source**: S3-compatible storage, OCI registry, PVC, URI +- **Key features**: + - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) + - PagedAttention for efficient memory management + - Tensor parallelism for multi-GPU inference + - Tool/function calling support (`--tool-call-parser`) + - Continuous batching for high throughput +- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name + +## NVIDIA NIM + +**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** + +- **API**: OpenAI-compatible REST +- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog +- **GPU support**: NVIDIA only (requires specific GPU models per model profile) +- **Model source**: NGC model catalog (pulled automatically via NGC credentials) +- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) +- **Key features**: + - TensorRT-LLM optimization for lower latency + - Pre-compiled model engines (no compilation on first load) + - Optimized and generic profiles per GPU type + - Automatic model download from NGC +- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. +- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM +- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. +- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state + +## Caikit+TGIS + +**Red Hat's Caikit format with Text Generation Inference Server.** + +- **API**: gRPC (not REST) +- **Model formats**: Caikit format (requires conversion from HuggingFace) +- **GPU support**: NVIDIA +- **Model source**: S3-compatible storage +- **Key features**: + - Red Hat-supported runtime + - gRPC API for streaming inference + - Integrated with RHOAI model serving platform +- **When to choose**: When the model is already in Caikit format or when gRPC API is required. +- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name + +## Custom Runtimes + +**User-provided ServingRuntime CRs for unsupported frameworks.** + +- **API**: Defined by the custom runtime +- **Model formats**: Defined by the custom runtime +- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. +- **How to create**: Use `/serving-runtime-config` skill +- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility + +## Runtime Selection Decision Tree + +``` +Is the user's preferred runtime explicitly stated? +├── Yes → Use that runtime +└── No → Continue + +Is the model available in the NGC NIM catalog? +├── Yes → Suggest NIM (with vLLM as fallback) +│ Note: Requires /nim-setup and NVIDIA GPUs +└── No → Continue + +Is the model in Caikit format? +├── Yes → Caikit+TGIS +└── No → Continue + +Is the model a standard open-source LLM (HuggingFace-compatible)? +├── Yes → vLLM (default) +└── No → Custom runtime via /serving-runtime-config +``` diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..e7a4d11c --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +"""Mock OpenShift MCP server for SkillsBench rh-ai-engineer task. + +Simulates Kubernetes resource CRUD, pod management, logs, and events. + +Key scenario elements: +- LimitRange in namespaces: min CPU=100m, min memory=128Mi + (conflicts with KServe sidecar containers hardcoded at 10m CPU/15Mi memory) +- GPU node with custom taint ai-workload=true:NoSchedule +- NIM Account CR in ml-production: not ready (NGC credentials invalid) +- text-gen-legacy pods: OOMKilled (max-model-len=32768 on A10G) +- nim-llama-prod: no pods created (Account CR not ready) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + +# ── Cluster state ──────────────────────────────────────────────────────── + +GPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "gpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + "nvidia.com/gpu.present": "true", + "nvidia.com/gpu.product": "NVIDIA-A10G", + }, + }, + "spec": { + "taints": [ + { + "key": "ai-workload", + "value": "true", + "effect": "NoSchedule", + }, + ], + }, + "status": { + "allocatable": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "capacity": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "conditions": [ + {"type": "Ready", "status": "True"}, + ], + }, +} + +CPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "cpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + }, + }, + "spec": {"taints": []}, + "status": { + "allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "capacity": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +MASTER_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "master-1", + "labels": { + "node-role.kubernetes.io/master": "", + "node-role.kubernetes.io/control-plane": "", + }, + }, + "spec": { + "taints": [ + {"key": "node-role.kubernetes.io/master", "effect": "NoSchedule"}, + ], + }, + "status": { + "allocatable": {"cpu": "8", "memory": "32Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +ALL_NODES = [GPU_NODE, CPU_NODE, MASTER_NODE] + +# LimitRange applied by cluster policy to all DS project namespaces +NAMESPACE_LIMITRANGE = { + "apiVersion": "v1", + "kind": "LimitRange", + "metadata": { + "name": "default-limits", + }, + "spec": { + "limits": [ + { + "type": "Container", + "default": { + "cpu": "2", + "memory": "4Gi", + }, + "defaultRequest": { + "cpu": "500m", + "memory": "256Mi", + }, + "min": { + "cpu": "100m", + "memory": "128Mi", + }, + "max": { + "cpu": "32", + "memory": "128Gi", + }, + }, + ], + }, +} + +NIM_ACCOUNT_CR = { + "apiVersion": "nim.opendatahub.io/v1", + "kind": "Account", + "metadata": { + "name": "nim-account", + "namespace": "ml-production", + }, + "spec": { + "apiKeySecret": { + "name": "ngc-api-key", + }, + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "NGCCredentialsInvalid", + "message": "NGC API key validation failed: 401 Unauthorized. " + "The API key in secret 'ngc-api-key' is expired or invalid. " + "Re-create the secret with a valid NGC API key from " + "https://ngc.nvidia.com/setup/api-key and restart the " + "Account reconciliation.", + "lastTransitionTime": "2026-03-14T12:00:00Z", + }, + ], + "nimPullSecretStatus": "Failed", + "nimConfigStatus": "Pending", + }, +} + +SERVING_RUNTIME_VLLM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "vllm-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "vLLM", "version": "1", "autoSelect": True}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "quay.io/modh/vllm:rhoai-2.16", + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + }, + ], + }, +} + +SERVING_RUNTIME_NIM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "nim-serving-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "NIM", "version": "1"}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "nvcr.io/nim/meta/llama-3.1-8b-instruct:latest", + "ports": [{"containerPort": 8000, "protocol": "TCP"}], + "env": [ + {"name": "NGC_API_KEY", "valueFrom": { + "secretKeyRef": {"name": "ngc-api-key", "key": "api_key"}, + }}, + ], + }, + ], + }, +} + +PODS_BY_NAMESPACE = { + "ml-production": [ + { + "name": "text-gen-legacy-predictor-00001-abc12", + "namespace": "ml-production", + "status": "CrashLoopBackOff", + "restarts": 5, + "node": "gpu-worker-1", + "containers": [ + { + "name": "kserve-container", + "state": "waiting", + "reason": "CrashLoopBackOff", + "last_termination_reason": "OOMKilled", + "last_termination_exit_code": 137, + }, + ], + "labels": { + "serving.kserve.io/inferenceservice": "text-gen-legacy", + }, + "gpu": "1", + }, + # nim-llama-prod: NO pods created (Account CR not ready) + ], +} + +POD_LOGS = { + "text-gen-legacy-predictor-00001-abc12": ( + "INFO 2026-03-01 10:00:00 vllm_engine.py:125] vLLM engine starting...\n" + "INFO 2026-03-01 10:00:01 config.py:89] Model: mistralai/Mistral-7B-Instruct-v0.3\n" + "INFO 2026-03-01 10:00:01 config.py:92] max_model_len = 32768\n" + "INFO 2026-03-01 10:00:02 gpu_executor.py:45] GPU 0: NVIDIA A10G (24576 MiB)\n" + "INFO 2026-03-01 10:00:03 model_runner.py:88] Loading model weights...\n" + "INFO 2026-03-01 10:00:15 model_runner.py:112] Model weights loaded: 13.5 GiB\n" + "INFO 2026-03-01 10:00:15 worker.py:201] Allocating KV cache...\n" + "ERROR 2026-03-01 10:00:16 worker.py:215] torch.cuda.OutOfMemoryError: " + "CUDA out of memory. Tried to allocate 28.5 GiB for KV cache but only " + "10.1 GiB available after loading model weights (13.5 GiB).\n" + "ERROR 2026-03-01 10:00:16 vllm_engine.py:178] Engine failed to start\n" + "Traceback (most recent call last):\n" + " File \"/opt/vllm/vllm/engine/engine.py\", line 175, in start\n" + " self._init_kv_cache()\n" + " File \"/opt/vllm/vllm/worker/worker.py\", line 215, in _init_kv_cache\n" + " raise torch.cuda.OutOfMemoryError(msg)\n" + "torch.cuda.OutOfMemoryError: CUDA out of memory\n" + ), +} + +EVENTS_BY_NAMESPACE = { + "ml-production": [ + { + "type": "Warning", + "reason": "BackOff", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Back-off restarting failed container kserve-container in pod " + "text-gen-legacy-predictor-00001-abc12", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Warning", + "reason": "OOMKilled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Container kserve-container was OOMKilled (exit code 137). " + "GPU memory exhausted during KV cache allocation.", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Normal", + "reason": "Scheduled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Successfully assigned ml-production/" + "text-gen-legacy-predictor-00001-abc12 to gpu-worker-1", + "count": 1, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-02-28T08:00:00Z", + }, + { + "type": "Warning", + "reason": "NIMAccountNotReady", + "object": "InferenceService/nim-llama-prod", + "message": "NIM Account 'nim-account' in namespace 'ml-production' " + "is not ready", + "count": 12, + "first_timestamp": "2026-03-14T12:00:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + { + "type": "Warning", + "reason": "ImagePullBackOff", + "object": "InferenceService/nim-llama-prod", + "message": "Failed to pull image 'nvcr.io/nim/meta/llama-3.1-8b-instruct:" + "latest': unauthorized: authentication required", + "count": 8, + "first_timestamp": "2026-03-14T12:05:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + ], +} + + +# ── Resource tools ─────────────────────────────────────────────────────── + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: str = "", +) -> str: + """Get a single Kubernetes resource by apiVersion, kind, and name.""" + if kind == "Node": + for node in ALL_NODES: + if node["metadata"]["name"] == name: + return json.dumps(node, indent=2) + raise ValueError(f"Node '{name}' not found") + + if kind == "ServingRuntime": + if name == "vllm-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_VLLM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + if name == "nim-serving-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_NIM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + raise ValueError(f"ServingRuntime '{name}' not found in namespace '{namespace}'") + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps(lr, indent=2) + + if kind == "Account" and "nim" in apiVersion.lower(): + if namespace == "ml-production" and name == "nim-account": + return json.dumps(NIM_ACCOUNT_CR, indent=2) + raise ValueError( + f"Account '{name}' not found in namespace '{namespace}'" + ) + + if kind == "ClusterVersion" and apiVersion == "config.openshift.io/v1": + return json.dumps({ + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterVersion", + "metadata": {"name": "version"}, + "status": {"desired": {"version": "4.16.3"}}, + }) + + raise ValueError(f"Resource {apiVersion}/{kind}/{name} not found") + + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: str = "", + labelSelector: str = "", +) -> str: + """List Kubernetes resources by apiVersion and kind.""" + if kind == "Node": + nodes = ALL_NODES + if labelSelector: + parts = labelSelector.split("=", 1) + key = parts[0] + value = parts[1] if len(parts) > 1 else "" + nodes = [ + n for n in nodes + if n["metadata"]["labels"].get(key) == value + ] + return json.dumps(nodes, indent=2) + + if kind == "Service" and apiVersion == "serving.knative.dev/v1": + return json.dumps({ + "kind": "ServiceList", + "apiVersion": "serving.knative.dev/v1", + "items": [], + "metadata": {}, + }) + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps({ + "kind": "LimitRangeList", + "items": [lr], + }) + + if kind == "InferenceService": + return json.dumps({ + "kind": "InferenceServiceList", + "items": [], + }) + + raise ValueError(f"Unsupported list: {apiVersion}/{kind}") + + +@mcp.tool() +def pods_list( + namespace: str, + labelSelector: str = "", +) -> str: + """List pods in a namespace with optional label selector.""" + pods = PODS_BY_NAMESPACE.get(namespace, []) + + if labelSelector: + key, _, value = labelSelector.partition("=") + pods = [p for p in pods if p.get("labels", {}).get(key) == value] + + results = [] + for pod in pods: + results.append({ + "name": pod["name"], + "namespace": pod["namespace"], + "status": pod["status"], + "restarts": pod.get("restarts", 0), + "node": pod.get("node", ""), + "containers": pod.get("containers", []), + "gpu": pod.get("gpu", "0"), + }) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def pods_log( + namespace: str, + name: str, + container: str = "", +) -> str: + """Get logs from a pod container.""" + logs = POD_LOGS.get(name) + if logs is None: + raise ValueError(f"Pod '{name}' not found in namespace '{namespace}'") + return logs + + +@mcp.tool() +def events_list(namespace: str) -> str: + """List events in a namespace.""" + events = EVENTS_BY_NAMESPACE.get(namespace, []) + return json.dumps(events, indent=2) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/mcp-servers/mock-rhoai-mcp.py b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/mcp-servers/mock-rhoai-mcp.py new file mode 100644 index 00000000..9b072b37 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/mcp-servers/mock-rhoai-mcp.py @@ -0,0 +1,796 @@ +#!/usr/bin/env python3 +"""Mock RHOAI MCP server for SkillsBench rh-ai-engineer task. + +Simulates Red Hat OpenShift AI operations: Data Science Projects, +model serving, data connections, serving runtimes, inference services. + +Scenario: +- ml-production: existing project with two broken deployments + - text-gen-legacy: vLLM OOMKilled (max-model-len=32768 on A10G) + - nim-llama-prod: NIM failing (Account CR not ready, NGC creds invalid) +- fraud-detection: does not exist yet (agent creates it) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("rhoai") + +# ── In-memory state ────────────────────────────────────────────────────── + +PROJECTS = { + "ml-production": { + "name": "ml-production", + "display_name": "ML Production", + "description": "Production ML workloads", + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": "single", + "pipeline_server": True, + }, +} + +DATA_CONNECTIONS = { + "ml-production": [ + { + "name": "prod-model-store", + "type": "S3", + "bucket": "ml-models-prod", + "endpoint": "https://s3.us-east-1.amazonaws.com", + "region": "us-east-1", + }, + ], +} + +SERVING_RUNTIMES = { + "__platform_templates__": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "REST", + "supported_model_formats": [ + {"name": "vLLM", "version": "1", "autoSelect": True} + ], + }, + { + "name": "caikit-tgis-runtime", + "display_name": "Caikit+TGIS ServingRuntime", + "model_formats": ["caikit"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "gRPC", + }, + ], + "ml-production": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + { + "name": "nim-serving-runtime", + "display_name": "NVIDIA NIM ServingRuntime", + "model_formats": ["NIM"], + "requires_instantiation": False, + "source": "nim-account", + "api_protocol": "REST", + }, + { + "name": "ovms-1", + "display_name": "OpenVINO Model Server", + "model_formats": ["openvino_ir", "onnx"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + ], +} + +INFERENCE_SERVICES = { + "ml-production": { + "text-gen-legacy": { + "name": "text-gen-legacy", + "namespace": "ml-production", + "runtime": "vllm-runtime", + "model_format": "vLLM", + "storage_uri": "hf://mistralai/Mistral-7B-Instruct-v0.3", + "display_name": "Mistral 7B Legacy", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "16Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "PredictorFailed", + "message": "Predictor pod is not ready", + }, + { + "type": "PredictorReady", + "status": "False", + "reason": "ContainerCrashLoop", + "message": "Container kserve-container terminated: " + "OOMKilled (exit code 137). 5 restarts.", + }, + { + "type": "IngressReady", + "status": "True", + "reason": "IngressReady", + "message": "Ingress is ready", + }, + ], + "age": "3d", + }, + "nim-llama-prod": { + "name": "nim-llama-prod", + "namespace": "ml-production", + "runtime": "nim-serving-runtime", + "model_format": "NIM", + "storage_uri": "nim://meta/llama-3.1-8b-instruct", + "display_name": "Llama 3.1 8B (NIM)", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "32Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "RuntimeNotReady", + "message": "ServingRuntime 'nim-serving-runtime' " + "is not in ready state", + }, + { + "type": "PredictorReady", + "status": "Unknown", + "reason": "PodNotCreated", + "message": "Predictor pod has not been created. " + "Waiting for ServingRuntime to become ready.", + }, + { + "type": "IngressReady", + "status": "Unknown", + "reason": "PredictorNotReady", + "message": "Waiting for predictor to become ready", + }, + ], + "age": "1d", + }, + }, +} + +DEPLOYED_MODELS = {} + +WORKBENCHES = { + "ml-production": [ + { + "name": "data-exploration-nb", + "display_name": "Data Exploration", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Running", + "cpu_request": "1", + "memory_request": "8Gi", + "gpu_count": 0, + "pvc_name": "data-exploration-nb-pvc", + "pvc_size": "20Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-10T09:00:00Z", + }, + { + "name": "model-training-nb", + "display_name": "Model Training", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Stopped", + "cpu_request": "4", + "memory_request": "16Gi", + "gpu_count": 1, + "pvc_name": "model-training-nb-pvc", + "pvc_size": "50Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-15T14:00:00Z", + }, + ], +} + +PIPELINE_SERVERS = { + "ml-production": { + "configured": True, + "data_connection": "prod-model-store", + "status": "Ready", + "database": "MariaDB", + }, +} + +NOTEBOOK_IMAGES = [ + {"name": "jupyter-pytorch-ubi9-python-3.9-2024.1", "display_name": "PyTorch 2024.1", "packages": ["torch", "transformers"]}, + {"name": "jupyter-tensorflow-ubi9-python-3.9-2024.1", "display_name": "TensorFlow 2024.1", "packages": ["tensorflow"]}, + {"name": "jupyter-datascience-ubi9-python-3.9-2024.1", "display_name": "Standard Data Science", "packages": ["pandas", "scikit-learn"]}, + {"name": "jupyter-minimal-ubi9-python-3.9-2024.1", "display_name": "Minimal Python", "packages": []}, +] + + +# ── Project tools ──────────────────────────────────────────────────────── + + +@mcp.tool() +def list_data_science_projects() -> str: + """List all RHOAI Data Science Projects on the cluster.""" + projects = [] + for name, proj in PROJECTS.items(): + projects.append({ + "name": name, + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + }) + return json.dumps(projects, indent=2) + + +@mcp.tool() +def create_data_science_project( + name: str, + display_name: str, + description: str = "", +) -> str: + """Create a new RHOAI Data Science Project (namespace with dashboard labels).""" + if name in PROJECTS: + raise ValueError( + f"Project '{name}' already exists. Choose a different name " + "or configure the existing project." + ) + if not name.replace("-", "").replace("_", "").isalnum() or len(name) > 63: + raise ValueError( + f"Invalid project name '{name}'. Must be DNS-compatible: " + "lowercase alphanumeric and hyphens, max 63 chars." + ) + + PROJECTS[name] = { + "name": name, + "display_name": display_name, + "description": description, + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": None, + "pipeline_server": False, + } + DATA_CONNECTIONS[name] = [] + SERVING_RUNTIMES[name] = [] + INFERENCE_SERVICES[name] = {} + + return json.dumps({ + "status": "created", + "name": name, + "display_name": display_name, + "namespace": name, + "labels": {"opendatahub.io/dashboard": "true"}, + }) + + +@mcp.tool() +def get_project_details(name: str) -> str: + """Get detailed information about an RHOAI Data Science Project.""" + if name not in PROJECTS: + raise ValueError(f"Project '{name}' not found") + proj = PROJECTS[name] + dc_count = len(DATA_CONNECTIONS.get(name, [])) + isvc_count = len(INFERENCE_SERVICES.get(name, {})) + return json.dumps({ + "name": proj["name"], + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "labels": proj["labels"], + "data_connections": dc_count, + "inference_services": isvc_count, + "model_serving_mode": proj.get("model_serving_mode"), + "pipeline_server": proj.get("pipeline_server", False), + }) + + +@mcp.tool() +def get_project_status(namespace: str) -> str: + """Get comprehensive status of an RHOAI Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Project '{namespace}' not found") + proj = PROJECTS[namespace] + dcs = DATA_CONNECTIONS.get(namespace, []) + isvcs = INFERENCE_SERVICES.get(namespace, {}) + return json.dumps({ + "namespace": namespace, + "display_name": proj["display_name"], + "status": "Active", + "components": { + "data_connections": len(dcs), + "inference_services": len(isvcs), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + "pipeline_server": "configured" if proj.get("pipeline_server") else "not configured", + }, + }) + + +# ── Data connection tools ──────────────────────────────────────────────── + + +@mcp.tool() +def create_s3_data_connection( + namespace: str, + name: str, + bucket: str, + endpoint: str, + access_key: str, + secret_key: str, + region: str = "", +) -> str: + """Create an S3-compatible data connection in an RHOAI project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + existing = DATA_CONNECTIONS.get(namespace, []) + if any(dc["name"] == name for dc in existing): + raise ValueError( + f"Data connection '{name}' already exists in namespace '{namespace}'" + ) + + dc = { + "name": name, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + "region": region, + } + DATA_CONNECTIONS.setdefault(namespace, []).append(dc) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + }) + + +@mcp.tool() +def list_data_connections(namespace: str) -> str: + """List data connections in an RHOAI project namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + return json.dumps(dcs, indent=2) + + +# ── Model serving tools ───────────────────────────────────────────────── + + +@mcp.tool() +def set_model_serving_mode(namespace: str, mode: str) -> str: + """Enable model serving on a Data Science Project (single or multi mode).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + if mode not in ("single", "multi"): + raise ValueError(f"Invalid mode '{mode}'. Must be 'single' or 'multi'.") + + PROJECTS[namespace]["model_serving_mode"] = mode + + if not SERVING_RUNTIMES.get(namespace): + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + SERVING_RUNTIMES[namespace] = [ + {**t, "requires_instantiation": False, "source": "existing"} + for t in templates + ] + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "mode": mode, + }) + + +@mcp.tool() +def list_serving_runtimes( + namespace: str, + include_templates: bool = False, +) -> str: + """List available ServingRuntimes in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + runtimes = list(SERVING_RUNTIMES.get(namespace, [])) + if include_templates: + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + existing_names = {r["name"] for r in runtimes} + for t in templates: + if t["name"] not in existing_names: + runtimes.append(t) + + return json.dumps(runtimes, indent=2) + + +# ── Inference service tools ────────────────────────────────────────────── + + +@mcp.tool() +def deploy_model( + name: str, + namespace: str, + runtime: str, + model_format: str, + storage_uri: str, + display_name: str = "", + min_replicas: int = 1, + max_replicas: int = 1, + cpu_request: str = "1", + cpu_limit: str = "2", + memory_request: str = "4Gi", + memory_limit: str = "8Gi", + gpu_count: int = 0, +) -> str: + """Deploy an AI/ML model as a KServe InferenceService.""" + if namespace not in PROJECTS: + raise ValueError( + f"Namespace '{namespace}' is not a Data Science Project. " + "Create one via create_data_science_project first." + ) + + ns_runtimes = SERVING_RUNTIMES.get(namespace, []) + runtime_names = [r["name"] for r in ns_runtimes] + if runtime not in runtime_names: + available = ", ".join(runtime_names) or "none" + raise ValueError( + f"ServingRuntime '{runtime}' not found in namespace '{namespace}'. " + f"Available runtimes: {available}" + ) + + endpoint = f"https://{name}-{namespace}.apps.ocp-cluster.example.com" + isvc = { + "name": name, + "namespace": namespace, + "runtime": runtime, + "model_format": model_format, + "storage_uri": storage_uri, + "display_name": display_name or name, + "gpu_count": gpu_count, + "cpu_request": cpu_request, + "memory_request": memory_request, + "min_replicas": min_replicas, + "max_replicas": max_replicas, + "ready": True, + "url": endpoint, + "conditions": [ + {"type": "Ready", "status": "True", "reason": "Ready", "message": ""}, + {"type": "PredictorReady", "status": "True", "reason": "PodReady", "message": ""}, + {"type": "IngressReady", "status": "True", "reason": "IngressReady", "message": ""}, + ], + "age": "0s", + } + + INFERENCE_SERVICES.setdefault(namespace, {})[name] = isvc + DEPLOYED_MODELS[f"{namespace}/{name}"] = isvc + + return json.dumps({ + "status": "deployed", + "name": name, + "namespace": namespace, + "runtime": runtime, + "endpoint": endpoint, + "ready": True, + }) + + +@mcp.tool() +def list_inference_services( + namespace: str, + verbosity: str = "standard", +) -> str: + """List deployed InferenceServices in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + isvcs = INFERENCE_SERVICES.get(namespace, {}) + results = [] + for isvc_name, isvc in isvcs.items(): + entry = { + "name": isvc["name"], + "runtime": isvc["runtime"], + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "age": isvc.get("age", ""), + } + if verbosity == "full": + entry["conditions"] = isvc.get("conditions", []) + entry["storage_uri"] = isvc.get("storage_uri", "") + entry["gpu_count"] = isvc.get("gpu_count", 0) + results.append(entry) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def get_inference_service( + name: str, + namespace: str, + verbosity: str = "standard", +) -> str: + """Get detailed status of a specific InferenceService.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + + isvc = isvcs[name] + result = { + "name": isvc["name"], + "namespace": isvc["namespace"], + "runtime": isvc["runtime"], + "model_format": isvc.get("model_format", ""), + "storage_uri": isvc.get("storage_uri", ""), + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "conditions": isvc.get("conditions", []), + "gpu_count": isvc.get("gpu_count", 0), + "replicas": {"min": isvc.get("min_replicas", 1), "max": isvc.get("max_replicas", 1)}, + "resources": { + "cpu_request": isvc.get("cpu_request", "1"), + "memory_request": isvc.get("memory_request", "4Gi"), + "memory_limit": isvc.get("memory_limit", "8Gi"), + }, + "age": isvc.get("age", ""), + } + return json.dumps(result, indent=2) + + +@mcp.tool() +def get_model_endpoint(name: str, namespace: str) -> str: + """Get the inference endpoint URL for a deployed model.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + isvc = isvcs[name] + if not isvc["ready"]: + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": "", + "error": "InferenceService is not ready. Check conditions for details.", + }) + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": isvc["url"], + }) + + +# ── Workbench tools ────────────────────────────────────────────────────── + + +@mcp.tool() +def list_workbenches(namespace: str) -> str: + """List workbenches (Jupyter notebooks) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + return json.dumps(wbs, indent=2) + + +@mcp.tool() +def create_workbench( + namespace: str, + name: str, + display_name: str = "", + image: str = "jupyter-datascience-ubi9-python-3.9-2024.1", + cpu_request: str = "1", + memory_request: str = "4Gi", + gpu_count: int = 0, + pvc_size: str = "20Gi", +) -> str: + """Create a new workbench (Jupyter notebook) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + valid_images = [img["name"] for img in NOTEBOOK_IMAGES] + if image not in valid_images: + raise ValueError( + f"Image '{image}' not found. Available: {', '.join(valid_images)}" + ) + + wb = { + "name": name, + "display_name": display_name or name, + "image": image, + "status": "Running", + "cpu_request": cpu_request, + "memory_request": memory_request, + "gpu_count": gpu_count, + "pvc_name": f"{name}-pvc", + "pvc_size": pvc_size, + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-03-02T12:00:00Z", + } + WORKBENCHES.setdefault(namespace, []).append(wb) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "image": image, + "pvc": f"{name}-pvc", + }) + + +@mcp.tool() +def stop_workbench(namespace: str, name: str) -> str: + """Stop a running workbench (preserves data).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Stopped" + return json.dumps({"status": "stopped", "name": name, "namespace": namespace}) + + +@mcp.tool() +def start_workbench(namespace: str, name: str) -> str: + """Start a stopped workbench.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Running" + return json.dumps({"status": "running", "name": name, "namespace": namespace}) + + +@mcp.tool() +def delete_workbench(namespace: str, name: str) -> str: + """Delete a workbench. WARNING: PVC data may be lost if not backed up.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wbs.remove(wb) + return json.dumps({ + "status": "deleted", + "name": name, + "namespace": namespace, + "warning": "Associated PVC data has been deleted", + }) + + +@mcp.tool() +def list_notebook_images() -> str: + """List available notebook images for workbench creation.""" + return json.dumps(NOTEBOOK_IMAGES, indent=2) + + +# ── Pipeline server tools ─────────────────────────────────────────────── + + +@mcp.tool() +def configure_pipeline_server( + namespace: str, + data_connection: str, + database: str = "MariaDB", +) -> str: + """Configure a pipeline server for a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + if not any(dc["name"] == data_connection for dc in dcs): + available = [dc["name"] for dc in dcs] + raise ValueError( + f"Data connection '{data_connection}' not found. Available: {available}" + ) + + PIPELINE_SERVERS[namespace] = { + "configured": True, + "data_connection": data_connection, + "status": "Ready", + "database": database, + } + PROJECTS[namespace]["pipeline_server"] = True + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "data_connection": data_connection, + "database": database, + }) + + +@mcp.tool() +def get_pipeline_server_status(namespace: str) -> str: + """Get the status of the pipeline server in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + ps = PIPELINE_SERVERS.get(namespace) + if not ps: + return json.dumps({"namespace": namespace, "configured": False}) + return json.dumps({ + "namespace": namespace, + "configured": ps["configured"], + "data_connection": ps["data_connection"], + "status": ps["status"], + "database": ps["database"], + }) + + +@mcp.tool() +def setup_pipeline_server( + namespace: str, + data_connection: str, + database: str = "MariaDB", +) -> str: + """Alias for configure_pipeline_server. Configure a pipeline server for a Data Science Project.""" + return configure_pipeline_server(namespace, data_connection, database) + + +@mcp.tool() +def get_pipeline_status(namespace: str) -> str: + """Alias for get_pipeline_server_status. Get the status of the pipeline server.""" + return get_pipeline_server_status(namespace) + + +# ── Serving runtime creation ──────────────────────────────────────────── + + +@mcp.tool() +def create_serving_runtime( + namespace: str, + name: str, + display_name: str = "", + model_formats: list = None, + container_image: str = "", + container_port: int = 8080, + multi_model: bool = False, + api_protocol: str = "REST", +) -> str: + """Create a custom ServingRuntime in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + if not model_formats: + raise ValueError("model_formats must specify at least one model format") + + runtime = { + "name": name, + "display_name": display_name or name, + "model_formats": model_formats, + "requires_instantiation": False, + "source": "custom", + "api_protocol": api_protocol, + "container_image": container_image, + "container_port": container_port, + "multi_model": multi_model, + } + SERVING_RUNTIMES.setdefault(namespace, []).append(runtime) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "model_formats": model_formats, + }) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/ds-project-setup/SKILL.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/ds-project-setup/SKILL.md new file mode 100644 index 00000000..af660b2b --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/ds-project-setup/SKILL.md @@ -0,0 +1,336 @@ +--- +name: ds-project-setup +description: | + Create and configure Data Science Projects on OpenShift AI with namespace setup, S3 data connections, pipeline server, and model serving enablement. + + Use when: + - "Create a data science project" + - "Set up a new namespace for ML work" + - "Add an S3 data connection to my project" + - "Configure the pipeline server" + - "Enable model serving on my project" + + Bootstraps an RHOAI Data Science Project with proper labels, data connections, pipeline infrastructure, and model serving configuration. + + NOT for deploying models (use /model-deploy). + NOT for creating workbenches (use /workbench-manage). + NOT for managing pipelines after setup (use /pipeline-manage). +color: green +model: inherit +metadata: + author: "Red Hat Ecosystem Engineering" + version: "1.0" +--- + +# /ds-project-setup Skill + +Bootstrap a Red Hat OpenShift AI Data Science Project from scratch. Creates a namespace with RHOAI dashboard labels, configures S3-compatible data connections, sets up the pipeline server with external storage, and enables model serving on the project. + +## Prerequisites + +**Required MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) + +**Required MCP Tools** (from rhoai): +- `list_data_science_projects` - List existing RHOAI projects to check for duplicates +- `create_data_science_project` - Create namespace with RHOAI labels and dashboard integration +- `get_project_details` - Verify project creation and inspect configuration +- `get_project_status` - Get comprehensive project status including components +- `create_s3_data_connection` - Create S3-compatible data connection secret +- `list_data_connections` - List existing data connections in the project +- `get_pipeline_server` - Check pipeline server configuration +- `create_pipeline_server` - Configure pipeline server with S3 data connection +- `set_model_serving_mode` - Enable single-model or multi-model serving + +**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools** (from openshift): +- `resources_get` (from openshift) - Inspect namespace labels, LimitRange, ResourceQuota + +**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). + +**Additional cluster requirements**: +- Cluster admin or namespace creation privileges for the user + +## When to Use This Skill + +**Use this skill when you need to:** +- Create a new Data Science Project namespace for an ML team +- Add S3 data connections to an existing project +- Configure the pipeline server on a project +- Enable or change the model serving mode (single vs multi-model) +- Bootstrap a complete project environment before deploying models or workbenches + +**Do NOT use this skill when:** +- You want to deploy a model (use `/model-deploy`) +- You need to create a notebook workbench (use `/workbench-manage`) +- You need to manage pipeline runs (use `/pipeline-manage`) +- You need to configure a custom serving runtime (use `/serving-runtime-config`) + +## Workflow + +### Step 1: Gather Requirements + +**Ask the user for:** +- **Project name**: DNS-compatible name for the namespace (lowercase, no spaces, max 63 chars) +- **Display name**: Human-readable project name for the RHOAI dashboard +- **Description**: Optional project description +- **Data connections**: Whether to configure S3 data connections (yes/no) +- **Pipeline server**: Whether to configure the pipeline server (yes/no, requires data connection) +- **Model serving mode**: "single" (default, one model per endpoint) or "multi" (multiple models per endpoint) + +**Present configuration table:** + +| Setting | Value | +|---------|-------| +| Project name | [name] | +| Display name | [display_name] | +| Description | [description] | +| Data connections | [yes/no] | +| Pipeline server | [yes/no] | +| Model serving mode | [single/multi] | + +Proceed automatically without waiting for user input. + +### Step 2: Check Existing Projects + +**MCP Tool**: `list_data_science_projects` (from rhoai) + +**Parameters**: none + +Check if the project name already exists in the cluster. + +- If project **exists**: Report to user and offer options: "Project `[name]` already exists. Would you like to: (a) configure additional components on it, or (b) choose a different name?" +- If project **does not exist**: Proceed to Step 3 + +Proceed automatically without waiting for user input. + +### Step 3: Create Data Science Project + +**MCP Tool**: `create_data_science_project` (from rhoai) + +**Parameters**: +- `name`: project name from Step 1 - REQUIRED (DNS-compatible: lowercase alphanumeric and hyphens, max 63 chars) +- `display_name`: human-readable display name - REQUIRED +- `description`: project description - OPTIONAL + +**Verify creation:** + +**MCP Tool**: `get_project_details` (from rhoai) + +**Parameters**: +- `name`: the created project name - REQUIRED + +Confirm the project was created with proper RHOAI labels (`opendatahub.io/dashboard: "true"`). + +**Error Handling**: +- If name already taken -> Offer alternative name or configure existing project +- If RBAC error -> Report: "Insufficient permissions to create namespaces. Contact your cluster administrator." +- If name invalid -> Report DNS naming constraints and suggest a valid name + +**Output to user**: "Data Science Project `[name]` created successfully." + +### Step 4: Configure Data Connections (Optional) + +Skip this step if user declined data connections in Step 1. + +**Ask the user for S3 connection details:** +- **Connection name**: Identifier for this data connection +- **S3 bucket**: Target bucket name +- **S3 endpoint**: S3-compatible endpoint URL (e.g., `https://s3.amazonaws.com`, MinIO endpoint) +- **Access key**: AWS access key ID or S3-compatible access key +- **Secret key**: AWS secret access key or S3-compatible secret key +- **Region**: AWS region or empty for non-AWS S3 + +**Display connection configuration** (credentials REDACTED): + +| Setting | Value | +|---------|-------| +| Connection name | [name] | +| Bucket | [bucket] | +| Endpoint | [endpoint] | +| Access key | [first-4-chars]**** | +| Secret key | ******** | +| Region | [region] | + +Proceed automatically without waiting for user input. + +**MCP Tool**: `create_s3_data_connection` (from rhoai) + +**Parameters**: +- `namespace`: project name from Step 3 - REQUIRED +- `name`: connection name - REQUIRED +- `bucket`: S3 bucket name - REQUIRED +- `endpoint`: S3 endpoint URL - REQUIRED +- `access_key`: access key ID - REQUIRED +- `secret_key`: secret access key - REQUIRED +- `region`: AWS region - OPTIONAL (omit for non-AWS S3) + +**Verify creation:** + +**MCP Tool**: `list_data_connections` (from rhoai) + +**Parameters**: +- `namespace`: project name - REQUIRED + +Confirm the data connection appears in the list. + +**Error Handling**: +- If connection name already exists -> Ask: "Data connection `[name]` already exists. Create with a different name?" +- If RBAC error -> Report insufficient permissions to create Secrets in namespace + +**Output to user**: "Data connection `[name]` created in project `[namespace]`." + +**Repeat this step** if user wants to create multiple data connections. + +### Step 5: Configure Pipeline Server (Optional) + +Skip this step if user declined pipeline server in Step 1. + +**Prerequisite check**: A data connection must exist in the project (from Step 4 or pre-existing). If no data connections exist, inform user: "Pipeline server requires an S3 data connection for artifact storage. Would you like to create one now?" and return to Step 4. + +**MCP Tool**: `get_pipeline_server` (from rhoai) + +**Parameters**: +- `namespace`: project name - REQUIRED + +If pipeline server already exists, report its status and ask if user wants to reconfigure. + +**Display pipeline server configuration:** + +| Setting | Value | +|---------|-------| +| Namespace | [namespace] | +| Data connection | [data_connection_name] | + +Proceed automatically without waiting for user input. + +**MCP Tool**: `create_pipeline_server` (from rhoai) + +**Parameters**: +- `namespace`: project name - REQUIRED +- `data_connection`: name of the S3 data connection to use for pipeline artifacts - REQUIRED + +**Verify creation:** + +**MCP Tool**: `get_pipeline_server` (from rhoai) + +**Parameters**: +- `namespace`: project name - REQUIRED + +Confirm the pipeline server is configured and initializing. + +**Error Handling**: +- If data connection not found -> Report: "Data connection `[name]` not found in namespace. Create it first." +- If pipeline server already exists -> Ask user whether to reconfigure or keep existing +- If RBAC error -> Report insufficient permissions + +**Output to user**: "Pipeline server configured in project `[namespace]` using data connection `[data_connection]`." + +### Step 6: Enable Model Serving and Report + +**MCP Tool**: `set_model_serving_mode` (from rhoai) + +**Parameters**: +- `namespace`: project name - REQUIRED +- `mode`: "single" or "multi" - REQUIRED (default: "single") + +**Final validation:** + +**MCP Tool**: `get_project_status` (from rhoai) + +**Parameters**: +- `namespace`: project name - REQUIRED + +**Report project summary:** + +| Component | Status | +|-----------|--------| +| Project | [name] (created / existing) | +| Data connections | [count] configured | +| Pipeline server | [configured / not configured] | +| Model serving | [single / multi] mode enabled | + +**Suggest next steps:** +- `/workbench-manage` - Create a notebook workbench in this project +- `/model-deploy` - Deploy a model to this project +- `/pipeline-manage` - Create and run data science pipelines + +## Common Issues + +### Issue 1: Project Name Already Exists + +**Error**: `create_data_science_project` returns conflict error + +**Cause**: A namespace with the same name already exists in the cluster, either as an RHOAI project or a regular OpenShift project. + +**Solution:** +1. Use `list_data_science_projects` to check if it is an existing RHOAI project +2. If it is an RHOAI project, offer to configure additional components on it +3. If it is a regular namespace (not an RHOAI project), suggest a different name or advise converting it by adding the `opendatahub.io/dashboard: "true"` label + +### Issue 2: S3 Endpoint Unreachable + +**Error**: Data connection created but pipeline server or model serving cannot access storage + +**Cause**: The S3 endpoint URL is malformed, unreachable from the cluster, or requires TLS configuration. + +**Solution:** +1. Verify the endpoint URL format includes the protocol (`https://`) +2. For MinIO: use the internal cluster service URL (e.g., `http://minio.minio-ns.svc:9000`) +3. For AWS: use the regional endpoint (e.g., `https://s3.us-east-1.amazonaws.com`) +4. Check if the cluster has network egress restrictions that block external S3 access + +### Issue 3: Pipeline Server Fails to Initialize + +**Error**: Pipeline server status remains unhealthy or pods crash + +**Cause**: Usually caused by an invalid data connection (wrong credentials or unreachable bucket), or insufficient cluster resources. + +**Solution:** +1. Verify the data connection credentials are correct (re-create if needed) +2. Check that the S3 bucket exists and is accessible with the provided credentials +3. Check namespace ResourceQuota for pod limits +4. Review pipeline server pod logs via `pods_log` (from openshift) for specific error messages + +### Issue 4: Namespace Quota Exceeded + +**Error**: Resource creation fails with quota exceeded error + +**Cause**: The cluster has ResourceQuota or LimitRange policies that restrict resource creation in the namespace. + +**Solution:** +1. Use `resources_get` (from openshift) to inspect ResourceQuota in the namespace +2. Report the quota limits to the user +3. Suggest contacting the cluster administrator to increase quotas or clean up unused resources + +## Dependencies + +### MCP Tools +See [Prerequisites](#prerequisites) for the complete list of required and optional MCP tools. + +### Related Skills +- `/workbench-manage` - Create notebook workbenches in the project +- `/model-deploy` - Deploy models to the project +- `/pipeline-manage` - Create and manage pipeline runs +- `/serving-runtime-config` - Configure custom serving runtimes in the project + +### Reference Documentation +- [skill-conventions.md](../references/skill-conventions.md) - Shared prerequisite, HITL, and security conventions + +## Example Usage + +**User**: "Create a data science project called fraud-detection with an S3 connection and pipeline server" + +**Skill response**: Gathers requirements, presents configuration table, creates project `fraud-detection`, configures S3 data connection (credentials redacted in display), sets up pipeline server, enables single-model serving, and reports final project status with next steps. + +## Critical: Human-in-the-Loop Requirements + +See [skill-conventions.md](../references/skill-conventions.md) for general HITL and security conventions. + +**Skill-specific checkpoints:** +- After gathering requirements (Step 1): confirm project configuration table +- Before creating data connections (Step 4): display connection config with credentials REDACTED, confirm +- Before configuring pipeline server (Step 5): confirm data connection selection +- If project already exists (Step 2): confirm whether to configure existing or choose new name +- **NEVER** create data connections without user confirming credential details +- **NEVER** display actual S3 access keys or secret keys in output diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/common-issues.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/instruction.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/instruction.md new file mode 100644 index 00000000..03edef95 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/instruction.md @@ -0,0 +1,20 @@ +# Data Science Project Setup Task + +You are an AI engineer on Red Hat OpenShift AI. Your team needs a new project to host a fraud detection model, with access to an S3 model store and model serving enabled. + +## Requirements +- Create a new data science project named `fraud-detection` with a descriptive display name and appropriate labels +- Configure an S3-compatible data connection for model artifact storage using these details: + - Connection name: `model-store` + - Bucket: `fraud-models` + - Endpoint: `https://s3.us-east-1.amazonaws.com` + - Access key: `AKIA1234EXAMPLE` + - Secret key: `wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY` + - Region: `us-east-1` +- Enable model serving on the project and confirm which serving runtimes become available +- Configure the pipeline server using the data connection you created +- Verify the project is fully configured and ready for model deployment + +Document your setup steps, the final project configuration, and any issues encountered in `/root/report.md`. Redact any credentials in the report. + +Use MCP tools to interact with the platform. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/solution/solve.sh b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/solution/solve.sh new file mode 100644 index 00000000..91e7d226 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/solution/solve.sh @@ -0,0 +1,32 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Data Science Project Setup + +## Project: fraud-detection + +### Namespace Configuration +- Label: `opendatahub.io/dashboard: "true"` (required for RHOAI dashboard visibility) +- Model serving mode: `single` (one model per namespace) + +### S3 Data Connection +- Name: model-artifacts +- Bucket: fraud-detection-models +- Endpoint: https://s3.amazonaws.com +- Access key / Secret key configured (credentials REDACTED in display) +- Region: us-east-1 + +### Pipeline Server +- Data connection: model-artifacts (required for pipeline artifact storage) +- Pipeline server uses data connection for artifacts + +### Project Status (get_project_status) +| Component | Status | +|-----------|--------| +| Project | fraud-detection (created) | +| Data connections | 1 configured | +| Pipeline server | configured | +| Model serving | single mode enabled | + +### Validation +- list_data_science_projects: checked for duplicate project name before create +REPORT_EOF diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/task.toml b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/task.toml new file mode 100644 index 00000000..83c47b69 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-ai-engineer__ds-project-setup" +name = "rh-ai-engineer Data Science Project Setup Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-ai-engineer", "ds-project-setup", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/tests/llm_judge.py b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/tests/llm_judge.py new file mode 100644 index 00000000..2d6a34c6 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/tests/llm_judge.py @@ -0,0 +1,102 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "partial_credential_redaction", + "file": "/root/report.md", + "question": "Does the report display S3 credentials using partial redaction — showing the first few characters of the access key (like AKIA****) with the rest masked — rather than using placeholder values or full redaction?", + "reference": "A skilled report shows credentials partially redacted: the first few characters of the access key visible followed by asterisks (e.g., 'AKIA****'), and the secret key fully masked ('********'). Using PLACEHOLDER_ACCESS_KEY or completely hiding the access key is insufficient — partial redaction allows verification without exposing the full credential." + }, + { + "id": "secret_manifest_structure", + "file": "/root/report.md", + "question": "Does the report include a Kubernetes Secret manifest (with kind: Secret, apiVersion, metadata, and data fields) showing how the S3 data connection is stored as a K8s resource, rather than just describing the connection narratively?", + "reference": "A skilled report shows the actual K8s Secret YAML structure with kind: Secret, metadata (namespace, name, labels), and data fields containing base64-encoded values. An unskilled report describes the data connection configuration narratively without showing the underlying K8s resource structure." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/tests/test.sh b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/tests/test_outputs.py b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/tests/test_outputs.py new file mode 100644 index 00000000..8978be1d --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/tests/test_outputs.py @@ -0,0 +1,113 @@ +""" +Tests for rh-ai-engineer__ds-project-setup per-skill evaluation. +Baseline tests: any competent agent should pass. +Skill-dependent tests: based on empirical gaps between skilled and unskilled agent outputs. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ["project", "data science", "namespace"]), ( + "report should mention the project" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 200, "report should have substantial content" + + +class TestSkillDependent: + def test_data_connection_secret_keys(self): + """Skill teaches RHOAI data connections are stored as K8s Secrets with specific + key names: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_S3_BUCKET, + AWS_S3_ENDPOINT. Without skill, agents describe connections abstractly.""" + c = read_report() + aws_keys = ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_S3_BUCKET", + "AWS_S3_ENDPOINT", "AWS_DEFAULT_REGION"] + mentioned = sum(1 for k in aws_keys if k in c) + assert mentioned >= 2, ( + "should reference specific RHOAI data connection secret key names (AWS_*)" + ) + + def test_credential_partial_redaction(self): + """Skill teaches showing first 4 chars + **** for credentials (e.g., AKIA****). + Without skill, agents use PLACEHOLDER values or full redaction.""" + c = read_report() + has_partial = any(t in c for t in [ + "AKIA****", "AKIA*", "wJal****", "wJal*", + "1234****", "1234*", + ]) + has_stars_with_prefix = "****" in c and any(t in c for t in ["AKIA", "akia"]) + assert has_partial or has_stars_with_prefix, ( + "should use partial credential redaction (first chars visible + ****)" + ) + + def test_k8s_secret_yaml_manifest(self): + """Skill teaches showing the K8s Secret manifest structure for data connections. + Without skill, agents describe connections narratively without YAML.""" + c = read_report() + has_secret_kind = "kind: Secret" in c or "kind:Secret" in c + has_secret_ref = "Secret" in c and ("apiVersion" in c or "metadata" in c) + assert has_secret_kind or has_secret_ref, ( + "should include K8s Secret manifest structure for data connection" + ) + + def test_pipeline_server_with_data_connection(self): + """Skill teaches pipeline server requires a data connection (prerequisite chain). + Without skill, agents skip pipeline server or configure it generically.""" + c = read_report().lower() + has_pipeline = any(t in c for t in ["pipeline server", "pipeline"]) + has_linkage = any(t in c for t in [ + "data connection", "model-store", "artifact storage", + "s3 bucket", "data_connection", + ]) + pipeline_configured = "pipeline" in c and "configured" in c and "not configured" not in c + assert has_pipeline and (has_linkage or pipeline_configured), ( + "should configure pipeline server linked to a data connection" + ) + + def test_base64_secret_values(self): + """Skill teaches showing actual base64-encoded secret values in K8s + Secret YAML manifests. Without skill, agents show credentials in + plain text or fully redacted format.""" + c = read_report() + import re + has_base64 = bool(re.search(r'[A-Za-z0-9+/]{12,}={0,2}', c)) + has_opaque = "Opaque" in c + assert has_base64 or has_opaque, ( + "should include base64-encoded values or Opaque secret type in K8s manifest" + ) + + def test_model_serving_mode(self): + """Both agents should configure model serving — easy test.""" + c = read_report().lower() + assert any(t in c for t in [ + "single", "multi", "model serving", "serving mode", + ]), "should configure model serving mode" + + def test_runtime_selection_context(self): + """Docs teach decision context across runtimes: vLLM (PagedAttention), + NIM (TensorRT-LLM, no compilation), Caikit+TGIS (gRPC-only). + Without docs, agents don't provide runtime comparison context.""" + c = read_report().lower() + assert any(t in c for t in [ + "pagedattention", "paged attention", "tensorrt", "grpc", + "caikit", "vllm", "nim", + ]) and any(t in c for t in ["runtime", "serving", "comparison", "select"]), ( + "should compare runtimes with technical characteristics" + ) diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/Dockerfile b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/Dockerfile new file mode 100644 index 00000000..d4978abe --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/Dockerfile @@ -0,0 +1,74 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "rhoai": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-rhoai-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/common-issues.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/examples/model-deploy.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/examples/model-deploy.md new file mode 100644 index 00000000..dcd79e44 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/examples/model-deploy.md @@ -0,0 +1,160 @@ +# /model-deploy Examples + +Example interactions demonstrating the model-deploy skill workflow. + +## Example 1: Deploy Llama 3.1 8B with vLLM + +``` +User: "Deploy Llama 3.1 8B on my cluster" + +Agent: [Invokes /model-deploy skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand runtime capabilities. + +Step 1 of 10: Gathering deployment information. + +| Setting | Value | Source | +|---------|-------|--------| +| Model | Llama 3.1 8B Instruct | user input | +| Runtime | TBD | auto-detect | +| Namespace | ? | needs input | +| Model Source | ? | needs input | +| Deployment Mode | Serverless | default | + +What namespace should I deploy to, and where are the model weights stored? +(e.g., S3 bucket, OCI registry, PVC) + +User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" + +Agent: Step 2 of 10: Determining runtime. + +I consulted supported-runtimes.md to understand runtime capabilities. + +Based on Llama 3.1 8B (open-source HuggingFace model): + +Recommended runtime: vLLM +Reason: Default runtime for open-source LLMs. Broad compatibility, + OpenAI-compatible API, no additional setup required. + +Proceed with vLLM? (yes / no / suggest alternative) + +User: "yes" + +Agent: Step 3 of 10: Looking up hardware profile. + +I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. + +| Requirement | Value | +|-------------|-------| +| GPUs | 1x A10G/L4/A100/H100 | +| VRAM | 16GB | +| Key Args | --max-model-len=4096 --tool-call-parser hermes | + +Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). +Ensure your cluster has at least 1 GPU with 16GB VRAM. + +Step 5 of 10: Skipped (NIM platform check not needed for vLLM). + +Step 6-7 of 10: Generating InferenceService YAML. + +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: llama-3-1-8b + namespace: ai-models + annotations: + serving.kserve.io/deploymentMode: Serverless +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: vllm-runtime + storageUri: s3://my-bucket/llama-3.1-8b + resources: + limits: + nvidia.com/gpu: "1" + requests: + cpu: "4" + memory: "16Gi" + args: + - --max-model-len=4096 + - --tool-call-parser=hermes + +| Setting | Value | +|---------|-------| +| Model | llama-3-1-8b | +| Runtime | vllm-runtime | +| GPUs | 1 | +| Namespace | ai-models | +| Mode | Serverless | + +Proceed with creating this InferenceService? (yes/no/modify) + +User: "yes" + +Agent: Step 8 of 10: Creating InferenceService... + [Creates via resources_create_or_update] + + InferenceService created. Monitoring rollout... + +Step 9 of 10: Monitoring rollout. + +| Stage | Status | +|-------|--------| +| Pod Scheduled | Complete | +| Image Pulled | Complete | +| Container Started | Complete | +| Model Loaded | Complete | +| Ready | Complete | + +Step 10 of 10: Deployment complete! + +Model Deployment Successful + +Model: llama-3-1-8b +Runtime: vLLM +Namespace: ai-models +GPUs: 1 + +Inference Endpoint: + URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com + API: OpenAI-compatible REST + +Quick Test: + curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' + +Next Steps: + Monitor performance: "/ai-observability llama-3-1-8b" +``` + +## Example 2: Deploy a Model with NIM (NIM Not Set Up) + +``` +User: "Deploy Llama 3.1 70B with NIM" + +Agent: [Invokes /model-deploy skill] + ... + Step 2: User requested NIM runtime. + + Step 5: Checking NIM platform... + +NIM platform is not set up in namespace ai-models. + +The NIM runtime requires a one-time platform setup. Would you like to: +1. Run /nim-setup now to configure the NIM platform +2. Switch to vLLM runtime instead (no setup required) +3. Abort deployment + +User: "1" + +Agent: [Invokes /nim-setup skill — walks through NGC credential setup] + ... + NIM Platform Setup Complete. + + Resuming model deployment... + [Continues from Step 6 with NIM runtime] +``` diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/examples/nim-setup.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/examples/nim-setup.md new file mode 100644 index 00000000..ccfc1a5d --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/examples/nim-setup.md @@ -0,0 +1,115 @@ +# /nim-setup Examples + +Example interactions demonstrating the nim-setup skill workflow. + +## Example 1: First-Time NIM Setup + +``` +User: "Set up NIM on my cluster" + +Agent: [Invokes /nim-setup skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand NIM platform requirements. + +Let me verify the prerequisites for NIM setup. + +Step 1 of 7: Checking GPU Operator and NFD Operator... + +GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) +NFD Operator: nfd.v4.19.0 (Succeeded) + +Both operators are installed and healthy. + +Step 2 of 7: I need your NGC credentials. + +To set up NIM, I need your NVIDIA NGC API key. +You can generate one at: https://ngc.nvidia.com/setup/api-key + +Please provide: +1. Your NGC API key +2. The target namespace for NIM resources + +User: "Here's my key: [user provides NGC API key], namespace: ai-models" + +Agent: Step 3 of 7: Creating NGC image pull secret... + +I'll create a docker-registry secret for pulling NIM images from nvcr.io: + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-image-pull-secret + namespace: ai-models + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] + +Should I create this image pull secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + Image pull secret created successfully. + +Step 4 of 7: Creating NGC API key secret... + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-api-key + namespace: ai-models + type: Opaque + stringData: + NGC_API_KEY: "[REDACTED]" + +Should I create this API key secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + API key secret created successfully. + +Step 5 of 7: Creating NIM Account CR... + + apiVersion: nim.opendatahub.io/v1 + kind: Account + metadata: + name: nim-account + namespace: ai-models + spec: + apiKeySecret: + name: ngc-api-key + imagePullSecret: + name: ngc-image-pull-secret + +Should I create this NIM Account CR in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates Account CR via resources_create_or_update] + Account CR created successfully. + +Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) + +User: "no" + +Agent: Step 7 of 7: Validating NIM platform readiness... + +NIM Platform Setup Complete + +Account CR: `nim-account` (namespace: ai-models) + Status: Ready + +NGC Credentials: + Image Pull Secret: ngc-image-pull-secret [created] + API Key Secret: ngc-api-key [created] + +NIM ServingRuntimes Available: + - nim-llama-3-1-8b-instruct + - nim-llama-3-1-70b-instruct + - nim-mistral-7b-instruct + +Next Steps: + To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" +``` diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/references/known-model-profiles.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/references/known-model-profiles.md new file mode 100644 index 00000000..40148981 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/references/known-model-profiles.md @@ -0,0 +1,83 @@ +--- +title: Known Model Hardware Profiles +category: references +tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] +semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] +use_cases: [model-deploy, debug-inference, ai-observability] +last_updated: 2026-03-10 +--- + +# Known Model Hardware Profiles + +Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). + +**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. + +## Model Source Conventions + +Each model lists a recommended `storageUri` with its authentication requirements: +- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** +- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). +- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. + +When the user does not specify a model source, use the `hf://` URI listed in the profile below. + +## Llama 3.x (Meta) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | +| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | +| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | +| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | + +- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) +- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly + +## Granite 3.x (IBM/Red Hat) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | +| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | + +- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) +- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser granite --chat-template` +- Red Hat-supported model family on RHOAI + +## Mixtral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | +| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) +- Mixture-of-Experts architecture: only ~13B/45B parameters active per token + +## Mistral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | +| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) + +## When a Model Is Not Listed + +If the requested model is not in this file, the agent MUST use the live doc lookup protocol: + +1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure +2. Fetch hardware specs from the appropriate source +3. Report findings to the user before proceeding with deployment + +Common cases requiring live lookup: +- Newly released models (after this file's last update) +- Domain-specific fine-tuned models +- Models with custom quantization +- NIM-specific optimized profiles diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/references/supported-runtimes.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/references/supported-runtimes.md new file mode 100644 index 00000000..9d018651 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/references/supported-runtimes.md @@ -0,0 +1,104 @@ +--- +title: Supported Serving Runtimes +category: references +tags: [runtimes, vllm, nim, caikit, tgis, serving] +semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] +use_cases: [model-deploy, serving-runtime-config, nim-setup] +last_updated: 2026-02-26 +--- + +# Supported Serving Runtimes + +This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. + +## Runtime Comparison + +| Runtime | API Type | Model Format | GPU Support | License | Setup Required | +|---------|----------|-------------|-------------|---------|----------------| +| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | +| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | +| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | +| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | + +## vLLM + +**Default runtime for most open-source models.** + +- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) +- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized +- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) +- **Model source**: S3-compatible storage, OCI registry, PVC, URI +- **Key features**: + - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) + - PagedAttention for efficient memory management + - Tensor parallelism for multi-GPU inference + - Tool/function calling support (`--tool-call-parser`) + - Continuous batching for high throughput +- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name + +## NVIDIA NIM + +**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** + +- **API**: OpenAI-compatible REST +- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog +- **GPU support**: NVIDIA only (requires specific GPU models per model profile) +- **Model source**: NGC model catalog (pulled automatically via NGC credentials) +- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) +- **Key features**: + - TensorRT-LLM optimization for lower latency + - Pre-compiled model engines (no compilation on first load) + - Optimized and generic profiles per GPU type + - Automatic model download from NGC +- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. +- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM +- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. +- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state + +## Caikit+TGIS + +**Red Hat's Caikit format with Text Generation Inference Server.** + +- **API**: gRPC (not REST) +- **Model formats**: Caikit format (requires conversion from HuggingFace) +- **GPU support**: NVIDIA +- **Model source**: S3-compatible storage +- **Key features**: + - Red Hat-supported runtime + - gRPC API for streaming inference + - Integrated with RHOAI model serving platform +- **When to choose**: When the model is already in Caikit format or when gRPC API is required. +- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name + +## Custom Runtimes + +**User-provided ServingRuntime CRs for unsupported frameworks.** + +- **API**: Defined by the custom runtime +- **Model formats**: Defined by the custom runtime +- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. +- **How to create**: Use `/serving-runtime-config` skill +- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility + +## Runtime Selection Decision Tree + +``` +Is the user's preferred runtime explicitly stated? +├── Yes → Use that runtime +└── No → Continue + +Is the model available in the NGC NIM catalog? +├── Yes → Suggest NIM (with vLLM as fallback) +│ Note: Requires /nim-setup and NVIDIA GPUs +└── No → Continue + +Is the model in Caikit format? +├── Yes → Caikit+TGIS +└── No → Continue + +Is the model a standard open-source LLM (HuggingFace-compatible)? +├── Yes → vLLM (default) +└── No → Custom runtime via /serving-runtime-config +``` diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..e7a4d11c --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +"""Mock OpenShift MCP server for SkillsBench rh-ai-engineer task. + +Simulates Kubernetes resource CRUD, pod management, logs, and events. + +Key scenario elements: +- LimitRange in namespaces: min CPU=100m, min memory=128Mi + (conflicts with KServe sidecar containers hardcoded at 10m CPU/15Mi memory) +- GPU node with custom taint ai-workload=true:NoSchedule +- NIM Account CR in ml-production: not ready (NGC credentials invalid) +- text-gen-legacy pods: OOMKilled (max-model-len=32768 on A10G) +- nim-llama-prod: no pods created (Account CR not ready) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + +# ── Cluster state ──────────────────────────────────────────────────────── + +GPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "gpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + "nvidia.com/gpu.present": "true", + "nvidia.com/gpu.product": "NVIDIA-A10G", + }, + }, + "spec": { + "taints": [ + { + "key": "ai-workload", + "value": "true", + "effect": "NoSchedule", + }, + ], + }, + "status": { + "allocatable": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "capacity": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "conditions": [ + {"type": "Ready", "status": "True"}, + ], + }, +} + +CPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "cpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + }, + }, + "spec": {"taints": []}, + "status": { + "allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "capacity": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +MASTER_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "master-1", + "labels": { + "node-role.kubernetes.io/master": "", + "node-role.kubernetes.io/control-plane": "", + }, + }, + "spec": { + "taints": [ + {"key": "node-role.kubernetes.io/master", "effect": "NoSchedule"}, + ], + }, + "status": { + "allocatable": {"cpu": "8", "memory": "32Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +ALL_NODES = [GPU_NODE, CPU_NODE, MASTER_NODE] + +# LimitRange applied by cluster policy to all DS project namespaces +NAMESPACE_LIMITRANGE = { + "apiVersion": "v1", + "kind": "LimitRange", + "metadata": { + "name": "default-limits", + }, + "spec": { + "limits": [ + { + "type": "Container", + "default": { + "cpu": "2", + "memory": "4Gi", + }, + "defaultRequest": { + "cpu": "500m", + "memory": "256Mi", + }, + "min": { + "cpu": "100m", + "memory": "128Mi", + }, + "max": { + "cpu": "32", + "memory": "128Gi", + }, + }, + ], + }, +} + +NIM_ACCOUNT_CR = { + "apiVersion": "nim.opendatahub.io/v1", + "kind": "Account", + "metadata": { + "name": "nim-account", + "namespace": "ml-production", + }, + "spec": { + "apiKeySecret": { + "name": "ngc-api-key", + }, + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "NGCCredentialsInvalid", + "message": "NGC API key validation failed: 401 Unauthorized. " + "The API key in secret 'ngc-api-key' is expired or invalid. " + "Re-create the secret with a valid NGC API key from " + "https://ngc.nvidia.com/setup/api-key and restart the " + "Account reconciliation.", + "lastTransitionTime": "2026-03-14T12:00:00Z", + }, + ], + "nimPullSecretStatus": "Failed", + "nimConfigStatus": "Pending", + }, +} + +SERVING_RUNTIME_VLLM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "vllm-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "vLLM", "version": "1", "autoSelect": True}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "quay.io/modh/vllm:rhoai-2.16", + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + }, + ], + }, +} + +SERVING_RUNTIME_NIM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "nim-serving-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "NIM", "version": "1"}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "nvcr.io/nim/meta/llama-3.1-8b-instruct:latest", + "ports": [{"containerPort": 8000, "protocol": "TCP"}], + "env": [ + {"name": "NGC_API_KEY", "valueFrom": { + "secretKeyRef": {"name": "ngc-api-key", "key": "api_key"}, + }}, + ], + }, + ], + }, +} + +PODS_BY_NAMESPACE = { + "ml-production": [ + { + "name": "text-gen-legacy-predictor-00001-abc12", + "namespace": "ml-production", + "status": "CrashLoopBackOff", + "restarts": 5, + "node": "gpu-worker-1", + "containers": [ + { + "name": "kserve-container", + "state": "waiting", + "reason": "CrashLoopBackOff", + "last_termination_reason": "OOMKilled", + "last_termination_exit_code": 137, + }, + ], + "labels": { + "serving.kserve.io/inferenceservice": "text-gen-legacy", + }, + "gpu": "1", + }, + # nim-llama-prod: NO pods created (Account CR not ready) + ], +} + +POD_LOGS = { + "text-gen-legacy-predictor-00001-abc12": ( + "INFO 2026-03-01 10:00:00 vllm_engine.py:125] vLLM engine starting...\n" + "INFO 2026-03-01 10:00:01 config.py:89] Model: mistralai/Mistral-7B-Instruct-v0.3\n" + "INFO 2026-03-01 10:00:01 config.py:92] max_model_len = 32768\n" + "INFO 2026-03-01 10:00:02 gpu_executor.py:45] GPU 0: NVIDIA A10G (24576 MiB)\n" + "INFO 2026-03-01 10:00:03 model_runner.py:88] Loading model weights...\n" + "INFO 2026-03-01 10:00:15 model_runner.py:112] Model weights loaded: 13.5 GiB\n" + "INFO 2026-03-01 10:00:15 worker.py:201] Allocating KV cache...\n" + "ERROR 2026-03-01 10:00:16 worker.py:215] torch.cuda.OutOfMemoryError: " + "CUDA out of memory. Tried to allocate 28.5 GiB for KV cache but only " + "10.1 GiB available after loading model weights (13.5 GiB).\n" + "ERROR 2026-03-01 10:00:16 vllm_engine.py:178] Engine failed to start\n" + "Traceback (most recent call last):\n" + " File \"/opt/vllm/vllm/engine/engine.py\", line 175, in start\n" + " self._init_kv_cache()\n" + " File \"/opt/vllm/vllm/worker/worker.py\", line 215, in _init_kv_cache\n" + " raise torch.cuda.OutOfMemoryError(msg)\n" + "torch.cuda.OutOfMemoryError: CUDA out of memory\n" + ), +} + +EVENTS_BY_NAMESPACE = { + "ml-production": [ + { + "type": "Warning", + "reason": "BackOff", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Back-off restarting failed container kserve-container in pod " + "text-gen-legacy-predictor-00001-abc12", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Warning", + "reason": "OOMKilled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Container kserve-container was OOMKilled (exit code 137). " + "GPU memory exhausted during KV cache allocation.", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Normal", + "reason": "Scheduled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Successfully assigned ml-production/" + "text-gen-legacy-predictor-00001-abc12 to gpu-worker-1", + "count": 1, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-02-28T08:00:00Z", + }, + { + "type": "Warning", + "reason": "NIMAccountNotReady", + "object": "InferenceService/nim-llama-prod", + "message": "NIM Account 'nim-account' in namespace 'ml-production' " + "is not ready", + "count": 12, + "first_timestamp": "2026-03-14T12:00:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + { + "type": "Warning", + "reason": "ImagePullBackOff", + "object": "InferenceService/nim-llama-prod", + "message": "Failed to pull image 'nvcr.io/nim/meta/llama-3.1-8b-instruct:" + "latest': unauthorized: authentication required", + "count": 8, + "first_timestamp": "2026-03-14T12:05:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + ], +} + + +# ── Resource tools ─────────────────────────────────────────────────────── + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: str = "", +) -> str: + """Get a single Kubernetes resource by apiVersion, kind, and name.""" + if kind == "Node": + for node in ALL_NODES: + if node["metadata"]["name"] == name: + return json.dumps(node, indent=2) + raise ValueError(f"Node '{name}' not found") + + if kind == "ServingRuntime": + if name == "vllm-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_VLLM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + if name == "nim-serving-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_NIM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + raise ValueError(f"ServingRuntime '{name}' not found in namespace '{namespace}'") + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps(lr, indent=2) + + if kind == "Account" and "nim" in apiVersion.lower(): + if namespace == "ml-production" and name == "nim-account": + return json.dumps(NIM_ACCOUNT_CR, indent=2) + raise ValueError( + f"Account '{name}' not found in namespace '{namespace}'" + ) + + if kind == "ClusterVersion" and apiVersion == "config.openshift.io/v1": + return json.dumps({ + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterVersion", + "metadata": {"name": "version"}, + "status": {"desired": {"version": "4.16.3"}}, + }) + + raise ValueError(f"Resource {apiVersion}/{kind}/{name} not found") + + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: str = "", + labelSelector: str = "", +) -> str: + """List Kubernetes resources by apiVersion and kind.""" + if kind == "Node": + nodes = ALL_NODES + if labelSelector: + parts = labelSelector.split("=", 1) + key = parts[0] + value = parts[1] if len(parts) > 1 else "" + nodes = [ + n for n in nodes + if n["metadata"]["labels"].get(key) == value + ] + return json.dumps(nodes, indent=2) + + if kind == "Service" and apiVersion == "serving.knative.dev/v1": + return json.dumps({ + "kind": "ServiceList", + "apiVersion": "serving.knative.dev/v1", + "items": [], + "metadata": {}, + }) + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps({ + "kind": "LimitRangeList", + "items": [lr], + }) + + if kind == "InferenceService": + return json.dumps({ + "kind": "InferenceServiceList", + "items": [], + }) + + raise ValueError(f"Unsupported list: {apiVersion}/{kind}") + + +@mcp.tool() +def pods_list( + namespace: str, + labelSelector: str = "", +) -> str: + """List pods in a namespace with optional label selector.""" + pods = PODS_BY_NAMESPACE.get(namespace, []) + + if labelSelector: + key, _, value = labelSelector.partition("=") + pods = [p for p in pods if p.get("labels", {}).get(key) == value] + + results = [] + for pod in pods: + results.append({ + "name": pod["name"], + "namespace": pod["namespace"], + "status": pod["status"], + "restarts": pod.get("restarts", 0), + "node": pod.get("node", ""), + "containers": pod.get("containers", []), + "gpu": pod.get("gpu", "0"), + }) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def pods_log( + namespace: str, + name: str, + container: str = "", +) -> str: + """Get logs from a pod container.""" + logs = POD_LOGS.get(name) + if logs is None: + raise ValueError(f"Pod '{name}' not found in namespace '{namespace}'") + return logs + + +@mcp.tool() +def events_list(namespace: str) -> str: + """List events in a namespace.""" + events = EVENTS_BY_NAMESPACE.get(namespace, []) + return json.dumps(events, indent=2) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/mcp-servers/mock-rhoai-mcp.py b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/mcp-servers/mock-rhoai-mcp.py new file mode 100644 index 00000000..0ae9e4cb --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/mcp-servers/mock-rhoai-mcp.py @@ -0,0 +1,780 @@ +#!/usr/bin/env python3 +"""Mock RHOAI MCP server for SkillsBench rh-ai-engineer task. + +Simulates Red Hat OpenShift AI operations: Data Science Projects, +model serving, data connections, serving runtimes, inference services. + +Scenario: +- ml-production: existing project with two broken deployments + - text-gen-legacy: vLLM OOMKilled (max-model-len=32768 on A10G) + - nim-llama-prod: NIM failing (Account CR not ready, NGC creds invalid) +- fraud-detection: does not exist yet (agent creates it) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("rhoai") + +# ── In-memory state ────────────────────────────────────────────────────── + +PROJECTS = { + "ml-production": { + "name": "ml-production", + "display_name": "ML Production", + "description": "Production ML workloads", + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": "single", + "pipeline_server": True, + }, +} + +DATA_CONNECTIONS = { + "ml-production": [ + { + "name": "prod-model-store", + "type": "S3", + "bucket": "ml-models-prod", + "endpoint": "https://s3.us-east-1.amazonaws.com", + "region": "us-east-1", + }, + ], +} + +SERVING_RUNTIMES = { + "__platform_templates__": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "REST", + "supported_model_formats": [ + {"name": "vLLM", "version": "1", "autoSelect": True} + ], + }, + { + "name": "caikit-tgis-runtime", + "display_name": "Caikit+TGIS ServingRuntime", + "model_formats": ["caikit"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "gRPC", + }, + ], + "ml-production": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + { + "name": "nim-serving-runtime", + "display_name": "NVIDIA NIM ServingRuntime", + "model_formats": ["NIM"], + "requires_instantiation": False, + "source": "nim-account", + "api_protocol": "REST", + }, + { + "name": "ovms-1", + "display_name": "OpenVINO Model Server", + "model_formats": ["openvino_ir", "onnx"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + ], +} + +INFERENCE_SERVICES = { + "ml-production": { + "text-gen-legacy": { + "name": "text-gen-legacy", + "namespace": "ml-production", + "runtime": "vllm-runtime", + "model_format": "vLLM", + "storage_uri": "hf://mistralai/Mistral-7B-Instruct-v0.3", + "display_name": "Mistral 7B Legacy", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "16Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "PredictorFailed", + "message": "Predictor pod is not ready", + }, + { + "type": "PredictorReady", + "status": "False", + "reason": "ContainerCrashLoop", + "message": "Container kserve-container terminated: " + "OOMKilled (exit code 137). 5 restarts.", + }, + { + "type": "IngressReady", + "status": "True", + "reason": "IngressReady", + "message": "Ingress is ready", + }, + ], + "age": "3d", + }, + "nim-llama-prod": { + "name": "nim-llama-prod", + "namespace": "ml-production", + "runtime": "nim-serving-runtime", + "model_format": "NIM", + "storage_uri": "nim://meta/llama-3.1-8b-instruct", + "display_name": "Llama 3.1 8B (NIM)", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "32Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "RuntimeNotReady", + "message": "ServingRuntime 'nim-serving-runtime' " + "is not in ready state", + }, + { + "type": "PredictorReady", + "status": "Unknown", + "reason": "PodNotCreated", + "message": "Predictor pod has not been created. " + "Waiting for ServingRuntime to become ready.", + }, + { + "type": "IngressReady", + "status": "Unknown", + "reason": "PredictorNotReady", + "message": "Waiting for predictor to become ready", + }, + ], + "age": "1d", + }, + }, +} + +DEPLOYED_MODELS = {} + +WORKBENCHES = { + "ml-production": [ + { + "name": "data-exploration-nb", + "display_name": "Data Exploration", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Running", + "cpu_request": "1", + "memory_request": "8Gi", + "gpu_count": 0, + "pvc_name": "data-exploration-nb-pvc", + "pvc_size": "20Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-10T09:00:00Z", + }, + { + "name": "model-training-nb", + "display_name": "Model Training", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Stopped", + "cpu_request": "4", + "memory_request": "16Gi", + "gpu_count": 1, + "pvc_name": "model-training-nb-pvc", + "pvc_size": "50Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-15T14:00:00Z", + }, + ], +} + +PIPELINE_SERVERS = { + "ml-production": { + "configured": True, + "data_connection": "prod-model-store", + "status": "Ready", + "database": "MariaDB", + }, +} + +NOTEBOOK_IMAGES = [ + {"name": "jupyter-pytorch-ubi9-python-3.9-2024.1", "display_name": "PyTorch 2024.1", "packages": ["torch", "transformers"]}, + {"name": "jupyter-tensorflow-ubi9-python-3.9-2024.1", "display_name": "TensorFlow 2024.1", "packages": ["tensorflow"]}, + {"name": "jupyter-datascience-ubi9-python-3.9-2024.1", "display_name": "Standard Data Science", "packages": ["pandas", "scikit-learn"]}, + {"name": "jupyter-minimal-ubi9-python-3.9-2024.1", "display_name": "Minimal Python", "packages": []}, +] + + +# ── Project tools ──────────────────────────────────────────────────────── + + +@mcp.tool() +def list_data_science_projects() -> str: + """List all RHOAI Data Science Projects on the cluster.""" + projects = [] + for name, proj in PROJECTS.items(): + projects.append({ + "name": name, + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + }) + return json.dumps(projects, indent=2) + + +@mcp.tool() +def create_data_science_project( + name: str, + display_name: str, + description: str = "", +) -> str: + """Create a new RHOAI Data Science Project (namespace with dashboard labels).""" + if name in PROJECTS: + raise ValueError( + f"Project '{name}' already exists. Choose a different name " + "or configure the existing project." + ) + if not name.replace("-", "").replace("_", "").isalnum() or len(name) > 63: + raise ValueError( + f"Invalid project name '{name}'. Must be DNS-compatible: " + "lowercase alphanumeric and hyphens, max 63 chars." + ) + + PROJECTS[name] = { + "name": name, + "display_name": display_name, + "description": description, + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": None, + "pipeline_server": False, + } + DATA_CONNECTIONS[name] = [] + SERVING_RUNTIMES[name] = [] + INFERENCE_SERVICES[name] = {} + + return json.dumps({ + "status": "created", + "name": name, + "display_name": display_name, + "namespace": name, + "labels": {"opendatahub.io/dashboard": "true"}, + }) + + +@mcp.tool() +def get_project_details(name: str) -> str: + """Get detailed information about an RHOAI Data Science Project.""" + if name not in PROJECTS: + raise ValueError(f"Project '{name}' not found") + proj = PROJECTS[name] + dc_count = len(DATA_CONNECTIONS.get(name, [])) + isvc_count = len(INFERENCE_SERVICES.get(name, {})) + return json.dumps({ + "name": proj["name"], + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "labels": proj["labels"], + "data_connections": dc_count, + "inference_services": isvc_count, + "model_serving_mode": proj.get("model_serving_mode"), + "pipeline_server": proj.get("pipeline_server", False), + }) + + +@mcp.tool() +def get_project_status(namespace: str) -> str: + """Get comprehensive status of an RHOAI Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Project '{namespace}' not found") + proj = PROJECTS[namespace] + dcs = DATA_CONNECTIONS.get(namespace, []) + isvcs = INFERENCE_SERVICES.get(namespace, {}) + return json.dumps({ + "namespace": namespace, + "display_name": proj["display_name"], + "status": "Active", + "components": { + "data_connections": len(dcs), + "inference_services": len(isvcs), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + "pipeline_server": "configured" if proj.get("pipeline_server") else "not configured", + }, + }) + + +# ── Data connection tools ──────────────────────────────────────────────── + + +@mcp.tool() +def create_s3_data_connection( + namespace: str, + name: str, + bucket: str, + endpoint: str, + access_key: str, + secret_key: str, + region: str = "", +) -> str: + """Create an S3-compatible data connection in an RHOAI project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + existing = DATA_CONNECTIONS.get(namespace, []) + if any(dc["name"] == name for dc in existing): + raise ValueError( + f"Data connection '{name}' already exists in namespace '{namespace}'" + ) + + dc = { + "name": name, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + "region": region, + } + DATA_CONNECTIONS.setdefault(namespace, []).append(dc) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + }) + + +@mcp.tool() +def list_data_connections(namespace: str) -> str: + """List data connections in an RHOAI project namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + return json.dumps(dcs, indent=2) + + +# ── Model serving tools ───────────────────────────────────────────────── + + +@mcp.tool() +def set_model_serving_mode(namespace: str, mode: str) -> str: + """Enable model serving on a Data Science Project (single or multi mode).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + if mode not in ("single", "multi"): + raise ValueError(f"Invalid mode '{mode}'. Must be 'single' or 'multi'.") + + PROJECTS[namespace]["model_serving_mode"] = mode + + if not SERVING_RUNTIMES.get(namespace): + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + SERVING_RUNTIMES[namespace] = [ + {**t, "requires_instantiation": False, "source": "existing"} + for t in templates + ] + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "mode": mode, + }) + + +@mcp.tool() +def list_serving_runtimes( + namespace: str, + include_templates: bool = False, +) -> str: + """List available ServingRuntimes in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + runtimes = list(SERVING_RUNTIMES.get(namespace, [])) + if include_templates: + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + existing_names = {r["name"] for r in runtimes} + for t in templates: + if t["name"] not in existing_names: + runtimes.append(t) + + return json.dumps(runtimes, indent=2) + + +# ── Inference service tools ────────────────────────────────────────────── + + +@mcp.tool() +def deploy_model( + name: str, + namespace: str, + runtime: str, + model_format: str, + storage_uri: str, + display_name: str = "", + min_replicas: int = 1, + max_replicas: int = 1, + cpu_request: str = "1", + cpu_limit: str = "2", + memory_request: str = "4Gi", + memory_limit: str = "8Gi", + gpu_count: int = 0, +) -> str: + """Deploy an AI/ML model as a KServe InferenceService.""" + if namespace not in PROJECTS: + raise ValueError( + f"Namespace '{namespace}' is not a Data Science Project. " + "Create one via create_data_science_project first." + ) + + ns_runtimes = SERVING_RUNTIMES.get(namespace, []) + runtime_names = [r["name"] for r in ns_runtimes] + if runtime not in runtime_names: + available = ", ".join(runtime_names) or "none" + raise ValueError( + f"ServingRuntime '{runtime}' not found in namespace '{namespace}'. " + f"Available runtimes: {available}" + ) + + endpoint = f"https://{name}-{namespace}.apps.ocp-cluster.example.com" + isvc = { + "name": name, + "namespace": namespace, + "runtime": runtime, + "model_format": model_format, + "storage_uri": storage_uri, + "display_name": display_name or name, + "gpu_count": gpu_count, + "cpu_request": cpu_request, + "memory_request": memory_request, + "min_replicas": min_replicas, + "max_replicas": max_replicas, + "ready": True, + "url": endpoint, + "conditions": [ + {"type": "Ready", "status": "True", "reason": "Ready", "message": ""}, + {"type": "PredictorReady", "status": "True", "reason": "PodReady", "message": ""}, + {"type": "IngressReady", "status": "True", "reason": "IngressReady", "message": ""}, + ], + "age": "0s", + } + + INFERENCE_SERVICES.setdefault(namespace, {})[name] = isvc + DEPLOYED_MODELS[f"{namespace}/{name}"] = isvc + + return json.dumps({ + "status": "deployed", + "name": name, + "namespace": namespace, + "runtime": runtime, + "endpoint": endpoint, + "ready": True, + }) + + +@mcp.tool() +def list_inference_services( + namespace: str, + verbosity: str = "standard", +) -> str: + """List deployed InferenceServices in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + isvcs = INFERENCE_SERVICES.get(namespace, {}) + results = [] + for isvc_name, isvc in isvcs.items(): + entry = { + "name": isvc["name"], + "runtime": isvc["runtime"], + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "age": isvc.get("age", ""), + } + if verbosity == "full": + entry["conditions"] = isvc.get("conditions", []) + entry["storage_uri"] = isvc.get("storage_uri", "") + entry["gpu_count"] = isvc.get("gpu_count", 0) + results.append(entry) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def get_inference_service( + name: str, + namespace: str, + verbosity: str = "standard", +) -> str: + """Get detailed status of a specific InferenceService.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + + isvc = isvcs[name] + result = { + "name": isvc["name"], + "namespace": isvc["namespace"], + "runtime": isvc["runtime"], + "model_format": isvc.get("model_format", ""), + "storage_uri": isvc.get("storage_uri", ""), + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "conditions": isvc.get("conditions", []), + "gpu_count": isvc.get("gpu_count", 0), + "replicas": {"min": isvc.get("min_replicas", 1), "max": isvc.get("max_replicas", 1)}, + "resources": { + "cpu_request": isvc.get("cpu_request", "1"), + "memory_request": isvc.get("memory_request", "4Gi"), + "memory_limit": isvc.get("memory_limit", "8Gi"), + }, + "age": isvc.get("age", ""), + } + return json.dumps(result, indent=2) + + +@mcp.tool() +def get_model_endpoint(name: str, namespace: str) -> str: + """Get the inference endpoint URL for a deployed model.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + isvc = isvcs[name] + if not isvc["ready"]: + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": "", + "error": "InferenceService is not ready. Check conditions for details.", + }) + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": isvc["url"], + }) + + +# ── Workbench tools ────────────────────────────────────────────────────── + + +@mcp.tool() +def list_workbenches(namespace: str) -> str: + """List workbenches (Jupyter notebooks) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + return json.dumps(wbs, indent=2) + + +@mcp.tool() +def create_workbench( + namespace: str, + name: str, + display_name: str = "", + image: str = "jupyter-datascience-ubi9-python-3.9-2024.1", + cpu_request: str = "1", + memory_request: str = "4Gi", + gpu_count: int = 0, + pvc_size: str = "20Gi", +) -> str: + """Create a new workbench (Jupyter notebook) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + valid_images = [img["name"] for img in NOTEBOOK_IMAGES] + if image not in valid_images: + raise ValueError( + f"Image '{image}' not found. Available: {', '.join(valid_images)}" + ) + + wb = { + "name": name, + "display_name": display_name or name, + "image": image, + "status": "Running", + "cpu_request": cpu_request, + "memory_request": memory_request, + "gpu_count": gpu_count, + "pvc_name": f"{name}-pvc", + "pvc_size": pvc_size, + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-03-02T12:00:00Z", + } + WORKBENCHES.setdefault(namespace, []).append(wb) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "image": image, + "pvc": f"{name}-pvc", + }) + + +@mcp.tool() +def stop_workbench(namespace: str, name: str) -> str: + """Stop a running workbench (preserves data).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Stopped" + return json.dumps({"status": "stopped", "name": name, "namespace": namespace}) + + +@mcp.tool() +def start_workbench(namespace: str, name: str) -> str: + """Start a stopped workbench.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Running" + return json.dumps({"status": "running", "name": name, "namespace": namespace}) + + +@mcp.tool() +def delete_workbench(namespace: str, name: str) -> str: + """Delete a workbench. WARNING: PVC data may be lost if not backed up.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wbs.remove(wb) + return json.dumps({ + "status": "deleted", + "name": name, + "namespace": namespace, + "warning": "Associated PVC data has been deleted", + }) + + +@mcp.tool() +def list_notebook_images() -> str: + """List available notebook images for workbench creation.""" + return json.dumps(NOTEBOOK_IMAGES, indent=2) + + +# ── Pipeline server tools ─────────────────────────────────────────────── + + +@mcp.tool() +def configure_pipeline_server( + namespace: str, + data_connection: str, + database: str = "MariaDB", +) -> str: + """Configure a pipeline server for a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + if not any(dc["name"] == data_connection for dc in dcs): + available = [dc["name"] for dc in dcs] + raise ValueError( + f"Data connection '{data_connection}' not found. Available: {available}" + ) + + PIPELINE_SERVERS[namespace] = { + "configured": True, + "data_connection": data_connection, + "status": "Ready", + "database": database, + } + PROJECTS[namespace]["pipeline_server"] = True + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "data_connection": data_connection, + "database": database, + }) + + +@mcp.tool() +def get_pipeline_server_status(namespace: str) -> str: + """Get the status of the pipeline server in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + ps = PIPELINE_SERVERS.get(namespace) + if not ps: + return json.dumps({"namespace": namespace, "configured": False}) + return json.dumps({ + "namespace": namespace, + "configured": ps["configured"], + "data_connection": ps["data_connection"], + "status": ps["status"], + "database": ps["database"], + }) + + +# ── Serving runtime creation ──────────────────────────────────────────── + + +@mcp.tool() +def create_serving_runtime( + namespace: str, + name: str, + display_name: str = "", + model_formats: list = None, + container_image: str = "", + container_port: int = 8080, + multi_model: bool = False, + api_protocol: str = "REST", +) -> str: + """Create a custom ServingRuntime in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + if not model_formats: + raise ValueError("model_formats must specify at least one model format") + + runtime = { + "name": name, + "display_name": display_name or name, + "model_formats": model_formats, + "requires_instantiation": False, + "source": "custom", + "api_protocol": api_protocol, + "container_image": container_image, + "container_port": container_port, + "multi_model": multi_model, + } + SERVING_RUNTIMES.setdefault(namespace, []).append(runtime) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "model_formats": model_formats, + }) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/model-deploy/SKILL.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/model-deploy/SKILL.md new file mode 100644 index 00000000..26a4fcb5 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/model-deploy/SKILL.md @@ -0,0 +1,382 @@ +--- +name: model-deploy +description: | + Deploy AI/ML models on OpenShift AI using KServe with vLLM, NVIDIA NIM, or Caikit+TGIS runtimes. + + Use when: + - "Deploy Llama 3 on my cluster" + - "Set up a vLLM inference endpoint" + - "Deploy a model with NIM" + - "Create an InferenceService for Granite" + - "I need to serve a model on OpenShift AI" + + Handles runtime selection, GPU validation, InferenceService CR creation, and rollout monitoring. + + NOT for NIM platform setup (use /nim-setup first). + NOT for custom runtime creation (use /serving-runtime-config). +model: inherit +color: green +--- + +# /model-deploy Skill + +Deploy AI/ML models on Red Hat OpenShift AI using KServe. Supports vLLM, NVIDIA NIM, and Caikit+TGIS serving runtimes. Handles runtime selection, hardware profile lookup (with live doc fallback), GPU pre-flight checks, InferenceService CR creation, rollout monitoring, and post-deployment validation. + +## Prerequisites + +**Required MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) + +**Required MCP Tools** (from rhoai): +- `deploy_model` - Create InferenceService with high-level parameters (no YAML construction needed) +- `list_inference_services` - List deployed models with structured status data +- `get_inference_service` - Get detailed model deployment status (conditions, endpoint, ready state) +- `get_model_endpoint` - Get inference endpoint URL directly +- `list_serving_runtimes` - List available runtimes including platform templates with supported model formats +- `list_data_science_projects` - Discover RHOAI projects for namespace validation +- `list_data_connections` - Verify model storage access (S3 data connections) + +**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools** (from openshift): +- `resources_get` (from openshift) - Check NIM Account CR, LimitRange, GPU node taints +- `resources_list` (from openshift) - Check Knative availability, GPU nodes, existing deployments +- `pods_list` (from openshift) - Check predictor pod status during rollout +- `pods_log` (from openshift) - Retrieve pod logs for debugging +- `events_list` (from openshift) - Check events for errors + +**Optional MCP Server**: `ai-observability` ([AI Observability MCP](https://github.com/rh-ai-quickstart/ai-observability-summarizer)) + +**Optional MCP Tools** (from ai-observability): +- `get_gpu_info` - Pre-flight GPU inventory check +- `get_deployment_info` - Post-deployment validation +- `analyze_vllm` - Verify metrics are flowing after deployment + +**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, KServe, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). + +**Additional cluster requirements**: +- For NIM runtime: NIM platform set up via `/nim-setup` +- For vLLM/NIM: NVIDIA GPU nodes available in the cluster + +## When to Use This Skill + +**Use this skill when you need to:** +- Deploy an AI/ML model on OpenShift AI (KServe InferenceService) +- Set up vLLM, NIM, or Caikit+TGIS inference endpoints +- Look up hardware profiles and GPU requirements for a model +- Perform pre-flight validation before model deployment (GPU availability, namespace readiness, LimitRange conflicts) + +**Do NOT use this skill when:** +- You need to set up the NIM platform first (use `/nim-setup`) +- You need to create or customize a ServingRuntime (use `/serving-runtime-config`) +- You need to troubleshoot a failed or slow deployment (use `/debug-inference`) +- You need to analyze model performance or GPU metrics (use `/ai-observability`) + +## Workflow + +### Step 1: Gather Deployment Information + +Collect the following from the user. Use defaults where sensible, but always confirm. + +**Ask the user for:** +- **Model name**: Which model to deploy (e.g., "Llama 3.1 8B", "Granite 3.1 8B") +- **Runtime preference**: vLLM (default), NIM, or Caikit+TGIS (auto-detect if not specified) +- **Namespace**: Target namespace (must have model serving enabled) +- **Model source**: Where the model weights are stored (S3, OCI registry, PVC, or NGC for NIM) +- **Deployment mode**: Serverless (Knative, default) or RawDeployment + +**Present configuration table for review:** + +| Setting | Value | Source | +|---------|-------|--------| +| Model | [model-name] | user input | +| Runtime | [to be determined in Step 2] | auto-detect / user input | +| Namespace | [namespace] | user input | +| Model Source | [source-uri] | user input | +| Deployment Mode | [Serverless/RawDeployment] | user input / default: Serverless | + +Proceed automatically without waiting for user input. + +### Step 1.5: Pre-flight Environment Validation + +**CRITICAL**: Run these checks BEFORE deploying to avoid repeated deployment failures. + +Read [model-deploy-preflight-checklist.md](references/model-deploy-preflight-checklist.md) for the full pre-flight protocol. The checklist validates: +- Namespace is an RHOAI Data Science Project +- Model storage access (S3 data connections) +- Deployment mode support (Knative availability) +- Namespace resource constraints (LimitRange conflicts with KServe sidecars) +- GPU node taints (auto-generate tolerations) +- Existing deployments (reference configuration) +- Model source accessibility (OCI registry entitlements) + +**Present pre-flight results** in a summary table and note any adjustments made. Proceed automatically without waiting for user input. + +### Step 2: Determine Runtime + +**Document Consultation** (read before selecting runtime): +1. **Action**: Read [supported-runtimes.md](../../docs/references/supported-runtimes.md) using the Read tool to understand runtime capabilities and selection criteria +2. **Output to user**: "I consulted [supported-runtimes.md](../../docs/references/supported-runtimes.md) to understand runtime capabilities." + +**Runtime Selection Logic:** + +- User explicitly requested a runtime -> Use that runtime +- Model available in NGC NIM catalog -> Suggest NIM (with vLLM as fallback) +- Model is a standard open-source LLM (HuggingFace-compatible) -> Default to vLLM +- Model is in Caikit format -> Caikit+TGIS +- None of the above -> Suggest custom runtime via `/serving-runtime-config` + +**Present recommendation** with rationale. Proceed automatically without waiting for user input. + +### Step 3: Look Up Model Hardware Profile + +**Document Consultation** (read before determining hardware requirements): +1. **Action**: Read [known-model-profiles.md](../../docs/references/known-model-profiles.md) using the Read tool to find hardware profile for the requested model +2. **Output to user**: "I consulted [known-model-profiles.md](../../docs/references/known-model-profiles.md) to find hardware requirements for [model-name]." + +**If model IS in known-model-profiles.md:** +- Extract: GPU count, GPU type, VRAM, key vLLM args +- Present to user + +**If model is NOT in known-model-profiles.md -> Trigger live doc lookup:** +1. **Action**: Read [live-doc-lookup.md](../references/live-doc-lookup.md) using the Read tool for the lookup protocol +2. **Output to user**: "Model [model-name] is not in my cached profiles. I'll look up its hardware requirements." +3. Use **WebFetch** tool to retrieve specs from: + - For NIM models: `https://build.nvidia.com/models` or `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + - For other models: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` +4. Extract: GPU requirements, model-specific args, known issues +5. **Output to user**: "I looked up [model-name] on [source] to confirm its hardware requirements: [summary]" + +**Present hardware requirements** in a table (GPUs, VRAM, Key Args). + +### Step 4: Pre-flight GPU Check (Optional) + +**Condition**: Only if `ai-observability` MCP server is available. + +**MCP Tool**: `get_gpu_info` (from ai-observability) + +Compare available GPUs against model requirements from Step 3: +- If sufficient GPUs available -> Report match and proceed +- If insufficient -> Warn user with options: smaller model, quantized version, different cluster, or proceed at user's risk + +**If ai-observability not available**: Skip with note: "GPU pre-flight check skipped (ai-observability MCP not configured)." + +### Step 5: Verify NIM Platform (NIM Runtime Only) + +**Condition**: Only when the selected runtime is NIM. + +**MCP Tool**: `resources_get` (from openshift) + +**Parameters**: +- `apiVersion`: `"nim.opendatahub.io/v1alpha1"` - REQUIRED +- `kind`: `"Account"` - REQUIRED +- `namespace`: target namespace - REQUIRED +- `name`: `"nim-account"` - REQUIRED + +**If Account CR not found or not ready:** +Offer options: (1) Run `/nim-setup` now, (2) Switch to vLLM, (3) Abort. Proceed automatically without waiting for user input. + +### Step 6: Select ServingRuntime and Prepare Deployment Parameters + +**Verify available ServingRuntimes:** + +**MCP Tool**: `list_serving_runtimes` (from rhoai) + +**Parameters**: +- `namespace`: target namespace - REQUIRED +- `include_templates`: `true` - REQUIRED (shows both existing runtimes and platform templates) + +The response shows existing runtimes and available templates with their supported model formats and `requires_instantiation` flag. + +If the needed runtime shows `requires_instantiation: true`, it must first be instantiated via `/serving-runtime-config` or the rhoai `create_serving_runtime` tool. + +Use the runtime list to select the correct `runtime` name for the deployment. + +**Prepare deployment parameters** from Steps 1-3 and environment data from Step 1.5: + +| Parameter | Value | Source | +|-----------|-------|--------| +| `name` | [model-deployment-name] | user input (DNS-compatible) | +| `namespace` | [namespace] | user input | +| `runtime` | [serving-runtime-name] | selected from `list_serving_runtimes` | +| `model_format` | [vLLM/pytorch/onnx/caikit/etc.] | runtime selection | +| `storage_uri` | [model-source-uri] | user input (prefer `hf://` for public models) | +| `gpu_count` | [gpu-count] | from hardware profile (Step 3) | +| `cpu_request` | [cpu] | from profile, adjusted for LimitRange | +| `memory_request` | [memory] | from profile, adjusted for LimitRange | +| `min_replicas` | [1] | default 1 (0 for scale-to-zero) | +| `max_replicas` | [1] | default 1 | + +**Model sizing guide** for LLMs: +- 7B models: 1x 24GB GPU (e.g., A10G) or 1x 16GB GPU with quantization +- 13B models: 2x 24GB GPUs +- 70B models: 4+ 80GB GPUs (A100/H100) or quantized on fewer GPUs + +**Scale-to-zero note**: Setting `min_replicas=0` saves resources but introduces cold start latency (30s-2min for model loading). + +### Step 7: User Review and Confirmation + +**Display the deployment parameters table** and a configuration summary to the user. + +**Ask**: "Proceed with deploying this model? (yes/no/modify)" + +Proceed automatically without waiting for user input. + +- If **yes** -> Proceed to Step 8 +- If **no** -> Abort +- If **modify** -> Ask what to change, update parameters, return to this step + +### Step 8: Deploy Model + +**MCP Tool**: `deploy_model` (from rhoai) + +**Parameters**: +- `name`: deployment name (DNS-compatible) - REQUIRED +- `namespace`: target namespace - REQUIRED +- `runtime`: serving runtime name from Step 6 - REQUIRED +- `model_format`: model format string (e.g., `"vLLM"`, `"pytorch"`, `"onnx"`) - REQUIRED +- `storage_uri`: model location (e.g., `"hf://ibm-granite/granite-3.1-2b-instruct"`, `"s3://bucket/path"`, `"pvc://pvc-name/path"`) - REQUIRED +- `display_name`: human-readable display name - OPTIONAL +- `min_replicas`: minimum replicas (default: 1, 0 for scale-to-zero) - OPTIONAL +- `max_replicas`: maximum replicas (default: 1) - OPTIONAL +- `cpu_request`: CPU request per replica (default: `"1"`) - OPTIONAL +- `cpu_limit`: CPU limit per replica (default: `"2"`) - OPTIONAL +- `memory_request`: memory request per replica (default: `"4Gi"`) - OPTIONAL +- `memory_limit`: memory limit per replica (default: `"8Gi"`) - OPTIONAL +- `gpu_count`: number of GPUs per replica (default: 0) - OPTIONAL + +**Note**: For NIM deployments, ensure the NGC API key secret is referenced. If `deploy_model` does not support NIM-specific env vars, fall back to `resources_create_or_update` (from openshift) with a NIM InferenceService YAML that includes `spec.predictor.env` referencing the `ngc-api-key` secretKeyRef. + +**Error Handling**: +- If namespace not found -> Report error, suggest creating namespace or using `/ds-project-setup` +- If ServingRuntime not found -> Report error, verify runtime name, suggest `/serving-runtime-config` +- If quota exceeded -> Report error, suggest reducing resource requests +- If RBAC error -> Report insufficient permissions + +### Step 9: Monitor Rollout + +Poll InferenceService status until ready or timeout (10 minutes). + +**MCP Tool**: `get_inference_service` (from rhoai) +- `name`: deployment name, `namespace`: target namespace, `verbosity`: `"full"` + +Check the Ready condition and status. Repeat every 15-30 seconds until Ready=True or timeout. + +**Check predictor pod status:** + +**MCP Tool**: `pods_list` (from openshift) +- `namespace`: target namespace, `labelSelector`: `"serving.kserve.io/inferenceservice=[model-name]"` + +Show deployment progress tracking: Pod Scheduled, Image Pulled, Container Started, Model Loaded, Ready. Include pod name, status, and restart count. + +**On failure:** Check pod logs (`pods_log`) and events (`events_list`) for diagnostics. Present options: (1) View full pod logs, (2) Check namespace events, (3) Invoke `/debug-inference`, (4) Delete and retry, (5) Continue waiting. Proceed automatically without waiting for user input. + +### Step 10: Deployment Complete + +**Get endpoint URL:** + +**MCP Tool**: `get_model_endpoint` (from rhoai) +- `name`: deployment name, `namespace`: target namespace + +**Report success** showing: model name, runtime, namespace, GPUs, inference endpoint URL, API type (OpenAI-compatible REST), and next steps (`/ai-observability`, `/model-monitor`, `/guardrails-config`). + +**Provide test commands** based on runtime: +- **vLLM (OpenAI-compatible)**: `curl -X POST [endpoint]/v1/completions -H "Content-Type: application/json" -d '{"model":"[model-name]","prompt":"Hello","max_tokens":100}'` +- **KServe v2**: `curl -X POST [endpoint]/v2/models/[model-name]/infer -H "Content-Type: application/json" -d '{"inputs":[...]}'` + +**Post-deployment validation** (if ai-observability MCP available): +- `get_deployment_info` to confirm model appears in monitoring +- `analyze_vllm` with a short time range to verify initial metrics are flowing +- Report findings to user + +## Common Issues + +For common issues (GPU scheduling, OOMKilled, image pull errors, RBAC), see [common-issues.md](../references/common-issues.md). + +### Issue 1: InferenceService Stuck in "Unknown" + +**Error**: InferenceService `status.conditions` shows "Unknown" state + +**Cause**: ServingRuntime not found in the namespace, or model serving platform not enabled. + +**Solution:** +1. Verify ServingRuntime exists: `resources_list` for `servingruntimes` in namespace +2. Ensure model serving is enabled: namespace has label `opendatahub.io/dashboard: "true"` +3. Check the runtime name in the InferenceService matches an available ServingRuntime +4. If no matching runtime, use `/serving-runtime-config` to create one + +### Issue 2: Model Download Timeout + +**Error**: Pod starts but times out while downloading model weights from S3 or OCI registry + +**Cause**: Large model size combined with slow network connection to storage. + +**Solution:** +1. Add `serving.knative.dev/progress-deadline` annotation with a longer timeout (e.g., `"1800s"`) +2. Verify S3/storage credentials are valid +3. Consider using a PVC with pre-downloaded model weights instead +4. Check network connectivity between the pod and storage endpoint + +### Issue 3: LimitRange Conflicts with KServe Sidecars + +**Error**: Pod rejected with `minimum cpu usage per Container is 50m, but request is 10m` or `minimum memory usage per Container is 64Mi, but request is 15Mi` + +**Cause**: The namespace has a LimitRange with minimum resource constraints that exceed the hardcoded resource requests of KServe-injected sidecar containers (oauth-proxy, queue-proxy, or modelcar containers request 10m CPU / 15Mi memory). These sidecar resource values cannot be controlled through the InferenceService spec. + +**Solution:** +1. Check LimitRange: `resources_list` for `LimitRange` in the namespace +2. If LimitRange minimum CPU > 10m or minimum memory > 15Mi, the LimitRange must be adjusted +3. Options: (a) Lower LimitRange minimums to accommodate sidecars (min CPU ≤ 10m, min memory ≤ 15Mi), (b) Remove the LimitRange entirely, (c) Deploy in a different namespace without restrictive LimitRanges +4. **Prevention**: Step 1.5 pre-flight validation now checks for this conflict before deployment + +### Issue 4: GPU Node Taints Prevent Scheduling + +**Error**: Pod stuck in Pending with events showing `node(s) had untolerated taint {ai-app: true}` or similar custom taint messages, while also showing `Insufficient nvidia.com/gpu` on remaining nodes + +**Cause**: GPU nodes are tainted with custom taints (e.g., `ai-app=true:NoSchedule`) to reserve them for AI workloads. The InferenceService predictor pod does not have matching tolerations, so it cannot be scheduled on GPU nodes. + +**Solution:** +1. Identify GPU node taints: `resources_get` for GPU nodes, check `.spec.taints` +2. Add matching tolerations to the InferenceService predictor spec: + ```yaml + spec: + predictor: + tolerations: + - key: "ai-app" + operator: "Equal" + value: "true" + effect: "NoSchedule" + ``` +3. **Prevention**: Step 1.5 pre-flight validation now auto-discovers GPU node taints and generates tolerations + +## Dependencies + +### MCP Tools +See [Prerequisites](#prerequisites) for the complete list of required and optional MCP tools. + +### Related Skills +- `/nim-setup` - Prerequisite for NIM runtime deployments +- `/debug-inference` - Troubleshoot InferenceService failures +- `/ai-observability` - Analyze deployed model performance +- `/serving-runtime-config` - Create custom ServingRuntime CRs +- `/ds-project-setup` - Create a namespace with model serving enabled + +### Reference Documentation +- [known-model-profiles.md](../../docs/references/known-model-profiles.md) - Hardware profiles for common models +- [supported-runtimes.md](../../docs/references/supported-runtimes.md) - Runtime capabilities and selection criteria +- [live-doc-lookup.md](../references/live-doc-lookup.md) - Protocol for fetching specs for unknown models + +## Critical: Human-in-the-Loop Requirements + +See [skill-conventions.md](../references/skill-conventions.md) for general HITL and security conventions. + +**Skill-specific checkpoints:** +- After gathering settings (Step 1): confirm configuration table +- After pre-flight validation (Step 1.5): confirm if significant adjustments were needed (deployment mode, tolerations, resource changes) +- After runtime selection (Step 2): confirm runtime choice +- Before calling deploy_model (Step 7): review and confirm deployment parameters +- On deployment failure (Step 9): present diagnostic options, wait for user decision +- **NEVER** auto-delete failed deployments or auto-select runtimes without confirmation + +## Example Usage + +See [model-deploy examples](../../docs/examples/model-deploy.md) for complete deployment walkthroughs (vLLM and NIM). diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/model-deploy/references/model-deploy-preflight-checklist.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/model-deploy/references/model-deploy-preflight-checklist.md new file mode 100644 index 00000000..02dbd83d --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/model-deploy/references/model-deploy-preflight-checklist.md @@ -0,0 +1,64 @@ +# Model Deploy Pre-flight Checklist + +Run these checks BEFORE deploying to avoid repeated deployment failures. Each check uses MCP tools to validate the target environment. + +## 0. Validate Namespace is an RHOAI Data Science Project + +**MCP Tool**: `list_data_science_projects` (from rhoai) + +Verify the target namespace appears in the project list. If not found, warn: "Namespace `[namespace]` is not a Data Science Project. Model serving may not be configured. Create one via the OpenShift AI dashboard or proceed at your own risk." + +## 0b. Check Model Storage Access (S3 Sources) + +**MCP Tool**: `list_data_connections` (from rhoai) +- `namespace`: target namespace + +If model source is S3-based, verify a matching data connection exists. If not found, inform user: "No S3 data connection found in namespace. Create one via the OpenShift AI dashboard or provide model source as PVC or HuggingFace URI." + +## 1. Check Deployment Mode Support + +**MCP Tool**: `resources_list` (from openshift) +- `apiVersion`: `"serving.knative.dev/v1"`, `kind`: `"Service"`, `namespace`: target namespace + +If Knative Services are not available (CRD not found or error) -> auto-select **RawDeployment** mode and inform the user: "Knative Services are not available on this cluster. Switching to RawDeployment mode." + +## 2. Check Namespace Resource Constraints + +**MCP Tool**: `resources_list` (from openshift) +- `apiVersion`: `"v1"`, `kind`: `"LimitRange"`, `namespace`: target namespace + +If a LimitRange exists: +- **Action**: `resources_get` for each LimitRange to extract min/max/default values +- Validate that planned resource requests fit within max limits +- **Warning**: If LimitRange minimum CPU > 10m or minimum memory > 15Mi, KServe-injected sidecar containers (with hardcoded 10m CPU / 15Mi memory requests) will fail to schedule. Warn the user: "LimitRange minimums conflict with KServe sidecar containers. The LimitRange must be adjusted or removed before deployment can succeed." +- Adjust planned resource requests/limits to fit within constraints +- Present adjusted values to user + +## 3. Discover GPU Node Taints + +**MCP Tool**: `resources_list` (from openshift) +- `apiVersion`: `"v1"`, `kind`: `"Node"`, `labelSelector`: `"nvidia.com/gpu.present=true"` + +For each GPU node, extract taints. If custom taints exist (beyond standard Kubernetes taints like `node-role.kubernetes.io/*`): +- Auto-generate matching tolerations for the InferenceService +- Present discovered taints and proposed tolerations to user for confirmation +- Common example: `ai-app=true:NoSchedule` requires toleration `{key: "ai-app", operator: "Equal", value: "true", effect: "NoSchedule"}` + +## 4. Check Existing Deployments in Namespace + +**MCP Tool**: `list_inference_services` (from rhoai) +- `namespace`: target namespace +- `verbosity`: `"standard"` + +If similar InferenceServices exist, inspect their `storageUri`, runtime, and tolerations as a reference for proven-working configuration in this environment. + +## 5. Validate Model Source Accessibility + +If using `oci://` source: +- Check namespace service account `imagePullSecrets` can access the registry +- For `registry.redhat.io/rhelai1/*` images: these require RHEL AI subscription entitlements -- verify pull secret has access or recommend switching to `hf://` (HuggingFace) source +- **Default preference**: For public open-source models, prefer `hf://` sources (e.g., `hf://ibm-granite/granite-3.1-2b-instruct`) as they require no authentication + +## Summary + +Present pre-flight results in a summary table and note any adjustments made. **WAIT for user confirmation if significant changes were needed** (e.g., deployment mode switch, resource adjustments, tolerations added). diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/common-issues.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/instruction.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/instruction.md new file mode 100644 index 00000000..44f79a58 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/instruction.md @@ -0,0 +1,15 @@ +# Model Deployment Task + +You are an AI engineer on Red Hat OpenShift AI. Your team has trained models ready to serve and needs them deployed as inference endpoints in the `ml-production` project. + +## Requirements +- Examine the existing project, available serving runtimes, and any existing deployments +- Diagnose any failing deployments: check pod conditions, container status, logs, and events to determine root causes +- For GPU memory issues, provide a VRAM budget analysis showing model weight size, KV cache requirements, and available GPU memory — distinguish GPU VRAM constraints from pod system memory limits +- Before recommending fixes, check the namespace environment for resource policies and GPU node scheduling constraints that could block redeployment +- For each failing deployment, provide a complete KServe InferenceService YAML manifest with your recommended fix +- Produce a deployment plan that addresses all identified issues and gets the models serving successfully + +Document your deployment plan, diagnosed issues, environment validation, and recommended fixes in `/root/report.md`. + +Use MCP tools to interact with the platform. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/solution/solve.sh b/evaluation/with_skills/rh-ai-engineer__model-deploy/solution/solve.sh new file mode 100644 index 00000000..05b7171e --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/solution/solve.sh @@ -0,0 +1,63 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Model Deployment Plan + +## Diagnosed Issues + +### GPU VRAM Budget Analysis +The vLLM OOM is a **GPU VRAM constraint**, not a pod system memory issue: +- Model weights: ~13.5 GiB loaded into GPU +- KV cache allocation: ~28.5 GiB (at default max_model_len=32768) +- Available VRAM after model load: ~10.1 GiB on A10G (24576 MiB total) +- **Root cause**: Default max_model_len=32768 causes KV cache to exhaust GPU VRAM +- **Fix**: Set MAX_MODEL_LEN=4096 or GPU_MEMORY_UTILIZATION=0.85 + +### LimitRange Conflict +- Namespace LimitRange min CPU: 100m +- KServe sidecar containers request: 10m CPU, 15Mi memory +- **CONFLICT**: Sidecar resources below LimitRange minimum +- Fix: Adjust LimitRange or use annotation to override + +### GPU Node Taints +- GPU nodes may have taint ai-app=true:NoSchedule +- Add matching tolerations to InferenceService predictor spec + +### NIMAccount Dependency +- NIM deployments require a NIMAccount CR to be ready before ServingRuntime can pull images +- Check for NIMAccountNotReady condition if ImagePullBackOff occurs + +## Recommended InferenceService YAML + +```yaml +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: llama-3-8b + namespace: ml-production + annotations: + serving.kserve.io/deploymentMode: RawDeployment +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: vllm-cuda-runtime + storageUri: "hf://meta-llama/Llama-3-8B" + resources: + requests: + cpu: "4" + memory: "32Gi" + nvidia.com/gpu: "1" + containers: + - name: kserve-container + env: + - name: MAX_MODEL_LEN + value: "4096" + - name: GPU_MEMORY_UTILIZATION + value: "0.85" +``` + +## Endpoint +- get_model_endpoint for inference URL +- vLLM: /v1/completions, KServe v2: /v2/models/[model]/infer +REPORT_EOF diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/task.toml b/evaluation/with_skills/rh-ai-engineer__model-deploy/task.toml new file mode 100644 index 00000000..90674851 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-ai-engineer__model-deploy" +name = "rh-ai-engineer Model Deployment Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-ai-engineer", "model-deploy", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/tests/llm_judge.py b/evaluation/with_skills/rh-ai-engineer__model-deploy/tests/llm_judge.py new file mode 100644 index 00000000..5cd7c20e --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "vram_budget_methodology", + "file": "/root/report.md", + "question": "Does the report present a specific GPU VRAM budget calculation for the vLLM OOM issue — showing the model weight size (~13.5 GiB), the KV cache allocation requirement (~28.5 GiB), and the available VRAM after model load (~10.1 GiB) — and explicitly state that this is a GPU VRAM constraint, NOT a pod system memory issue?", + "reference": "A skilled report shows a VRAM budget breakdown: model weights (~13.5 GiB) loaded into GPU, KV cache requiring ~28.5 GiB, but only ~10.1 GiB available on the 24 GB A10G after model load. It explicitly distinguishes GPU VRAM from pod memory (system RAM). A report that says 'OOMKilled' and recommends increasing pod memory from 16Gi to 32Gi WITHOUT this GPU VRAM analysis is insufficient." + }, + { + "id": "rhoai_deployment_conventions", + "file": "/root/report.md", + "question": "Does the report use RHOAI-specific deployment conventions such as the RawDeployment annotation and GPU_MEMORY_UTILIZATION environment variable configuration, rather than generic Kubernetes deployment patterns?", + "reference": "A skilled report uses serving.kserve.io/deploymentMode: RawDeployment annotation and configures vLLM tuning parameters (GPU_MEMORY_UTILIZATION, MAX_MODEL_LEN) as environment variables in the InferenceService spec. It also identifies NIMAccount CR dependencies for NIM deployments. A report that uses generic Kubernetes deployments or command-line args without RHOAI-specific annotations is insufficient." + }, + { + "id": "kserve_yaml_manifest", + "file": "/root/report.md", + "question": "Does the report include a complete KServe InferenceService YAML manifest with the serving.kserve.io/v1beta1 apiVersion, including metadata (name, namespace) and spec.predictor with model format, storage URI, resource requests, and GPU count?", + "reference": "A skilled report provides a deployable InferenceService YAML with apiVersion: serving.kserve.io/v1beta1, kind: InferenceService, and a complete spec including predictor with model format, runtime reference, storage URI, resource requests (CPU, memory, GPU), and environment variables (VLLM_MAX_MODEL_LEN). A report that only describes fixes in narrative or MCP tool call format without a formal YAML manifest is insufficient." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/tests/test.sh b/evaluation/with_skills/rh-ai-engineer__model-deploy/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/tests/test_outputs.py b/evaluation/with_skills/rh-ai-engineer__model-deploy/tests/test_outputs.py new file mode 100644 index 00000000..0669d687 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__model-deploy/tests/test_outputs.py @@ -0,0 +1,94 @@ +""" +Tests for rh-ai-engineer__model-deploy per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ["model", "deploy", "inference", "serving"]), ( + "report should mention model deployment" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 200, "report should have substantial content" + + +class TestSkillDependent: + def test_vram_budget_analysis(self): + """Skill teaches GPU VRAM budget: model weights (13.5 GiB) + KV cache (28.5 GiB) + exceeds A10G capacity (24 GB). Without skill, agents report OOM with approximate + numbers (~14GB) without KV cache sizing or available VRAM calculation.""" + c = read_report() + assert any(t in c for t in [ + "28.5", "10.1 GiB", "10.1 GB", "24576", + ]), ( + "should include specific VRAM budget numbers " + "(KV cache size ~28.5 GiB, available VRAM ~10.1 GiB, or total GPU VRAM 24576 MiB)" + ) + + def test_default_context_window_32768(self): + """Skill teaches that vLLM default max_model_len=32768 causes KV cache to exhaust + GPU VRAM on A10G. Without skill, agents report OOM without identifying the specific + default value that triggers the oversized KV cache allocation.""" + c = read_report() + assert "32768" in c or "32,768" in c, ( + "should identify max_model_len=32768 as the specific vLLM default causing GPU OOM" + ) + + def test_kserve_yaml_apiversion(self): + """Skill teaches creating InferenceService YAML with serving.kserve.io/v1beta1. + Without skill, agents describe fixes via MCP tool calls or narrative without + providing a formal KServe YAML manifest with the correct apiVersion.""" + c = read_report() + assert "serving.kserve.io/v1beta1" in c, ( + "should include InferenceService YAML manifest with serving.kserve.io/v1beta1 apiVersion" + ) + + def test_raw_deployment_mode(self): + """Skill teaches using serving.kserve.io/deploymentMode: RawDeployment annotation + for RHOAI model deployments. Without skill, agents omit this RHOAI-specific + annotation, which controls how KServe deploys the predictor.""" + c = read_report() + assert "RawDeployment" in c or "deploymentMode" in c, ( + "should include RawDeployment annotation (RHOAI deployment mode)" + ) + + def test_known_model_profile(self): + """Docs teach known model profiles: e.g., Llama 3.1 8B needs 1 GPU with 16GB VRAM, + --max-model-len=4096; 70B needs 4xA100 80GB with --tensor-parallel-size=4. + Without docs, agents can't size GPU allocation per model.""" + c = read_report().lower() + assert any(t in c for t in [ + "max-model-len", "max_model_len", "tensor-parallel-size", + "tensor_parallel_size", "16gb", "a100", "a10g", + ]) or ("gpu" in c and ("vram" in c or "model" in c and "profile" in c)), ( + "should reference known model GPU profiles for deployment sizing" + ) + + def test_nim_account_cr(self): + """Skill teaches that NIM deployments require a NIMAccount CR to be ready + before the ServingRuntime can pull images. Without skill, agents diagnose + ImagePullBackOff generically without identifying the NIMAccount dependency.""" + c = read_report() + assert any(t in c for t in [ + "NIMAccount", "NimAccount", "nim-account", "NIM Account", + "NIMAccountNotReady", + ]), "should identify NIMAccount CR as prerequisite for NIM deployment" diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/Dockerfile b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/Dockerfile new file mode 100644 index 00000000..d4978abe --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/Dockerfile @@ -0,0 +1,74 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "rhoai": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-rhoai-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/common-issues.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/examples/model-deploy.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/examples/model-deploy.md new file mode 100644 index 00000000..dcd79e44 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/examples/model-deploy.md @@ -0,0 +1,160 @@ +# /model-deploy Examples + +Example interactions demonstrating the model-deploy skill workflow. + +## Example 1: Deploy Llama 3.1 8B with vLLM + +``` +User: "Deploy Llama 3.1 8B on my cluster" + +Agent: [Invokes /model-deploy skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand runtime capabilities. + +Step 1 of 10: Gathering deployment information. + +| Setting | Value | Source | +|---------|-------|--------| +| Model | Llama 3.1 8B Instruct | user input | +| Runtime | TBD | auto-detect | +| Namespace | ? | needs input | +| Model Source | ? | needs input | +| Deployment Mode | Serverless | default | + +What namespace should I deploy to, and where are the model weights stored? +(e.g., S3 bucket, OCI registry, PVC) + +User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" + +Agent: Step 2 of 10: Determining runtime. + +I consulted supported-runtimes.md to understand runtime capabilities. + +Based on Llama 3.1 8B (open-source HuggingFace model): + +Recommended runtime: vLLM +Reason: Default runtime for open-source LLMs. Broad compatibility, + OpenAI-compatible API, no additional setup required. + +Proceed with vLLM? (yes / no / suggest alternative) + +User: "yes" + +Agent: Step 3 of 10: Looking up hardware profile. + +I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. + +| Requirement | Value | +|-------------|-------| +| GPUs | 1x A10G/L4/A100/H100 | +| VRAM | 16GB | +| Key Args | --max-model-len=4096 --tool-call-parser hermes | + +Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). +Ensure your cluster has at least 1 GPU with 16GB VRAM. + +Step 5 of 10: Skipped (NIM platform check not needed for vLLM). + +Step 6-7 of 10: Generating InferenceService YAML. + +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: llama-3-1-8b + namespace: ai-models + annotations: + serving.kserve.io/deploymentMode: Serverless +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: vllm-runtime + storageUri: s3://my-bucket/llama-3.1-8b + resources: + limits: + nvidia.com/gpu: "1" + requests: + cpu: "4" + memory: "16Gi" + args: + - --max-model-len=4096 + - --tool-call-parser=hermes + +| Setting | Value | +|---------|-------| +| Model | llama-3-1-8b | +| Runtime | vllm-runtime | +| GPUs | 1 | +| Namespace | ai-models | +| Mode | Serverless | + +Proceed with creating this InferenceService? (yes/no/modify) + +User: "yes" + +Agent: Step 8 of 10: Creating InferenceService... + [Creates via resources_create_or_update] + + InferenceService created. Monitoring rollout... + +Step 9 of 10: Monitoring rollout. + +| Stage | Status | +|-------|--------| +| Pod Scheduled | Complete | +| Image Pulled | Complete | +| Container Started | Complete | +| Model Loaded | Complete | +| Ready | Complete | + +Step 10 of 10: Deployment complete! + +Model Deployment Successful + +Model: llama-3-1-8b +Runtime: vLLM +Namespace: ai-models +GPUs: 1 + +Inference Endpoint: + URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com + API: OpenAI-compatible REST + +Quick Test: + curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' + +Next Steps: + Monitor performance: "/ai-observability llama-3-1-8b" +``` + +## Example 2: Deploy a Model with NIM (NIM Not Set Up) + +``` +User: "Deploy Llama 3.1 70B with NIM" + +Agent: [Invokes /model-deploy skill] + ... + Step 2: User requested NIM runtime. + + Step 5: Checking NIM platform... + +NIM platform is not set up in namespace ai-models. + +The NIM runtime requires a one-time platform setup. Would you like to: +1. Run /nim-setup now to configure the NIM platform +2. Switch to vLLM runtime instead (no setup required) +3. Abort deployment + +User: "1" + +Agent: [Invokes /nim-setup skill — walks through NGC credential setup] + ... + NIM Platform Setup Complete. + + Resuming model deployment... + [Continues from Step 6 with NIM runtime] +``` diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/examples/nim-setup.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/examples/nim-setup.md new file mode 100644 index 00000000..ccfc1a5d --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/examples/nim-setup.md @@ -0,0 +1,115 @@ +# /nim-setup Examples + +Example interactions demonstrating the nim-setup skill workflow. + +## Example 1: First-Time NIM Setup + +``` +User: "Set up NIM on my cluster" + +Agent: [Invokes /nim-setup skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand NIM platform requirements. + +Let me verify the prerequisites for NIM setup. + +Step 1 of 7: Checking GPU Operator and NFD Operator... + +GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) +NFD Operator: nfd.v4.19.0 (Succeeded) + +Both operators are installed and healthy. + +Step 2 of 7: I need your NGC credentials. + +To set up NIM, I need your NVIDIA NGC API key. +You can generate one at: https://ngc.nvidia.com/setup/api-key + +Please provide: +1. Your NGC API key +2. The target namespace for NIM resources + +User: "Here's my key: [user provides NGC API key], namespace: ai-models" + +Agent: Step 3 of 7: Creating NGC image pull secret... + +I'll create a docker-registry secret for pulling NIM images from nvcr.io: + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-image-pull-secret + namespace: ai-models + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] + +Should I create this image pull secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + Image pull secret created successfully. + +Step 4 of 7: Creating NGC API key secret... + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-api-key + namespace: ai-models + type: Opaque + stringData: + NGC_API_KEY: "[REDACTED]" + +Should I create this API key secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + API key secret created successfully. + +Step 5 of 7: Creating NIM Account CR... + + apiVersion: nim.opendatahub.io/v1 + kind: Account + metadata: + name: nim-account + namespace: ai-models + spec: + apiKeySecret: + name: ngc-api-key + imagePullSecret: + name: ngc-image-pull-secret + +Should I create this NIM Account CR in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates Account CR via resources_create_or_update] + Account CR created successfully. + +Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) + +User: "no" + +Agent: Step 7 of 7: Validating NIM platform readiness... + +NIM Platform Setup Complete + +Account CR: `nim-account` (namespace: ai-models) + Status: Ready + +NGC Credentials: + Image Pull Secret: ngc-image-pull-secret [created] + API Key Secret: ngc-api-key [created] + +NIM ServingRuntimes Available: + - nim-llama-3-1-8b-instruct + - nim-llama-3-1-70b-instruct + - nim-mistral-7b-instruct + +Next Steps: + To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" +``` diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/references/known-model-profiles.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/references/known-model-profiles.md new file mode 100644 index 00000000..40148981 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/references/known-model-profiles.md @@ -0,0 +1,83 @@ +--- +title: Known Model Hardware Profiles +category: references +tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] +semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] +use_cases: [model-deploy, debug-inference, ai-observability] +last_updated: 2026-03-10 +--- + +# Known Model Hardware Profiles + +Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). + +**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. + +## Model Source Conventions + +Each model lists a recommended `storageUri` with its authentication requirements: +- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** +- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). +- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. + +When the user does not specify a model source, use the `hf://` URI listed in the profile below. + +## Llama 3.x (Meta) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | +| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | +| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | +| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | + +- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) +- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly + +## Granite 3.x (IBM/Red Hat) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | +| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | + +- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) +- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser granite --chat-template` +- Red Hat-supported model family on RHOAI + +## Mixtral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | +| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) +- Mixture-of-Experts architecture: only ~13B/45B parameters active per token + +## Mistral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | +| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) + +## When a Model Is Not Listed + +If the requested model is not in this file, the agent MUST use the live doc lookup protocol: + +1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure +2. Fetch hardware specs from the appropriate source +3. Report findings to the user before proceeding with deployment + +Common cases requiring live lookup: +- Newly released models (after this file's last update) +- Domain-specific fine-tuned models +- Models with custom quantization +- NIM-specific optimized profiles diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/references/supported-runtimes.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/references/supported-runtimes.md new file mode 100644 index 00000000..9d018651 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/references/supported-runtimes.md @@ -0,0 +1,104 @@ +--- +title: Supported Serving Runtimes +category: references +tags: [runtimes, vllm, nim, caikit, tgis, serving] +semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] +use_cases: [model-deploy, serving-runtime-config, nim-setup] +last_updated: 2026-02-26 +--- + +# Supported Serving Runtimes + +This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. + +## Runtime Comparison + +| Runtime | API Type | Model Format | GPU Support | License | Setup Required | +|---------|----------|-------------|-------------|---------|----------------| +| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | +| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | +| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | +| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | + +## vLLM + +**Default runtime for most open-source models.** + +- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) +- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized +- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) +- **Model source**: S3-compatible storage, OCI registry, PVC, URI +- **Key features**: + - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) + - PagedAttention for efficient memory management + - Tensor parallelism for multi-GPU inference + - Tool/function calling support (`--tool-call-parser`) + - Continuous batching for high throughput +- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name + +## NVIDIA NIM + +**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** + +- **API**: OpenAI-compatible REST +- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog +- **GPU support**: NVIDIA only (requires specific GPU models per model profile) +- **Model source**: NGC model catalog (pulled automatically via NGC credentials) +- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) +- **Key features**: + - TensorRT-LLM optimization for lower latency + - Pre-compiled model engines (no compilation on first load) + - Optimized and generic profiles per GPU type + - Automatic model download from NGC +- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. +- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM +- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. +- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state + +## Caikit+TGIS + +**Red Hat's Caikit format with Text Generation Inference Server.** + +- **API**: gRPC (not REST) +- **Model formats**: Caikit format (requires conversion from HuggingFace) +- **GPU support**: NVIDIA +- **Model source**: S3-compatible storage +- **Key features**: + - Red Hat-supported runtime + - gRPC API for streaming inference + - Integrated with RHOAI model serving platform +- **When to choose**: When the model is already in Caikit format or when gRPC API is required. +- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name + +## Custom Runtimes + +**User-provided ServingRuntime CRs for unsupported frameworks.** + +- **API**: Defined by the custom runtime +- **Model formats**: Defined by the custom runtime +- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. +- **How to create**: Use `/serving-runtime-config` skill +- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility + +## Runtime Selection Decision Tree + +``` +Is the user's preferred runtime explicitly stated? +├── Yes → Use that runtime +└── No → Continue + +Is the model available in the NGC NIM catalog? +├── Yes → Suggest NIM (with vLLM as fallback) +│ Note: Requires /nim-setup and NVIDIA GPUs +└── No → Continue + +Is the model in Caikit format? +├── Yes → Caikit+TGIS +└── No → Continue + +Is the model a standard open-source LLM (HuggingFace-compatible)? +├── Yes → vLLM (default) +└── No → Custom runtime via /serving-runtime-config +``` diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..d43c891d --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,540 @@ +#!/usr/bin/env python3 +"""Mock OpenShift MCP server for SkillsBench rh-ai-engineer task. + +Simulates Kubernetes resource CRUD, pod management, logs, and events. + +Key scenario elements: +- LimitRange in namespaces: min CPU=100m, min memory=128Mi + (conflicts with KServe sidecar containers hardcoded at 10m CPU/15Mi memory) +- GPU node with custom taint ai-workload=true:NoSchedule +- NIM Account CR in ml-production: not ready (NGC credentials invalid) +- text-gen-legacy pods: OOMKilled (max-model-len=32768 on A10G) +- nim-llama-prod: no pods created (Account CR not ready) +""" + +import base64 +import json +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + +# ── Cluster state ──────────────────────────────────────────────────────── + +GPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "gpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + "nvidia.com/gpu.present": "true", + "nvidia.com/gpu.product": "NVIDIA-A10G", + }, + }, + "spec": { + "taints": [ + { + "key": "ai-workload", + "value": "true", + "effect": "NoSchedule", + }, + ], + }, + "status": { + "allocatable": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "capacity": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "conditions": [ + {"type": "Ready", "status": "True"}, + ], + }, +} + +CPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "cpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + }, + }, + "spec": {"taints": []}, + "status": { + "allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "capacity": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +MASTER_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "master-1", + "labels": { + "node-role.kubernetes.io/master": "", + "node-role.kubernetes.io/control-plane": "", + }, + }, + "spec": { + "taints": [ + {"key": "node-role.kubernetes.io/master", "effect": "NoSchedule"}, + ], + }, + "status": { + "allocatable": {"cpu": "8", "memory": "32Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +ALL_NODES = [GPU_NODE, CPU_NODE, MASTER_NODE] + +# LimitRange applied by cluster policy to all DS project namespaces +NAMESPACE_LIMITRANGE = { + "apiVersion": "v1", + "kind": "LimitRange", + "metadata": { + "name": "default-limits", + }, + "spec": { + "limits": [ + { + "type": "Container", + "default": { + "cpu": "2", + "memory": "4Gi", + }, + "defaultRequest": { + "cpu": "500m", + "memory": "256Mi", + }, + "min": { + "cpu": "100m", + "memory": "128Mi", + }, + "max": { + "cpu": "32", + "memory": "128Gi", + }, + }, + ], + }, +} + +NIM_ACCOUNT_CR = { + "apiVersion": "nim.opendatahub.io/v1", + "kind": "Account", + "metadata": { + "name": "nim-account", + "namespace": "ml-production", + }, + "spec": { + "apiKeySecret": { + "name": "ngc-api-key", + }, + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "NGCCredentialsInvalid", + "message": "NGC API key validation failed: 401 Unauthorized. " + "The API key in secret 'ngc-api-key' is expired or invalid. " + "Re-create the secret with a valid NGC API key from " + "https://ngc.nvidia.com/setup/api-key and restart the " + "Account reconciliation.", + "lastTransitionTime": "2026-03-14T12:00:00Z", + }, + ], + "nimPullSecretStatus": "Failed", + "nimConfigStatus": "Pending", + }, +} + +SERVING_RUNTIME_VLLM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "vllm-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "vLLM", "version": "1", "autoSelect": True}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "quay.io/modh/vllm:rhoai-2.16", + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + }, + ], + }, +} + +SERVING_RUNTIME_NIM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "nim-serving-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "NIM", "version": "1"}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "nvcr.io/nim/meta/llama-3.1-8b-instruct:latest", + "ports": [{"containerPort": 8000, "protocol": "TCP"}], + "env": [ + {"name": "NGC_API_KEY", "valueFrom": { + "secretKeyRef": {"name": "ngc-api-key", "key": "api_key"}, + }}, + ], + }, + ], + }, +} + +PODS_BY_NAMESPACE = { + "ml-production": [ + { + "name": "text-gen-legacy-predictor-00001-abc12", + "namespace": "ml-production", + "status": "CrashLoopBackOff", + "restarts": 5, + "node": "gpu-worker-1", + "containers": [ + { + "name": "kserve-container", + "state": "waiting", + "reason": "CrashLoopBackOff", + "last_termination_reason": "OOMKilled", + "last_termination_exit_code": 137, + }, + ], + "labels": { + "serving.kserve.io/inferenceservice": "text-gen-legacy", + }, + "gpu": "1", + }, + # nim-llama-prod: NO pods created (Account CR not ready) + ], +} + +POD_LOGS = { + "text-gen-legacy-predictor-00001-abc12": ( + "INFO 2026-03-01 10:00:00 vllm_engine.py:125] vLLM engine starting...\n" + "INFO 2026-03-01 10:00:01 config.py:89] Model: mistralai/Mistral-7B-Instruct-v0.3\n" + "INFO 2026-03-01 10:00:01 config.py:92] max_model_len = 32768\n" + "INFO 2026-03-01 10:00:02 gpu_executor.py:45] GPU 0: NVIDIA A10G (24576 MiB)\n" + "INFO 2026-03-01 10:00:03 model_runner.py:88] Loading model weights...\n" + "INFO 2026-03-01 10:00:15 model_runner.py:112] Model weights loaded: 13.5 GiB\n" + "INFO 2026-03-01 10:00:15 worker.py:201] Allocating KV cache...\n" + "ERROR 2026-03-01 10:00:16 worker.py:215] torch.cuda.OutOfMemoryError: " + "CUDA out of memory. Tried to allocate 28.5 GiB for KV cache but only " + "10.1 GiB available after loading model weights (13.5 GiB).\n" + "ERROR 2026-03-01 10:00:16 vllm_engine.py:178] Engine failed to start\n" + "Traceback (most recent call last):\n" + " File \"/opt/vllm/vllm/engine/engine.py\", line 175, in start\n" + " self._init_kv_cache()\n" + " File \"/opt/vllm/vllm/worker/worker.py\", line 215, in _init_kv_cache\n" + " raise torch.cuda.OutOfMemoryError(msg)\n" + "torch.cuda.OutOfMemoryError: CUDA out of memory\n" + ), +} + +EVENTS_BY_NAMESPACE = { + "ml-production": [ + { + "type": "Warning", + "reason": "BackOff", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Back-off restarting failed container kserve-container in pod " + "text-gen-legacy-predictor-00001-abc12", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Warning", + "reason": "OOMKilled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Container kserve-container was OOMKilled (exit code 137). " + "GPU memory exhausted during KV cache allocation.", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Normal", + "reason": "Scheduled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Successfully assigned ml-production/" + "text-gen-legacy-predictor-00001-abc12 to gpu-worker-1", + "count": 1, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-02-28T08:00:00Z", + }, + { + "type": "Warning", + "reason": "NIMAccountNotReady", + "object": "InferenceService/nim-llama-prod", + "message": "NIM Account 'nim-account' in namespace 'ml-production' " + "is not ready", + "count": 12, + "first_timestamp": "2026-03-14T12:00:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + { + "type": "Warning", + "reason": "ImagePullBackOff", + "object": "InferenceService/nim-llama-prod", + "message": "Failed to pull image 'nvcr.io/nim/meta/llama-3.1-8b-instruct:" + "latest': unauthorized: authentication required", + "count": 8, + "first_timestamp": "2026-03-14T12:05:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + ], +} + + +# ── Resource tools ─────────────────────────────────────────────────────── + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: str = "", +) -> str: + """Get a single Kubernetes resource by apiVersion, kind, and name.""" + if kind == "Node": + for node in ALL_NODES: + if node["metadata"]["name"] == name: + return json.dumps(node, indent=2) + raise ValueError(f"Node '{name}' not found") + + if kind == "ServingRuntime": + if name == "vllm-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_VLLM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + if name == "nim-serving-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_NIM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + raise ValueError(f"ServingRuntime '{name}' not found in namespace '{namespace}'") + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps(lr, indent=2) + + if kind == "Account" and "nim" in apiVersion.lower(): + if namespace == "ml-production" and name == "nim-account": + return json.dumps(NIM_ACCOUNT_CR, indent=2) + raise ValueError( + f"Account '{name}' not found in namespace '{namespace}'" + ) + + if kind == "ClusterVersion" and apiVersion == "config.openshift.io/v1": + return json.dumps({ + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterVersion", + "metadata": {"name": "version"}, + "status": {"desired": {"version": "4.16.3"}}, + }) + + raise ValueError(f"Resource {apiVersion}/{kind}/{name} not found") + + +@mcp.tool() +def resources_create_or_update( + api_version: str, + kind: str, + namespace: str, + name: str, + body: str, +) -> str: + """Create or update a Kubernetes resource. Accepts apiVersion, kind, namespace, name, and body (JSON).""" + try: + resource = json.loads(body) + except json.JSONDecodeError as e: + raise ValueError(f"Invalid JSON body: {e}") from e + + resource.setdefault("metadata", {}) + resource["metadata"]["name"] = name + resource["metadata"]["namespace"] = namespace + resource["apiVersion"] = api_version + resource["kind"] = kind + + if kind == "Secret": + resource.setdefault("type", "Opaque") + return json.dumps({ + "status": "created", + "resource": resource, + "message": f"Secret '{name}' created/updated in namespace '{namespace}'", + }, indent=2) + + if kind in ("NIMAccount", "Account") and "nim" in api_version.lower(): + resource.setdefault("status", {}) + resource["status"]["conditions"] = [ + { + "type": "Ready", + "status": "True", + "reason": "NGCCredentialsValid", + "message": "NGC API key validated successfully", + "lastTransitionTime": "2026-03-17T12:00:00Z", + }, + ] + resource["status"]["nimPullSecretStatus"] = "Ready" + resource["status"]["nimConfigStatus"] = "Ready" + return json.dumps({ + "status": "created", + "resource": resource, + "message": f"NIM Account '{name}' created/updated in namespace '{namespace}'", + }, indent=2) + + if kind == "ConfigMap": + return json.dumps({ + "status": "created", + "resource": resource, + "message": f"ConfigMap '{name}' created/updated in namespace '{namespace}'", + }, indent=2) + + raise ValueError(f"Unsupported kind for create/update: {kind}") + + +@mcp.tool() +def create_secret( + namespace: str, + name: str, + data: dict, + type: str = "Opaque", +) -> str: + """Create a Secret in a namespace. data is a dict of key-value pairs (values will be base64-encoded).""" + if isinstance(data, str): + data = json.loads(data) + encoded_data = {k: base64.b64encode(str(v).encode()).decode() for k, v in data.items()} + resource = { + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": name, "namespace": namespace}, + "type": type, + "data": encoded_data, + } + return json.dumps({ + "status": "created", + "resource": resource, + "message": f"Secret '{name}' created in namespace '{namespace}'", + }, indent=2) + + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: str = "", + labelSelector: str = "", +) -> str: + """List Kubernetes resources by apiVersion and kind.""" + if kind == "Node": + nodes = ALL_NODES + if labelSelector: + parts = labelSelector.split("=", 1) + key = parts[0] + value = parts[1] if len(parts) > 1 else "" + nodes = [ + n for n in nodes + if n["metadata"]["labels"].get(key) == value + ] + return json.dumps(nodes, indent=2) + + if kind == "Service" and apiVersion == "serving.knative.dev/v1": + return json.dumps({ + "kind": "ServiceList", + "apiVersion": "serving.knative.dev/v1", + "items": [], + "metadata": {}, + }) + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps({ + "kind": "LimitRangeList", + "items": [lr], + }) + + if kind == "InferenceService": + return json.dumps({ + "kind": "InferenceServiceList", + "items": [], + }) + + raise ValueError(f"Unsupported list: {apiVersion}/{kind}") + + +@mcp.tool() +def pods_list( + namespace: str, + labelSelector: str = "", +) -> str: + """List pods in a namespace with optional label selector.""" + pods = PODS_BY_NAMESPACE.get(namespace, []) + + if labelSelector: + key, _, value = labelSelector.partition("=") + pods = [p for p in pods if p.get("labels", {}).get(key) == value] + + results = [] + for pod in pods: + results.append({ + "name": pod["name"], + "namespace": pod["namespace"], + "status": pod["status"], + "restarts": pod.get("restarts", 0), + "node": pod.get("node", ""), + "containers": pod.get("containers", []), + "gpu": pod.get("gpu", "0"), + }) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def pods_log( + namespace: str, + name: str, + container: str = "", +) -> str: + """Get logs from a pod container.""" + logs = POD_LOGS.get(name) + if logs is None: + raise ValueError(f"Pod '{name}' not found in namespace '{namespace}'") + return logs + + +@mcp.tool() +def events_list(namespace: str) -> str: + """List events in a namespace.""" + events = EVENTS_BY_NAMESPACE.get(namespace, []) + return json.dumps(events, indent=2) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/mcp-servers/mock-rhoai-mcp.py b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/mcp-servers/mock-rhoai-mcp.py new file mode 100644 index 00000000..0ae9e4cb --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/mcp-servers/mock-rhoai-mcp.py @@ -0,0 +1,780 @@ +#!/usr/bin/env python3 +"""Mock RHOAI MCP server for SkillsBench rh-ai-engineer task. + +Simulates Red Hat OpenShift AI operations: Data Science Projects, +model serving, data connections, serving runtimes, inference services. + +Scenario: +- ml-production: existing project with two broken deployments + - text-gen-legacy: vLLM OOMKilled (max-model-len=32768 on A10G) + - nim-llama-prod: NIM failing (Account CR not ready, NGC creds invalid) +- fraud-detection: does not exist yet (agent creates it) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("rhoai") + +# ── In-memory state ────────────────────────────────────────────────────── + +PROJECTS = { + "ml-production": { + "name": "ml-production", + "display_name": "ML Production", + "description": "Production ML workloads", + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": "single", + "pipeline_server": True, + }, +} + +DATA_CONNECTIONS = { + "ml-production": [ + { + "name": "prod-model-store", + "type": "S3", + "bucket": "ml-models-prod", + "endpoint": "https://s3.us-east-1.amazonaws.com", + "region": "us-east-1", + }, + ], +} + +SERVING_RUNTIMES = { + "__platform_templates__": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "REST", + "supported_model_formats": [ + {"name": "vLLM", "version": "1", "autoSelect": True} + ], + }, + { + "name": "caikit-tgis-runtime", + "display_name": "Caikit+TGIS ServingRuntime", + "model_formats": ["caikit"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "gRPC", + }, + ], + "ml-production": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + { + "name": "nim-serving-runtime", + "display_name": "NVIDIA NIM ServingRuntime", + "model_formats": ["NIM"], + "requires_instantiation": False, + "source": "nim-account", + "api_protocol": "REST", + }, + { + "name": "ovms-1", + "display_name": "OpenVINO Model Server", + "model_formats": ["openvino_ir", "onnx"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + ], +} + +INFERENCE_SERVICES = { + "ml-production": { + "text-gen-legacy": { + "name": "text-gen-legacy", + "namespace": "ml-production", + "runtime": "vllm-runtime", + "model_format": "vLLM", + "storage_uri": "hf://mistralai/Mistral-7B-Instruct-v0.3", + "display_name": "Mistral 7B Legacy", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "16Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "PredictorFailed", + "message": "Predictor pod is not ready", + }, + { + "type": "PredictorReady", + "status": "False", + "reason": "ContainerCrashLoop", + "message": "Container kserve-container terminated: " + "OOMKilled (exit code 137). 5 restarts.", + }, + { + "type": "IngressReady", + "status": "True", + "reason": "IngressReady", + "message": "Ingress is ready", + }, + ], + "age": "3d", + }, + "nim-llama-prod": { + "name": "nim-llama-prod", + "namespace": "ml-production", + "runtime": "nim-serving-runtime", + "model_format": "NIM", + "storage_uri": "nim://meta/llama-3.1-8b-instruct", + "display_name": "Llama 3.1 8B (NIM)", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "32Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "RuntimeNotReady", + "message": "ServingRuntime 'nim-serving-runtime' " + "is not in ready state", + }, + { + "type": "PredictorReady", + "status": "Unknown", + "reason": "PodNotCreated", + "message": "Predictor pod has not been created. " + "Waiting for ServingRuntime to become ready.", + }, + { + "type": "IngressReady", + "status": "Unknown", + "reason": "PredictorNotReady", + "message": "Waiting for predictor to become ready", + }, + ], + "age": "1d", + }, + }, +} + +DEPLOYED_MODELS = {} + +WORKBENCHES = { + "ml-production": [ + { + "name": "data-exploration-nb", + "display_name": "Data Exploration", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Running", + "cpu_request": "1", + "memory_request": "8Gi", + "gpu_count": 0, + "pvc_name": "data-exploration-nb-pvc", + "pvc_size": "20Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-10T09:00:00Z", + }, + { + "name": "model-training-nb", + "display_name": "Model Training", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Stopped", + "cpu_request": "4", + "memory_request": "16Gi", + "gpu_count": 1, + "pvc_name": "model-training-nb-pvc", + "pvc_size": "50Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-15T14:00:00Z", + }, + ], +} + +PIPELINE_SERVERS = { + "ml-production": { + "configured": True, + "data_connection": "prod-model-store", + "status": "Ready", + "database": "MariaDB", + }, +} + +NOTEBOOK_IMAGES = [ + {"name": "jupyter-pytorch-ubi9-python-3.9-2024.1", "display_name": "PyTorch 2024.1", "packages": ["torch", "transformers"]}, + {"name": "jupyter-tensorflow-ubi9-python-3.9-2024.1", "display_name": "TensorFlow 2024.1", "packages": ["tensorflow"]}, + {"name": "jupyter-datascience-ubi9-python-3.9-2024.1", "display_name": "Standard Data Science", "packages": ["pandas", "scikit-learn"]}, + {"name": "jupyter-minimal-ubi9-python-3.9-2024.1", "display_name": "Minimal Python", "packages": []}, +] + + +# ── Project tools ──────────────────────────────────────────────────────── + + +@mcp.tool() +def list_data_science_projects() -> str: + """List all RHOAI Data Science Projects on the cluster.""" + projects = [] + for name, proj in PROJECTS.items(): + projects.append({ + "name": name, + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + }) + return json.dumps(projects, indent=2) + + +@mcp.tool() +def create_data_science_project( + name: str, + display_name: str, + description: str = "", +) -> str: + """Create a new RHOAI Data Science Project (namespace with dashboard labels).""" + if name in PROJECTS: + raise ValueError( + f"Project '{name}' already exists. Choose a different name " + "or configure the existing project." + ) + if not name.replace("-", "").replace("_", "").isalnum() or len(name) > 63: + raise ValueError( + f"Invalid project name '{name}'. Must be DNS-compatible: " + "lowercase alphanumeric and hyphens, max 63 chars." + ) + + PROJECTS[name] = { + "name": name, + "display_name": display_name, + "description": description, + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": None, + "pipeline_server": False, + } + DATA_CONNECTIONS[name] = [] + SERVING_RUNTIMES[name] = [] + INFERENCE_SERVICES[name] = {} + + return json.dumps({ + "status": "created", + "name": name, + "display_name": display_name, + "namespace": name, + "labels": {"opendatahub.io/dashboard": "true"}, + }) + + +@mcp.tool() +def get_project_details(name: str) -> str: + """Get detailed information about an RHOAI Data Science Project.""" + if name not in PROJECTS: + raise ValueError(f"Project '{name}' not found") + proj = PROJECTS[name] + dc_count = len(DATA_CONNECTIONS.get(name, [])) + isvc_count = len(INFERENCE_SERVICES.get(name, {})) + return json.dumps({ + "name": proj["name"], + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "labels": proj["labels"], + "data_connections": dc_count, + "inference_services": isvc_count, + "model_serving_mode": proj.get("model_serving_mode"), + "pipeline_server": proj.get("pipeline_server", False), + }) + + +@mcp.tool() +def get_project_status(namespace: str) -> str: + """Get comprehensive status of an RHOAI Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Project '{namespace}' not found") + proj = PROJECTS[namespace] + dcs = DATA_CONNECTIONS.get(namespace, []) + isvcs = INFERENCE_SERVICES.get(namespace, {}) + return json.dumps({ + "namespace": namespace, + "display_name": proj["display_name"], + "status": "Active", + "components": { + "data_connections": len(dcs), + "inference_services": len(isvcs), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + "pipeline_server": "configured" if proj.get("pipeline_server") else "not configured", + }, + }) + + +# ── Data connection tools ──────────────────────────────────────────────── + + +@mcp.tool() +def create_s3_data_connection( + namespace: str, + name: str, + bucket: str, + endpoint: str, + access_key: str, + secret_key: str, + region: str = "", +) -> str: + """Create an S3-compatible data connection in an RHOAI project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + existing = DATA_CONNECTIONS.get(namespace, []) + if any(dc["name"] == name for dc in existing): + raise ValueError( + f"Data connection '{name}' already exists in namespace '{namespace}'" + ) + + dc = { + "name": name, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + "region": region, + } + DATA_CONNECTIONS.setdefault(namespace, []).append(dc) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + }) + + +@mcp.tool() +def list_data_connections(namespace: str) -> str: + """List data connections in an RHOAI project namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + return json.dumps(dcs, indent=2) + + +# ── Model serving tools ───────────────────────────────────────────────── + + +@mcp.tool() +def set_model_serving_mode(namespace: str, mode: str) -> str: + """Enable model serving on a Data Science Project (single or multi mode).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + if mode not in ("single", "multi"): + raise ValueError(f"Invalid mode '{mode}'. Must be 'single' or 'multi'.") + + PROJECTS[namespace]["model_serving_mode"] = mode + + if not SERVING_RUNTIMES.get(namespace): + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + SERVING_RUNTIMES[namespace] = [ + {**t, "requires_instantiation": False, "source": "existing"} + for t in templates + ] + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "mode": mode, + }) + + +@mcp.tool() +def list_serving_runtimes( + namespace: str, + include_templates: bool = False, +) -> str: + """List available ServingRuntimes in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + runtimes = list(SERVING_RUNTIMES.get(namespace, [])) + if include_templates: + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + existing_names = {r["name"] for r in runtimes} + for t in templates: + if t["name"] not in existing_names: + runtimes.append(t) + + return json.dumps(runtimes, indent=2) + + +# ── Inference service tools ────────────────────────────────────────────── + + +@mcp.tool() +def deploy_model( + name: str, + namespace: str, + runtime: str, + model_format: str, + storage_uri: str, + display_name: str = "", + min_replicas: int = 1, + max_replicas: int = 1, + cpu_request: str = "1", + cpu_limit: str = "2", + memory_request: str = "4Gi", + memory_limit: str = "8Gi", + gpu_count: int = 0, +) -> str: + """Deploy an AI/ML model as a KServe InferenceService.""" + if namespace not in PROJECTS: + raise ValueError( + f"Namespace '{namespace}' is not a Data Science Project. " + "Create one via create_data_science_project first." + ) + + ns_runtimes = SERVING_RUNTIMES.get(namespace, []) + runtime_names = [r["name"] for r in ns_runtimes] + if runtime not in runtime_names: + available = ", ".join(runtime_names) or "none" + raise ValueError( + f"ServingRuntime '{runtime}' not found in namespace '{namespace}'. " + f"Available runtimes: {available}" + ) + + endpoint = f"https://{name}-{namespace}.apps.ocp-cluster.example.com" + isvc = { + "name": name, + "namespace": namespace, + "runtime": runtime, + "model_format": model_format, + "storage_uri": storage_uri, + "display_name": display_name or name, + "gpu_count": gpu_count, + "cpu_request": cpu_request, + "memory_request": memory_request, + "min_replicas": min_replicas, + "max_replicas": max_replicas, + "ready": True, + "url": endpoint, + "conditions": [ + {"type": "Ready", "status": "True", "reason": "Ready", "message": ""}, + {"type": "PredictorReady", "status": "True", "reason": "PodReady", "message": ""}, + {"type": "IngressReady", "status": "True", "reason": "IngressReady", "message": ""}, + ], + "age": "0s", + } + + INFERENCE_SERVICES.setdefault(namespace, {})[name] = isvc + DEPLOYED_MODELS[f"{namespace}/{name}"] = isvc + + return json.dumps({ + "status": "deployed", + "name": name, + "namespace": namespace, + "runtime": runtime, + "endpoint": endpoint, + "ready": True, + }) + + +@mcp.tool() +def list_inference_services( + namespace: str, + verbosity: str = "standard", +) -> str: + """List deployed InferenceServices in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + isvcs = INFERENCE_SERVICES.get(namespace, {}) + results = [] + for isvc_name, isvc in isvcs.items(): + entry = { + "name": isvc["name"], + "runtime": isvc["runtime"], + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "age": isvc.get("age", ""), + } + if verbosity == "full": + entry["conditions"] = isvc.get("conditions", []) + entry["storage_uri"] = isvc.get("storage_uri", "") + entry["gpu_count"] = isvc.get("gpu_count", 0) + results.append(entry) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def get_inference_service( + name: str, + namespace: str, + verbosity: str = "standard", +) -> str: + """Get detailed status of a specific InferenceService.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + + isvc = isvcs[name] + result = { + "name": isvc["name"], + "namespace": isvc["namespace"], + "runtime": isvc["runtime"], + "model_format": isvc.get("model_format", ""), + "storage_uri": isvc.get("storage_uri", ""), + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "conditions": isvc.get("conditions", []), + "gpu_count": isvc.get("gpu_count", 0), + "replicas": {"min": isvc.get("min_replicas", 1), "max": isvc.get("max_replicas", 1)}, + "resources": { + "cpu_request": isvc.get("cpu_request", "1"), + "memory_request": isvc.get("memory_request", "4Gi"), + "memory_limit": isvc.get("memory_limit", "8Gi"), + }, + "age": isvc.get("age", ""), + } + return json.dumps(result, indent=2) + + +@mcp.tool() +def get_model_endpoint(name: str, namespace: str) -> str: + """Get the inference endpoint URL for a deployed model.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + isvc = isvcs[name] + if not isvc["ready"]: + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": "", + "error": "InferenceService is not ready. Check conditions for details.", + }) + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": isvc["url"], + }) + + +# ── Workbench tools ────────────────────────────────────────────────────── + + +@mcp.tool() +def list_workbenches(namespace: str) -> str: + """List workbenches (Jupyter notebooks) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + return json.dumps(wbs, indent=2) + + +@mcp.tool() +def create_workbench( + namespace: str, + name: str, + display_name: str = "", + image: str = "jupyter-datascience-ubi9-python-3.9-2024.1", + cpu_request: str = "1", + memory_request: str = "4Gi", + gpu_count: int = 0, + pvc_size: str = "20Gi", +) -> str: + """Create a new workbench (Jupyter notebook) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + valid_images = [img["name"] for img in NOTEBOOK_IMAGES] + if image not in valid_images: + raise ValueError( + f"Image '{image}' not found. Available: {', '.join(valid_images)}" + ) + + wb = { + "name": name, + "display_name": display_name or name, + "image": image, + "status": "Running", + "cpu_request": cpu_request, + "memory_request": memory_request, + "gpu_count": gpu_count, + "pvc_name": f"{name}-pvc", + "pvc_size": pvc_size, + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-03-02T12:00:00Z", + } + WORKBENCHES.setdefault(namespace, []).append(wb) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "image": image, + "pvc": f"{name}-pvc", + }) + + +@mcp.tool() +def stop_workbench(namespace: str, name: str) -> str: + """Stop a running workbench (preserves data).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Stopped" + return json.dumps({"status": "stopped", "name": name, "namespace": namespace}) + + +@mcp.tool() +def start_workbench(namespace: str, name: str) -> str: + """Start a stopped workbench.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Running" + return json.dumps({"status": "running", "name": name, "namespace": namespace}) + + +@mcp.tool() +def delete_workbench(namespace: str, name: str) -> str: + """Delete a workbench. WARNING: PVC data may be lost if not backed up.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wbs.remove(wb) + return json.dumps({ + "status": "deleted", + "name": name, + "namespace": namespace, + "warning": "Associated PVC data has been deleted", + }) + + +@mcp.tool() +def list_notebook_images() -> str: + """List available notebook images for workbench creation.""" + return json.dumps(NOTEBOOK_IMAGES, indent=2) + + +# ── Pipeline server tools ─────────────────────────────────────────────── + + +@mcp.tool() +def configure_pipeline_server( + namespace: str, + data_connection: str, + database: str = "MariaDB", +) -> str: + """Configure a pipeline server for a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + if not any(dc["name"] == data_connection for dc in dcs): + available = [dc["name"] for dc in dcs] + raise ValueError( + f"Data connection '{data_connection}' not found. Available: {available}" + ) + + PIPELINE_SERVERS[namespace] = { + "configured": True, + "data_connection": data_connection, + "status": "Ready", + "database": database, + } + PROJECTS[namespace]["pipeline_server"] = True + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "data_connection": data_connection, + "database": database, + }) + + +@mcp.tool() +def get_pipeline_server_status(namespace: str) -> str: + """Get the status of the pipeline server in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + ps = PIPELINE_SERVERS.get(namespace) + if not ps: + return json.dumps({"namespace": namespace, "configured": False}) + return json.dumps({ + "namespace": namespace, + "configured": ps["configured"], + "data_connection": ps["data_connection"], + "status": ps["status"], + "database": ps["database"], + }) + + +# ── Serving runtime creation ──────────────────────────────────────────── + + +@mcp.tool() +def create_serving_runtime( + namespace: str, + name: str, + display_name: str = "", + model_formats: list = None, + container_image: str = "", + container_port: int = 8080, + multi_model: bool = False, + api_protocol: str = "REST", +) -> str: + """Create a custom ServingRuntime in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + if not model_formats: + raise ValueError("model_formats must specify at least one model format") + + runtime = { + "name": name, + "display_name": display_name or name, + "model_formats": model_formats, + "requires_instantiation": False, + "source": "custom", + "api_protocol": api_protocol, + "container_image": container_image, + "container_port": container_port, + "multi_model": multi_model, + } + SERVING_RUNTIMES.setdefault(namespace, []).append(runtime) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "model_formats": model_formats, + }) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/nim-setup/SKILL.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/nim-setup/SKILL.md new file mode 100644 index 00000000..34df2121 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/nim-setup/SKILL.md @@ -0,0 +1,370 @@ +--- +name: nim-setup +description: | + Configure NVIDIA NIM platform on OpenShift AI for optimized model inference. + + Use when: + - "Set up NIM on my cluster" + - "Configure NGC credentials for NIM" + - "I want to deploy a NIM model but haven't set up the platform" + - "Create the NIM Account CR" + + One-time prerequisite before deploying models with NVIDIA NIM runtime via /model-deploy. + + NOT for deploying models (use /model-deploy instead). + NOT for vLLM or Caikit deployments (NIM-specific only). +model: inherit +color: blue +--- + +# /nim-setup Skill + +Configure the NVIDIA NIM platform on OpenShift AI. This is a one-time setup that creates NGC credentials and the NIM Account custom resource, enabling NIM-based model deployments via `/model-deploy`. + +## Prerequisites + +**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools** (from openshift): +- `resources_get` - Check operator installations and existing resources +- `resources_list` - List resources in a namespace +- `resources_create_or_update` - Create secrets, Account CR, ConfigMap +- `events_list` - Check events for errors during setup + +**Optional MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) + +**Optional MCP Tools** (from rhoai): +- `list_data_science_projects` - Validate namespace is an RHOAI Data Science Project +- `list_serving_runtimes` - Verify NIM ServingRuntimes after setup + +**Optional MCP Server**: `ai-observability` (for `get_gpu_info` to verify GPU availability) + +**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). + +**Required User Input**: +- NGC API key (from https://ngc.nvidia.com) +- Target namespace for NIM resources + +**Additional cluster requirements**: +- OpenShift cluster >= 4.14 +- NVIDIA GPU Operator installed +- Node Feature Discovery (NFD) Operator installed +- ServiceAccount with RBAC permissions to create Secrets, Accounts, and ConfigMaps + +## When to Use This Skill + +**Use this skill when you need to:** +- Set up NVIDIA NIM platform on OpenShift AI for the first time +- Create or refresh NGC credentials (image pull secret + API key secret) +- Create the NIM Account custom resource +- Verify GPU Operator and NFD Operator are installed and healthy + +**Do NOT use this skill when:** +- You want to deploy a model (use `/model-deploy` after NIM setup is complete) +- You want to deploy with vLLM or Caikit+TGIS (NIM-specific only, use `/model-deploy` directly) +- You need to create a custom ServingRuntime (use `/serving-runtime-config`) + +## Workflow + +### Step 0: Validate Target Namespace (Optional) + +If the `rhoai` MCP server is available, validate that the target namespace is an RHOAI Data Science Project: + +**MCP Tool**: `list_data_science_projects` (from rhoai) + +If the namespace is not in the project list, warn: "Namespace `[namespace]` is not a Data Science Project. NIM setup may not work correctly. Consider creating a Data Science Project first." + +If `rhoai` MCP is not available, skip this check and proceed. + +### Step 1: Verify GPU Operator and Node Feature Discovery + +**Document Consultation** (read before verifying operators): +1. **Action**: Read [supported-runtimes.md](../../docs/references/supported-runtimes.md) using the Read tool to understand NIM platform requirements +2. **Output to user**: "I consulted [supported-runtimes.md](../../docs/references/supported-runtimes.md) to understand NIM platform requirements." + +Check that the NVIDIA GPU Operator and NFD Operator are installed and healthy. + +**MCP Tool**: `resources_get` (from openshift) + +**Parameters**: +- `apiVersion`: `"operators.coreos.com/v1alpha1"` - REQUIRED +- `kind`: `"ClusterServiceVersion"` - REQUIRED +- `namespace`: `"nvidia-gpu-operator"` - REQUIRED (namespace where GPU Operator CSV is installed) +- `name`: the CSV name matching `"gpu-operator-certified"` prefix + +**Expected Output**: ClusterServiceVersion object with `status.phase: "Succeeded"` + +Repeat for NFD Operator: +- `namespace`: `"openshift-nfd"` +- `name`: the CSV name matching `"nfd"` prefix + +**Error Handling**: +- If GPU Operator CSV not found -> Report to user: "NVIDIA GPU Operator is not installed. Install it from OperatorHub before proceeding." +- If NFD Operator CSV not found -> Report to user: "Node Feature Discovery Operator is not installed. Install it from OperatorHub before proceeding." +- If `status.phase` != `"Succeeded"` -> Report current phase and suggest troubleshooting +- Offer to skip this check if user confirms operators are installed via another method + +### Step 2: Collect NGC Credentials from User + +Ask the user for their NGC API key. This key is used for two purposes: +1. Pulling NIM container images from `nvcr.io` (image pull secret) +2. Authenticating NIM API calls at runtime (API key secret) + +**Ask the user**: +``` +To set up NIM, I need your NVIDIA NGC API key. + +You can generate one at: https://ngc.nvidia.com/setup/api-key + +Please provide: +1. Your NGC API key +2. The target namespace for NIM resources (e.g., "my-ai-project") +``` + +Proceed automatically without waiting for user input. + +**SECURITY**: Store the key in memory only for the duration of this skill. Never echo or display the actual key value in output. + +### Step 3: Create NGC Image Pull Secret + +Generate and display the docker-registry Secret YAML for pulling NIM images from `nvcr.io`. + +**Show the user the Secret manifest** (with API key value redacted): + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: ngc-image-pull-secret + namespace: [namespace] +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: [base64-encoded docker config for nvcr.io] +``` + +Note: The `.dockerconfigjson` contains: +- Registry: `nvcr.io` +- Username: `$oauthtoken` +- Password: `[NGC API key - REDACTED in display]` + +**Ask**: "Should I create this image pull secret in namespace `[namespace]`? (yes/no)" + +Proceed automatically without waiting for user input. + +**MCP Tool**: `resources_create_or_update` (from openshift) + +**Parameters**: +- `manifest`: full Secret manifest as JSON string - REQUIRED + - The JSON must include apiVersion, kind, metadata (name, namespace), type, and data fields +- `namespace`: user-specified namespace - REQUIRED + - Example: `"my-ai-project"` + +**Expected Output**: Created Secret object with `metadata.uid` + +**Error Handling**: +- If secret already exists -> Ask user: "Secret `ngc-image-pull-secret` already exists. Should I update it? (yes/no)" +- If namespace not found -> Report error, suggest creating namespace first +- If RBAC error -> Report insufficient permissions + +### Step 4: Create NGC API Key Secret + +Generate and display the generic Secret YAML for the NGC API key used at runtime. + +**Show the user the Secret manifest** (with API key value redacted): + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: ngc-api-key + namespace: [namespace] +type: Opaque +stringData: + NGC_API_KEY: "[REDACTED]" +``` + +**Ask**: "Should I create this API key secret in namespace `[namespace]`? (yes/no)" + +Proceed automatically without waiting for user input. + +**MCP Tool**: `resources_create_or_update` (from openshift) + +**Parameters**: +- `manifest`: full Secret manifest as JSON string - REQUIRED +- `namespace`: user-specified namespace - REQUIRED + +**Expected Output**: Created Secret object with `metadata.uid` + +**Error Handling**: +- If secret already exists -> Ask user if they want to update it +- If RBAC error -> Report insufficient permissions + +### Step 5: Create NIM Account CR + +Generate and display the NIM Account custom resource that manages the NIM platform lifecycle. + +**Show the user the Account CR manifest:** + +```yaml +apiVersion: nim.opendatahub.io/v1 +kind: Account +metadata: + name: nim-account + namespace: [namespace] +spec: + apiKeySecret: + name: ngc-api-key + imagePullSecret: + name: ngc-image-pull-secret +``` + +**Ask**: "Should I create this NIM Account CR in namespace `[namespace]`? (yes/no)" + +Proceed automatically without waiting for user input. + +**MCP Tool**: `resources_create_or_update` (from openshift) + +**Parameters**: +- `manifest`: full Account CR manifest as JSON string - REQUIRED +- `namespace`: user-specified namespace - REQUIRED + +**Expected Output**: Created Account object with `metadata.uid` + +**Error Handling**: +- If Account CR already exists -> Report current status, ask if user wants to update +- If CRD not found (`nim.opendatahub.io/v1` Account) -> Report: "NIM CRD not available. Ensure Red Hat OpenShift AI operator is installed and includes NIM support." +- If RBAC error -> Report insufficient permissions + +### Step 6: (Optional) Configure NIM Model Catalog + +**Ask**: "Would you like to customize which NIM models appear in the catalog? (yes/no, default: no)" + +If user says **no** -> Skip to Step 7 (default catalog is used). + +If user says **yes**: + +**Show the user the ConfigMap template:** + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: nim-model-catalog + namespace: [namespace] +data: + model-catalog.json: | + [ + { + "name": "[model-name]", + "displayName": "[display-name]", + "shortDescription": "[description]" + } + ] +``` + +Ask user which models to include, generate the ConfigMap, and confirm before creating. + +**MCP Tool**: `resources_create_or_update` (from openshift) + +### Step 7: Validate NIM Platform Readiness + +Check that the NIM platform is ready for model deployments. + +**Step 7a: Check Account CR Status** + +**MCP Tool**: `resources_get` (from openshift) + +**Parameters**: +- `apiVersion`: `"nim.opendatahub.io/v1"` - REQUIRED +- `kind`: `"Account"` - REQUIRED +- `namespace`: user-specified namespace - REQUIRED +- `name`: `"nim-account"` - REQUIRED + +**Expected Output**: Account object with `status.conditions` showing ready state + +**Step 7b: Verify NIM ServingRuntimes** + +**MCP Tool**: `list_serving_runtimes` (from rhoai) - preferred if rhoai MCP available + +**Parameters**: +- `namespace`: user-specified namespace - REQUIRED +- `include_templates`: `false` + +**Fallback MCP Tool**: `resources_list` (from openshift) +- `apiVersion`: `"serving.kserve.io/v1alpha1"`, `kind`: `"ServingRuntime"`, `namespace`: user-specified namespace + +**Expected Output**: List of ServingRuntime objects including NIM runtimes + +**Step 7c: (Optional) GPU Inventory Check** + +If `ai-observability` MCP server is available, use `get_gpu_info` to report cluster GPU inventory. + +**Report results** showing: Account CR status, credentials status (created/existing), available NIM ServingRuntimes, GPU inventory (if available), and next steps (`/model-deploy`). + +**On failure**: Report Account CR status details and error message. Suggest troubleshooting steps: check Account CR events, verify NGC API key validity, check OpenShift AI operator logs. Ask if user wants help troubleshooting. + +## Common Issues + +For common issues (GPU scheduling, OOMKilled, image pull errors, RBAC), see [common-issues.md](../references/common-issues.md). + +### Issue 1: Account CR Stuck in "Pending" + +**Error**: Account CR `status.conditions` shows pending state indefinitely + +**Cause**: NGC credentials are invalid, expired, or the RHOAI operator cannot reach NGC services. + +**Solution:** +1. Verify NGC API key is valid by testing at https://ngc.nvidia.com +2. Check Account CR events: use `events_list` filtered by namespace to find events related to the Account resource +3. Regenerate NGC API key and re-run `/nim-setup` with new credentials + +### Issue 2: GPU Operator Not Installed + +**Error**: ClusterServiceVersion for `gpu-operator-certified` not found + +**Cause**: NVIDIA GPU Operator was not installed from OperatorHub. + +**Solution:** +1. Install NVIDIA GPU Operator from OperatorHub in the OpenShift console +2. Wait for the operator to reach `Succeeded` phase +3. Verify GPU nodes are detected: check for `nvidia.com/gpu` resources on nodes +4. Re-run `/nim-setup` + +### Issue 3: NIM ServingRuntimes Not Appearing + +**Error**: `resources_list` for ServingRuntimes returns no NIM runtimes + +**Cause**: Account CR is not yet ready, or the RHOAI operator version does not include NIM support. + +**Solution:** +1. Check Account CR status — runtimes are created asynchronously after the Account becomes ready +2. Wait 2-3 minutes and re-check +3. Verify RHOAI operator version supports NIM integration +4. Check operator logs for errors + +## Dependencies + +### MCP Tools +See [Prerequisites](#prerequisites) for the complete list of required and optional MCP tools. + +### Related Skills +- `/model-deploy` - Deploy a model using NIM runtime after setup is complete +- `/serving-runtime-config` - Configure custom serving runtimes if NIM doesn't fit + +### Reference Documentation +- [supported-runtimes.md](../../docs/references/supported-runtimes.md) - NIM runtime capabilities and requirements +- [live-doc-lookup.md](../references/live-doc-lookup.md) - Protocol for fetching current RHOAI/NIM documentation + +## Critical: Human-in-the-Loop Requirements + +See [skill-conventions.md](../references/skill-conventions.md) for general HITL and security conventions. + +**Skill-specific checkpoints:** +- Before creating each Secret: display manifest (credentials REDACTED), confirm +- Before creating Account CR: display manifest, confirm +- Before creating ConfigMap (if applicable): display manifest, confirm +- **NEVER** display actual NGC API key values in output + +## Example Usage + +See [nim-setup examples](../../docs/examples/nim-setup.md) for a complete first-time NIM setup walkthrough. diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/common-issues.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/instruction.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/instruction.md new file mode 100644 index 00000000..f0b5fa2c --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/instruction.md @@ -0,0 +1,17 @@ +# NVIDIA NIM Setup Task + +You are an AI engineer on Red Hat OpenShift AI. Your team wants to deploy NVIDIA NIM for GPU-accelerated inference, but the cluster has not been set up for it yet. + +## Scenario +The ML team needs to serve models using NVIDIA's inference microservices. The cluster has GPUs available, but the necessary platform components and credentials have not been configured. You need to assess readiness and produce a complete setup plan. + +## Requirements +- Verify operator prerequisites (GPU Operator and NFD Operator) by checking their ClusterServiceVersion status +- Assess the current cluster state to determine what NIM infrastructure is already in place and what is missing +- Document the complete setup procedure including: the exact Kubernetes Secret manifests (with types, data key names, and structure) needed for NGC authentication, and the NIM Account custom resource with its correct API group and spec fields +- Provide the YAML manifests for each resource that needs to be created, using the correct RHOAI-specific API versions and resource naming conventions +- Flag any potential issues or blockers discovered during your assessment + +Document your assessment and setup plan in `/root/report.md`. + +Use MCP tools to interact with the platform. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/solution/solve.sh b/evaluation/with_skills/rh-ai-engineer__nim-setup/solution/solve.sh new file mode 100644 index 00000000..accbf7fe --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/solution/solve.sh @@ -0,0 +1,28 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# NIM Setup Plan + +## Prerequisites +- GPU Operator CSV in nvidia-gpu-operator namespace (gpu-operator-certified) +- NFD (Node Feature Discovery) in openshift-nfd + +## NGC Secrets +- API key secret: ngc-api-key (NGC_API_KEY) +- Image pull secret: ngc-image-pull-secret + - Registry: nvcr.io + - Username: $oauthtoken + - Password: NGC API key + +## NIM Account CR (nim.opendatahub.io/v1) +```yaml +apiVersion: nim.opendatahub.io/v1 +kind: Account +metadata: + name: nim-account +spec: + apiKeySecret: + name: ngc-api-key + imagePullSecret: + name: ngc-image-pull-secret +``` +REPORT_EOF diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/task.toml b/evaluation/with_skills/rh-ai-engineer__nim-setup/task.toml new file mode 100644 index 00000000..7b53288a --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-ai-engineer__nim-setup" +name = "rh-ai-engineer NVIDIA NIM Setup Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-ai-engineer", "nim-setup", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/tests/llm_judge.py b/evaluation/with_skills/rh-ai-engineer__nim-setup/tests/llm_judge.py new file mode 100644 index 00000000..a3c29b06 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "opendatahub_api_group", + "file": "/root/report.md", + "question": "Does the report use nim.opendatahub.io as the API group for the NIM Account custom resource, rather than the upstream nim.nvidia.com?", + "reference": "A skilled report specifies apiVersion: nim.opendatahub.io/v1 for the Account CR, which is the RHOAI-specific API group. An unskilled report uses nim.nvidia.com/v1alpha1 (the upstream NVIDIA API group) which is incorrect for Red Hat OpenShift AI." + }, + { + "id": "secret_naming_and_types", + "file": "/root/report.md", + "question": "Does the report create an image pull secret named ngc-image-pull-secret with type kubernetes.io/dockerconfigjson, and an API key secret with stringData containing the NGC_API_KEY field?", + "reference": "A skilled report creates ngc-image-pull-secret (type: kubernetes.io/dockerconfigjson) for nvcr.io registry access, and ngc-api-key (type: Opaque, stringData: NGC_API_KEY) for runtime auth. An unskilled report uses generic names like nvcr-credentials, kubectl shorthands without explicit types, or data.api_key instead of stringData.NGC_API_KEY." + }, + { + "id": "operator_csv_verification", + "file": "/root/report.md", + "question": "Does the report verify gpu-operator-certified and NFD (Node Feature Discovery) Operator as prerequisites, checking their ClusterServiceVersion status?", + "reference": "A skilled report checks for gpu-operator-certified (the specific CSV name, not just 'gpu-operator') and the NFD Operator in openshift-nfd namespace. An unskilled report either skips NFD entirely or uses generic gpu-operator references without the certified CSV name." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/tests/test.sh b/evaluation/with_skills/rh-ai-engineer__nim-setup/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/tests/test_outputs.py b/evaluation/with_skills/rh-ai-engineer__nim-setup/tests/test_outputs.py new file mode 100644 index 00000000..ad1f22ef --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__nim-setup/tests/test_outputs.py @@ -0,0 +1,89 @@ +""" +Tests for rh-ai-engineer__nim-setup per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert "nim" in content, "report should mention NIM" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 200, "report should have substantial content" + + +class TestSkillDependent: + def test_opendatahub_nim_api(self): + """Skill teaches nim.opendatahub.io as the RHOAI API group for NIM Account CR. + Without skill, agents use upstream nim.nvidia.com API group.""" + c = read_report() + assert "nim.opendatahub.io" in c, ( + "should use nim.opendatahub.io as the NIM Account CR API group (not nim.nvidia.com)" + ) + + def test_ngc_image_pull_secret_name(self): + """Skill teaches ngc-image-pull-secret as the specific secret name for nvcr.io. + Without skill, agents use generic names like nvcr-credentials.""" + c = read_report() + assert "ngc-image-pull-secret" in c, ( + "should use ngc-image-pull-secret as the image pull secret name" + ) + + def test_dockerconfigjson_secret_type(self): + """Skill teaches kubernetes.io/dockerconfigjson as the secret type for image pull. + Without skill, agents use kubectl docker-registry shorthand without explicit type.""" + c = read_report().lower() + assert "dockerconfigjson" in c, ( + "should specify dockerconfigjson as the image pull secret type" + ) + + def test_gpu_operator_certified_csv(self): + """Skill teaches checking gpu-operator-certified CSV by name. + Without skill, agents check generically for gpu-operator.""" + c = read_report().lower() + assert "gpu-operator-certified" in c, ( + "should verify gpu-operator-certified ClusterServiceVersion by name" + ) + + def test_nfd_operator_reference(self): + """Skill teaches verifying NFD (Node Feature Discovery) Operator as a prerequisite. + Without skill, agents skip NFD verification entirely.""" + c = read_report().lower() + assert "nfd" in c, ( + "should verify NFD (Node Feature Discovery) Operator as a prerequisite" + ) + + def test_stringdata_secret_field(self): + """Skill teaches using stringData in Secret YAML for NGC API key (no base64 needed). + Without skill, agents use kubectl --from-literal or data with base64.""" + c = read_report() + assert "stringData" in c or "stringdata" in c.lower(), ( + "should use stringData field in Secret YAML manifest for API key" + ) + + def test_nvidia_gpu_only(self): + """Docs emphasize NIM requires NVIDIA GPUs only; fallback to vLLM when + NVIDIA GPUs unavailable. Without docs, agents don't mention this constraint.""" + c = read_report().lower() + assert any(t in c for t in [ + "nvidia gpu", "nvidia only", "fallback", "vllm", + ]) and ("nim" in c or "gpu" in c), ( + "should note NIM requires NVIDIA GPUs with vLLM fallback" + ) diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/Dockerfile b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/Dockerfile new file mode 100644 index 00000000..d4978abe --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/Dockerfile @@ -0,0 +1,74 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "rhoai": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-rhoai-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/common-issues.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/model-deploy.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/model-deploy.md new file mode 100644 index 00000000..dcd79e44 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/model-deploy.md @@ -0,0 +1,160 @@ +# /model-deploy Examples + +Example interactions demonstrating the model-deploy skill workflow. + +## Example 1: Deploy Llama 3.1 8B with vLLM + +``` +User: "Deploy Llama 3.1 8B on my cluster" + +Agent: [Invokes /model-deploy skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand runtime capabilities. + +Step 1 of 10: Gathering deployment information. + +| Setting | Value | Source | +|---------|-------|--------| +| Model | Llama 3.1 8B Instruct | user input | +| Runtime | TBD | auto-detect | +| Namespace | ? | needs input | +| Model Source | ? | needs input | +| Deployment Mode | Serverless | default | + +What namespace should I deploy to, and where are the model weights stored? +(e.g., S3 bucket, OCI registry, PVC) + +User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" + +Agent: Step 2 of 10: Determining runtime. + +I consulted supported-runtimes.md to understand runtime capabilities. + +Based on Llama 3.1 8B (open-source HuggingFace model): + +Recommended runtime: vLLM +Reason: Default runtime for open-source LLMs. Broad compatibility, + OpenAI-compatible API, no additional setup required. + +Proceed with vLLM? (yes / no / suggest alternative) + +User: "yes" + +Agent: Step 3 of 10: Looking up hardware profile. + +I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. + +| Requirement | Value | +|-------------|-------| +| GPUs | 1x A10G/L4/A100/H100 | +| VRAM | 16GB | +| Key Args | --max-model-len=4096 --tool-call-parser hermes | + +Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). +Ensure your cluster has at least 1 GPU with 16GB VRAM. + +Step 5 of 10: Skipped (NIM platform check not needed for vLLM). + +Step 6-7 of 10: Generating InferenceService YAML. + +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: llama-3-1-8b + namespace: ai-models + annotations: + serving.kserve.io/deploymentMode: Serverless +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: vllm-runtime + storageUri: s3://my-bucket/llama-3.1-8b + resources: + limits: + nvidia.com/gpu: "1" + requests: + cpu: "4" + memory: "16Gi" + args: + - --max-model-len=4096 + - --tool-call-parser=hermes + +| Setting | Value | +|---------|-------| +| Model | llama-3-1-8b | +| Runtime | vllm-runtime | +| GPUs | 1 | +| Namespace | ai-models | +| Mode | Serverless | + +Proceed with creating this InferenceService? (yes/no/modify) + +User: "yes" + +Agent: Step 8 of 10: Creating InferenceService... + [Creates via resources_create_or_update] + + InferenceService created. Monitoring rollout... + +Step 9 of 10: Monitoring rollout. + +| Stage | Status | +|-------|--------| +| Pod Scheduled | Complete | +| Image Pulled | Complete | +| Container Started | Complete | +| Model Loaded | Complete | +| Ready | Complete | + +Step 10 of 10: Deployment complete! + +Model Deployment Successful + +Model: llama-3-1-8b +Runtime: vLLM +Namespace: ai-models +GPUs: 1 + +Inference Endpoint: + URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com + API: OpenAI-compatible REST + +Quick Test: + curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' + +Next Steps: + Monitor performance: "/ai-observability llama-3-1-8b" +``` + +## Example 2: Deploy a Model with NIM (NIM Not Set Up) + +``` +User: "Deploy Llama 3.1 70B with NIM" + +Agent: [Invokes /model-deploy skill] + ... + Step 2: User requested NIM runtime. + + Step 5: Checking NIM platform... + +NIM platform is not set up in namespace ai-models. + +The NIM runtime requires a one-time platform setup. Would you like to: +1. Run /nim-setup now to configure the NIM platform +2. Switch to vLLM runtime instead (no setup required) +3. Abort deployment + +User: "1" + +Agent: [Invokes /nim-setup skill — walks through NGC credential setup] + ... + NIM Platform Setup Complete. + + Resuming model deployment... + [Continues from Step 6 with NIM runtime] +``` diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/nim-setup.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/nim-setup.md new file mode 100644 index 00000000..ccfc1a5d --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/nim-setup.md @@ -0,0 +1,115 @@ +# /nim-setup Examples + +Example interactions demonstrating the nim-setup skill workflow. + +## Example 1: First-Time NIM Setup + +``` +User: "Set up NIM on my cluster" + +Agent: [Invokes /nim-setup skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand NIM platform requirements. + +Let me verify the prerequisites for NIM setup. + +Step 1 of 7: Checking GPU Operator and NFD Operator... + +GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) +NFD Operator: nfd.v4.19.0 (Succeeded) + +Both operators are installed and healthy. + +Step 2 of 7: I need your NGC credentials. + +To set up NIM, I need your NVIDIA NGC API key. +You can generate one at: https://ngc.nvidia.com/setup/api-key + +Please provide: +1. Your NGC API key +2. The target namespace for NIM resources + +User: "Here's my key: [user provides NGC API key], namespace: ai-models" + +Agent: Step 3 of 7: Creating NGC image pull secret... + +I'll create a docker-registry secret for pulling NIM images from nvcr.io: + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-image-pull-secret + namespace: ai-models + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] + +Should I create this image pull secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + Image pull secret created successfully. + +Step 4 of 7: Creating NGC API key secret... + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-api-key + namespace: ai-models + type: Opaque + stringData: + NGC_API_KEY: "[REDACTED]" + +Should I create this API key secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + API key secret created successfully. + +Step 5 of 7: Creating NIM Account CR... + + apiVersion: nim.opendatahub.io/v1 + kind: Account + metadata: + name: nim-account + namespace: ai-models + spec: + apiKeySecret: + name: ngc-api-key + imagePullSecret: + name: ngc-image-pull-secret + +Should I create this NIM Account CR in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates Account CR via resources_create_or_update] + Account CR created successfully. + +Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) + +User: "no" + +Agent: Step 7 of 7: Validating NIM platform readiness... + +NIM Platform Setup Complete + +Account CR: `nim-account` (namespace: ai-models) + Status: Ready + +NGC Credentials: + Image Pull Secret: ngc-image-pull-secret [created] + API Key Secret: ngc-api-key [created] + +NIM ServingRuntimes Available: + - nim-llama-3-1-8b-instruct + - nim-llama-3-1-70b-instruct + - nim-mistral-7b-instruct + +Next Steps: + To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" +``` diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/known-model-profiles.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/known-model-profiles.md new file mode 100644 index 00000000..40148981 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/known-model-profiles.md @@ -0,0 +1,83 @@ +--- +title: Known Model Hardware Profiles +category: references +tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] +semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] +use_cases: [model-deploy, debug-inference, ai-observability] +last_updated: 2026-03-10 +--- + +# Known Model Hardware Profiles + +Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). + +**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. + +## Model Source Conventions + +Each model lists a recommended `storageUri` with its authentication requirements: +- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** +- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). +- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. + +When the user does not specify a model source, use the `hf://` URI listed in the profile below. + +## Llama 3.x (Meta) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | +| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | +| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | +| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | + +- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) +- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly + +## Granite 3.x (IBM/Red Hat) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | +| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | + +- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) +- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser granite --chat-template` +- Red Hat-supported model family on RHOAI + +## Mixtral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | +| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) +- Mixture-of-Experts architecture: only ~13B/45B parameters active per token + +## Mistral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | +| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) + +## When a Model Is Not Listed + +If the requested model is not in this file, the agent MUST use the live doc lookup protocol: + +1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure +2. Fetch hardware specs from the appropriate source +3. Report findings to the user before proceeding with deployment + +Common cases requiring live lookup: +- Newly released models (after this file's last update) +- Domain-specific fine-tuned models +- Models with custom quantization +- NIM-specific optimized profiles diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/supported-runtimes.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/supported-runtimes.md new file mode 100644 index 00000000..9d018651 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/supported-runtimes.md @@ -0,0 +1,104 @@ +--- +title: Supported Serving Runtimes +category: references +tags: [runtimes, vllm, nim, caikit, tgis, serving] +semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] +use_cases: [model-deploy, serving-runtime-config, nim-setup] +last_updated: 2026-02-26 +--- + +# Supported Serving Runtimes + +This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. + +## Runtime Comparison + +| Runtime | API Type | Model Format | GPU Support | License | Setup Required | +|---------|----------|-------------|-------------|---------|----------------| +| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | +| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | +| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | +| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | + +## vLLM + +**Default runtime for most open-source models.** + +- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) +- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized +- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) +- **Model source**: S3-compatible storage, OCI registry, PVC, URI +- **Key features**: + - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) + - PagedAttention for efficient memory management + - Tensor parallelism for multi-GPU inference + - Tool/function calling support (`--tool-call-parser`) + - Continuous batching for high throughput +- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name + +## NVIDIA NIM + +**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** + +- **API**: OpenAI-compatible REST +- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog +- **GPU support**: NVIDIA only (requires specific GPU models per model profile) +- **Model source**: NGC model catalog (pulled automatically via NGC credentials) +- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) +- **Key features**: + - TensorRT-LLM optimization for lower latency + - Pre-compiled model engines (no compilation on first load) + - Optimized and generic profiles per GPU type + - Automatic model download from NGC +- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. +- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM +- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. +- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state + +## Caikit+TGIS + +**Red Hat's Caikit format with Text Generation Inference Server.** + +- **API**: gRPC (not REST) +- **Model formats**: Caikit format (requires conversion from HuggingFace) +- **GPU support**: NVIDIA +- **Model source**: S3-compatible storage +- **Key features**: + - Red Hat-supported runtime + - gRPC API for streaming inference + - Integrated with RHOAI model serving platform +- **When to choose**: When the model is already in Caikit format or when gRPC API is required. +- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name + +## Custom Runtimes + +**User-provided ServingRuntime CRs for unsupported frameworks.** + +- **API**: Defined by the custom runtime +- **Model formats**: Defined by the custom runtime +- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. +- **How to create**: Use `/serving-runtime-config` skill +- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility + +## Runtime Selection Decision Tree + +``` +Is the user's preferred runtime explicitly stated? +├── Yes → Use that runtime +└── No → Continue + +Is the model available in the NGC NIM catalog? +├── Yes → Suggest NIM (with vLLM as fallback) +│ Note: Requires /nim-setup and NVIDIA GPUs +└── No → Continue + +Is the model in Caikit format? +├── Yes → Caikit+TGIS +└── No → Continue + +Is the model a standard open-source LLM (HuggingFace-compatible)? +├── Yes → vLLM (default) +└── No → Custom runtime via /serving-runtime-config +``` diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..cad5f77b --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,529 @@ +#!/usr/bin/env python3 +"""Mock OpenShift MCP server for SkillsBench rh-ai-engineer task. + +Simulates Kubernetes resource CRUD, pod management, logs, and events. + +Key scenario elements: +- LimitRange in namespaces: min CPU=100m, min memory=128Mi + (conflicts with KServe sidecar containers hardcoded at 10m CPU/15Mi memory) +- GPU node with custom taint ai-workload=true:NoSchedule +- NIM Account CR in ml-production: not ready (NGC credentials invalid) +- text-gen-legacy pods: OOMKilled (max-model-len=32768 on A10G) +- nim-llama-prod: no pods created (Account CR not ready) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + +# ── Cluster state ──────────────────────────────────────────────────────── + +GPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "gpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + "nvidia.com/gpu.present": "true", + "nvidia.com/gpu.product": "NVIDIA-A10G", + }, + }, + "spec": { + "taints": [ + { + "key": "ai-workload", + "value": "true", + "effect": "NoSchedule", + }, + ], + }, + "status": { + "allocatable": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "capacity": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "conditions": [ + {"type": "Ready", "status": "True"}, + ], + }, +} + +CPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "cpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + }, + }, + "spec": {"taints": []}, + "status": { + "allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "capacity": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +MASTER_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "master-1", + "labels": { + "node-role.kubernetes.io/master": "", + "node-role.kubernetes.io/control-plane": "", + }, + }, + "spec": { + "taints": [ + {"key": "node-role.kubernetes.io/master", "effect": "NoSchedule"}, + ], + }, + "status": { + "allocatable": {"cpu": "8", "memory": "32Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +ALL_NODES = [GPU_NODE, CPU_NODE, MASTER_NODE] + +# LimitRange applied by cluster policy to all DS project namespaces +NAMESPACE_LIMITRANGE = { + "apiVersion": "v1", + "kind": "LimitRange", + "metadata": { + "name": "default-limits", + }, + "spec": { + "limits": [ + { + "type": "Container", + "default": { + "cpu": "2", + "memory": "4Gi", + }, + "defaultRequest": { + "cpu": "500m", + "memory": "256Mi", + }, + "min": { + "cpu": "100m", + "memory": "128Mi", + }, + "max": { + "cpu": "32", + "memory": "128Gi", + }, + }, + ], + }, +} + +NIM_ACCOUNT_CR = { + "apiVersion": "nim.opendatahub.io/v1", + "kind": "Account", + "metadata": { + "name": "nim-account", + "namespace": "ml-production", + }, + "spec": { + "apiKeySecret": { + "name": "ngc-api-key", + }, + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "NGCCredentialsInvalid", + "message": "NGC API key validation failed: 401 Unauthorized. " + "The API key in secret 'ngc-api-key' is expired or invalid. " + "Re-create the secret with a valid NGC API key from " + "https://ngc.nvidia.com/setup/api-key and restart the " + "Account reconciliation.", + "lastTransitionTime": "2026-03-14T12:00:00Z", + }, + ], + "nimPullSecretStatus": "Failed", + "nimConfigStatus": "Pending", + }, +} + +SERVING_RUNTIME_VLLM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "vllm-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "vLLM", "version": "1", "autoSelect": True}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "quay.io/modh/vllm:rhoai-2.16", + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + }, + ], + }, +} + +SERVING_RUNTIME_NIM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "nim-serving-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "NIM", "version": "1"}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "nvcr.io/nim/meta/llama-3.1-8b-instruct:latest", + "ports": [{"containerPort": 8000, "protocol": "TCP"}], + "env": [ + {"name": "NGC_API_KEY", "valueFrom": { + "secretKeyRef": {"name": "ngc-api-key", "key": "api_key"}, + }}, + ], + }, + ], + }, +} + +PODS_BY_NAMESPACE = { + "ml-production": [ + { + "name": "text-gen-legacy-predictor-00001-abc12", + "namespace": "ml-production", + "status": "CrashLoopBackOff", + "restarts": 5, + "node": "gpu-worker-1", + "containers": [ + { + "name": "kserve-container", + "state": "waiting", + "reason": "CrashLoopBackOff", + "last_termination_reason": "OOMKilled", + "last_termination_exit_code": 137, + }, + ], + "labels": { + "serving.kserve.io/inferenceservice": "text-gen-legacy", + }, + "gpu": "1", + }, + # nim-llama-prod: NO pods created (Account CR not ready) + ], +} + +POD_LOGS = { + "text-gen-legacy-predictor-00001-abc12": ( + "INFO 2026-03-01 10:00:00 vllm_engine.py:125] vLLM engine starting...\n" + "INFO 2026-03-01 10:00:01 config.py:89] Model: mistralai/Mistral-7B-Instruct-v0.3\n" + "INFO 2026-03-01 10:00:01 config.py:92] max_model_len = 32768\n" + "INFO 2026-03-01 10:00:02 gpu_executor.py:45] GPU 0: NVIDIA A10G (24576 MiB)\n" + "INFO 2026-03-01 10:00:03 model_runner.py:88] Loading model weights...\n" + "INFO 2026-03-01 10:00:15 model_runner.py:112] Model weights loaded: 13.5 GiB\n" + "INFO 2026-03-01 10:00:15 worker.py:201] Allocating KV cache...\n" + "ERROR 2026-03-01 10:00:16 worker.py:215] torch.cuda.OutOfMemoryError: " + "CUDA out of memory. Tried to allocate 28.5 GiB for KV cache but only " + "10.1 GiB available after loading model weights (13.5 GiB).\n" + "ERROR 2026-03-01 10:00:16 vllm_engine.py:178] Engine failed to start\n" + "Traceback (most recent call last):\n" + " File \"/opt/vllm/vllm/engine/engine.py\", line 175, in start\n" + " self._init_kv_cache()\n" + " File \"/opt/vllm/vllm/worker/worker.py\", line 215, in _init_kv_cache\n" + " raise torch.cuda.OutOfMemoryError(msg)\n" + "torch.cuda.OutOfMemoryError: CUDA out of memory\n" + ), +} + +EVENTS_BY_NAMESPACE = { + "ml-production": [ + { + "type": "Warning", + "reason": "BackOff", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Back-off restarting failed container kserve-container in pod " + "text-gen-legacy-predictor-00001-abc12", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Warning", + "reason": "OOMKilled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Container kserve-container was OOMKilled (exit code 137). " + "GPU memory exhausted during KV cache allocation.", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Normal", + "reason": "Scheduled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Successfully assigned ml-production/" + "text-gen-legacy-predictor-00001-abc12 to gpu-worker-1", + "count": 1, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-02-28T08:00:00Z", + }, + { + "type": "Warning", + "reason": "NIMAccountNotReady", + "object": "InferenceService/nim-llama-prod", + "message": "NIM Account 'nim-account' in namespace 'ml-production' " + "is not ready", + "count": 12, + "first_timestamp": "2026-03-14T12:00:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + { + "type": "Warning", + "reason": "ImagePullBackOff", + "object": "InferenceService/nim-llama-prod", + "message": "Failed to pull image 'nvcr.io/nim/meta/llama-3.1-8b-instruct:" + "latest': unauthorized: authentication required", + "count": 8, + "first_timestamp": "2026-03-14T12:05:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + ], +} + + +# ── Resource tools ─────────────────────────────────────────────────────── + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: str = "", +) -> str: + """Get a single Kubernetes resource by apiVersion, kind, and name.""" + if kind == "Node": + for node in ALL_NODES: + if node["metadata"]["name"] == name: + return json.dumps(node, indent=2) + raise ValueError(f"Node '{name}' not found") + + if kind == "ServingRuntime": + if name == "vllm-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_VLLM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + if name == "nim-serving-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_NIM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + raise ValueError(f"ServingRuntime '{name}' not found in namespace '{namespace}'") + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps(lr, indent=2) + + if kind == "Account" and "nim" in apiVersion.lower(): + if namespace == "ml-production" and name == "nim-account": + return json.dumps(NIM_ACCOUNT_CR, indent=2) + raise ValueError( + f"Account '{name}' not found in namespace '{namespace}'" + ) + + if kind == "ClusterVersion" and apiVersion == "config.openshift.io/v1": + return json.dumps({ + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterVersion", + "metadata": {"name": "version"}, + "status": {"desired": {"version": "4.16.3"}}, + }) + + raise ValueError(f"Resource {apiVersion}/{kind}/{name} not found") + + +@mcp.tool() +def resources_create_or_update( + api_version: str, + kind: str, + namespace: str, + name: str, + body: str, +) -> str: + """Create or update a Kubernetes resource. Accepts apiVersion, kind, namespace, name, and body (JSON).""" + try: + resource = json.loads(body) + except json.JSONDecodeError as e: + raise ValueError(f"Invalid JSON body: {e}") from e + + resource.setdefault("metadata", {}) + resource["metadata"]["name"] = name + resource["metadata"]["namespace"] = namespace + resource["apiVersion"] = api_version + resource["kind"] = kind + + if kind == "ServingRuntime": + resource.setdefault("status", {}) + resource["status"]["conditions"] = [ + { + "type": "Ready", + "status": "True", + "reason": "ServingRuntimeReady", + "message": "ServingRuntime is ready", + "lastTransitionTime": "2026-03-17T12:00:00Z", + }, + ] + return json.dumps({ + "status": "created", + "resource": resource, + "message": f"ServingRuntime '{name}' created/updated in namespace '{namespace}'", + }, indent=2) + + if kind == "Secret": + resource.setdefault("type", "Opaque") + return json.dumps({ + "status": "created", + "resource": resource, + "message": f"Secret '{name}' created/updated in namespace '{namespace}'", + }, indent=2) + + if kind in ("NIMAccount", "Account") and "nim" in api_version.lower(): + resource.setdefault("status", {}) + resource["status"]["conditions"] = [ + { + "type": "Ready", + "status": "True", + "reason": "NGCCredentialsValid", + "message": "NGC API key validated successfully", + "lastTransitionTime": "2026-03-17T12:00:00Z", + }, + ] + return json.dumps({ + "status": "created", + "resource": resource, + "message": f"NIM Account '{name}' created/updated in namespace '{namespace}'", + }, indent=2) + + if kind == "ConfigMap": + return json.dumps({ + "status": "created", + "resource": resource, + "message": f"ConfigMap '{name}' created/updated in namespace '{namespace}'", + }, indent=2) + + raise ValueError(f"Unsupported kind for create/update: {kind}") + + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: str = "", + labelSelector: str = "", +) -> str: + """List Kubernetes resources by apiVersion and kind.""" + if kind == "Node": + nodes = ALL_NODES + if labelSelector: + parts = labelSelector.split("=", 1) + key = parts[0] + value = parts[1] if len(parts) > 1 else "" + nodes = [ + n for n in nodes + if n["metadata"]["labels"].get(key) == value + ] + return json.dumps(nodes, indent=2) + + if kind == "Service" and apiVersion == "serving.knative.dev/v1": + return json.dumps({ + "kind": "ServiceList", + "apiVersion": "serving.knative.dev/v1", + "items": [], + "metadata": {}, + }) + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps({ + "kind": "LimitRangeList", + "items": [lr], + }) + + if kind == "InferenceService": + return json.dumps({ + "kind": "InferenceServiceList", + "items": [], + }) + + raise ValueError(f"Unsupported list: {apiVersion}/{kind}") + + +@mcp.tool() +def pods_list( + namespace: str, + labelSelector: str = "", +) -> str: + """List pods in a namespace with optional label selector.""" + pods = PODS_BY_NAMESPACE.get(namespace, []) + + if labelSelector: + key, _, value = labelSelector.partition("=") + pods = [p for p in pods if p.get("labels", {}).get(key) == value] + + results = [] + for pod in pods: + results.append({ + "name": pod["name"], + "namespace": pod["namespace"], + "status": pod["status"], + "restarts": pod.get("restarts", 0), + "node": pod.get("node", ""), + "containers": pod.get("containers", []), + "gpu": pod.get("gpu", "0"), + }) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def pods_log( + namespace: str, + name: str, + container: str = "", +) -> str: + """Get logs from a pod container.""" + logs = POD_LOGS.get(name) + if logs is None: + raise ValueError(f"Pod '{name}' not found in namespace '{namespace}'") + return logs + + +@mcp.tool() +def events_list(namespace: str) -> str: + """List events in a namespace.""" + events = EVENTS_BY_NAMESPACE.get(namespace, []) + return json.dumps(events, indent=2) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/mcp-servers/mock-rhoai-mcp.py b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/mcp-servers/mock-rhoai-mcp.py new file mode 100644 index 00000000..0ae9e4cb --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/mcp-servers/mock-rhoai-mcp.py @@ -0,0 +1,780 @@ +#!/usr/bin/env python3 +"""Mock RHOAI MCP server for SkillsBench rh-ai-engineer task. + +Simulates Red Hat OpenShift AI operations: Data Science Projects, +model serving, data connections, serving runtimes, inference services. + +Scenario: +- ml-production: existing project with two broken deployments + - text-gen-legacy: vLLM OOMKilled (max-model-len=32768 on A10G) + - nim-llama-prod: NIM failing (Account CR not ready, NGC creds invalid) +- fraud-detection: does not exist yet (agent creates it) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("rhoai") + +# ── In-memory state ────────────────────────────────────────────────────── + +PROJECTS = { + "ml-production": { + "name": "ml-production", + "display_name": "ML Production", + "description": "Production ML workloads", + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": "single", + "pipeline_server": True, + }, +} + +DATA_CONNECTIONS = { + "ml-production": [ + { + "name": "prod-model-store", + "type": "S3", + "bucket": "ml-models-prod", + "endpoint": "https://s3.us-east-1.amazonaws.com", + "region": "us-east-1", + }, + ], +} + +SERVING_RUNTIMES = { + "__platform_templates__": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "REST", + "supported_model_formats": [ + {"name": "vLLM", "version": "1", "autoSelect": True} + ], + }, + { + "name": "caikit-tgis-runtime", + "display_name": "Caikit+TGIS ServingRuntime", + "model_formats": ["caikit"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "gRPC", + }, + ], + "ml-production": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + { + "name": "nim-serving-runtime", + "display_name": "NVIDIA NIM ServingRuntime", + "model_formats": ["NIM"], + "requires_instantiation": False, + "source": "nim-account", + "api_protocol": "REST", + }, + { + "name": "ovms-1", + "display_name": "OpenVINO Model Server", + "model_formats": ["openvino_ir", "onnx"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + ], +} + +INFERENCE_SERVICES = { + "ml-production": { + "text-gen-legacy": { + "name": "text-gen-legacy", + "namespace": "ml-production", + "runtime": "vllm-runtime", + "model_format": "vLLM", + "storage_uri": "hf://mistralai/Mistral-7B-Instruct-v0.3", + "display_name": "Mistral 7B Legacy", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "16Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "PredictorFailed", + "message": "Predictor pod is not ready", + }, + { + "type": "PredictorReady", + "status": "False", + "reason": "ContainerCrashLoop", + "message": "Container kserve-container terminated: " + "OOMKilled (exit code 137). 5 restarts.", + }, + { + "type": "IngressReady", + "status": "True", + "reason": "IngressReady", + "message": "Ingress is ready", + }, + ], + "age": "3d", + }, + "nim-llama-prod": { + "name": "nim-llama-prod", + "namespace": "ml-production", + "runtime": "nim-serving-runtime", + "model_format": "NIM", + "storage_uri": "nim://meta/llama-3.1-8b-instruct", + "display_name": "Llama 3.1 8B (NIM)", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "32Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "RuntimeNotReady", + "message": "ServingRuntime 'nim-serving-runtime' " + "is not in ready state", + }, + { + "type": "PredictorReady", + "status": "Unknown", + "reason": "PodNotCreated", + "message": "Predictor pod has not been created. " + "Waiting for ServingRuntime to become ready.", + }, + { + "type": "IngressReady", + "status": "Unknown", + "reason": "PredictorNotReady", + "message": "Waiting for predictor to become ready", + }, + ], + "age": "1d", + }, + }, +} + +DEPLOYED_MODELS = {} + +WORKBENCHES = { + "ml-production": [ + { + "name": "data-exploration-nb", + "display_name": "Data Exploration", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Running", + "cpu_request": "1", + "memory_request": "8Gi", + "gpu_count": 0, + "pvc_name": "data-exploration-nb-pvc", + "pvc_size": "20Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-10T09:00:00Z", + }, + { + "name": "model-training-nb", + "display_name": "Model Training", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Stopped", + "cpu_request": "4", + "memory_request": "16Gi", + "gpu_count": 1, + "pvc_name": "model-training-nb-pvc", + "pvc_size": "50Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-15T14:00:00Z", + }, + ], +} + +PIPELINE_SERVERS = { + "ml-production": { + "configured": True, + "data_connection": "prod-model-store", + "status": "Ready", + "database": "MariaDB", + }, +} + +NOTEBOOK_IMAGES = [ + {"name": "jupyter-pytorch-ubi9-python-3.9-2024.1", "display_name": "PyTorch 2024.1", "packages": ["torch", "transformers"]}, + {"name": "jupyter-tensorflow-ubi9-python-3.9-2024.1", "display_name": "TensorFlow 2024.1", "packages": ["tensorflow"]}, + {"name": "jupyter-datascience-ubi9-python-3.9-2024.1", "display_name": "Standard Data Science", "packages": ["pandas", "scikit-learn"]}, + {"name": "jupyter-minimal-ubi9-python-3.9-2024.1", "display_name": "Minimal Python", "packages": []}, +] + + +# ── Project tools ──────────────────────────────────────────────────────── + + +@mcp.tool() +def list_data_science_projects() -> str: + """List all RHOAI Data Science Projects on the cluster.""" + projects = [] + for name, proj in PROJECTS.items(): + projects.append({ + "name": name, + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + }) + return json.dumps(projects, indent=2) + + +@mcp.tool() +def create_data_science_project( + name: str, + display_name: str, + description: str = "", +) -> str: + """Create a new RHOAI Data Science Project (namespace with dashboard labels).""" + if name in PROJECTS: + raise ValueError( + f"Project '{name}' already exists. Choose a different name " + "or configure the existing project." + ) + if not name.replace("-", "").replace("_", "").isalnum() or len(name) > 63: + raise ValueError( + f"Invalid project name '{name}'. Must be DNS-compatible: " + "lowercase alphanumeric and hyphens, max 63 chars." + ) + + PROJECTS[name] = { + "name": name, + "display_name": display_name, + "description": description, + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": None, + "pipeline_server": False, + } + DATA_CONNECTIONS[name] = [] + SERVING_RUNTIMES[name] = [] + INFERENCE_SERVICES[name] = {} + + return json.dumps({ + "status": "created", + "name": name, + "display_name": display_name, + "namespace": name, + "labels": {"opendatahub.io/dashboard": "true"}, + }) + + +@mcp.tool() +def get_project_details(name: str) -> str: + """Get detailed information about an RHOAI Data Science Project.""" + if name not in PROJECTS: + raise ValueError(f"Project '{name}' not found") + proj = PROJECTS[name] + dc_count = len(DATA_CONNECTIONS.get(name, [])) + isvc_count = len(INFERENCE_SERVICES.get(name, {})) + return json.dumps({ + "name": proj["name"], + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "labels": proj["labels"], + "data_connections": dc_count, + "inference_services": isvc_count, + "model_serving_mode": proj.get("model_serving_mode"), + "pipeline_server": proj.get("pipeline_server", False), + }) + + +@mcp.tool() +def get_project_status(namespace: str) -> str: + """Get comprehensive status of an RHOAI Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Project '{namespace}' not found") + proj = PROJECTS[namespace] + dcs = DATA_CONNECTIONS.get(namespace, []) + isvcs = INFERENCE_SERVICES.get(namespace, {}) + return json.dumps({ + "namespace": namespace, + "display_name": proj["display_name"], + "status": "Active", + "components": { + "data_connections": len(dcs), + "inference_services": len(isvcs), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + "pipeline_server": "configured" if proj.get("pipeline_server") else "not configured", + }, + }) + + +# ── Data connection tools ──────────────────────────────────────────────── + + +@mcp.tool() +def create_s3_data_connection( + namespace: str, + name: str, + bucket: str, + endpoint: str, + access_key: str, + secret_key: str, + region: str = "", +) -> str: + """Create an S3-compatible data connection in an RHOAI project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + existing = DATA_CONNECTIONS.get(namespace, []) + if any(dc["name"] == name for dc in existing): + raise ValueError( + f"Data connection '{name}' already exists in namespace '{namespace}'" + ) + + dc = { + "name": name, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + "region": region, + } + DATA_CONNECTIONS.setdefault(namespace, []).append(dc) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + }) + + +@mcp.tool() +def list_data_connections(namespace: str) -> str: + """List data connections in an RHOAI project namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + return json.dumps(dcs, indent=2) + + +# ── Model serving tools ───────────────────────────────────────────────── + + +@mcp.tool() +def set_model_serving_mode(namespace: str, mode: str) -> str: + """Enable model serving on a Data Science Project (single or multi mode).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + if mode not in ("single", "multi"): + raise ValueError(f"Invalid mode '{mode}'. Must be 'single' or 'multi'.") + + PROJECTS[namespace]["model_serving_mode"] = mode + + if not SERVING_RUNTIMES.get(namespace): + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + SERVING_RUNTIMES[namespace] = [ + {**t, "requires_instantiation": False, "source": "existing"} + for t in templates + ] + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "mode": mode, + }) + + +@mcp.tool() +def list_serving_runtimes( + namespace: str, + include_templates: bool = False, +) -> str: + """List available ServingRuntimes in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + runtimes = list(SERVING_RUNTIMES.get(namespace, [])) + if include_templates: + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + existing_names = {r["name"] for r in runtimes} + for t in templates: + if t["name"] not in existing_names: + runtimes.append(t) + + return json.dumps(runtimes, indent=2) + + +# ── Inference service tools ────────────────────────────────────────────── + + +@mcp.tool() +def deploy_model( + name: str, + namespace: str, + runtime: str, + model_format: str, + storage_uri: str, + display_name: str = "", + min_replicas: int = 1, + max_replicas: int = 1, + cpu_request: str = "1", + cpu_limit: str = "2", + memory_request: str = "4Gi", + memory_limit: str = "8Gi", + gpu_count: int = 0, +) -> str: + """Deploy an AI/ML model as a KServe InferenceService.""" + if namespace not in PROJECTS: + raise ValueError( + f"Namespace '{namespace}' is not a Data Science Project. " + "Create one via create_data_science_project first." + ) + + ns_runtimes = SERVING_RUNTIMES.get(namespace, []) + runtime_names = [r["name"] for r in ns_runtimes] + if runtime not in runtime_names: + available = ", ".join(runtime_names) or "none" + raise ValueError( + f"ServingRuntime '{runtime}' not found in namespace '{namespace}'. " + f"Available runtimes: {available}" + ) + + endpoint = f"https://{name}-{namespace}.apps.ocp-cluster.example.com" + isvc = { + "name": name, + "namespace": namespace, + "runtime": runtime, + "model_format": model_format, + "storage_uri": storage_uri, + "display_name": display_name or name, + "gpu_count": gpu_count, + "cpu_request": cpu_request, + "memory_request": memory_request, + "min_replicas": min_replicas, + "max_replicas": max_replicas, + "ready": True, + "url": endpoint, + "conditions": [ + {"type": "Ready", "status": "True", "reason": "Ready", "message": ""}, + {"type": "PredictorReady", "status": "True", "reason": "PodReady", "message": ""}, + {"type": "IngressReady", "status": "True", "reason": "IngressReady", "message": ""}, + ], + "age": "0s", + } + + INFERENCE_SERVICES.setdefault(namespace, {})[name] = isvc + DEPLOYED_MODELS[f"{namespace}/{name}"] = isvc + + return json.dumps({ + "status": "deployed", + "name": name, + "namespace": namespace, + "runtime": runtime, + "endpoint": endpoint, + "ready": True, + }) + + +@mcp.tool() +def list_inference_services( + namespace: str, + verbosity: str = "standard", +) -> str: + """List deployed InferenceServices in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + isvcs = INFERENCE_SERVICES.get(namespace, {}) + results = [] + for isvc_name, isvc in isvcs.items(): + entry = { + "name": isvc["name"], + "runtime": isvc["runtime"], + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "age": isvc.get("age", ""), + } + if verbosity == "full": + entry["conditions"] = isvc.get("conditions", []) + entry["storage_uri"] = isvc.get("storage_uri", "") + entry["gpu_count"] = isvc.get("gpu_count", 0) + results.append(entry) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def get_inference_service( + name: str, + namespace: str, + verbosity: str = "standard", +) -> str: + """Get detailed status of a specific InferenceService.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + + isvc = isvcs[name] + result = { + "name": isvc["name"], + "namespace": isvc["namespace"], + "runtime": isvc["runtime"], + "model_format": isvc.get("model_format", ""), + "storage_uri": isvc.get("storage_uri", ""), + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "conditions": isvc.get("conditions", []), + "gpu_count": isvc.get("gpu_count", 0), + "replicas": {"min": isvc.get("min_replicas", 1), "max": isvc.get("max_replicas", 1)}, + "resources": { + "cpu_request": isvc.get("cpu_request", "1"), + "memory_request": isvc.get("memory_request", "4Gi"), + "memory_limit": isvc.get("memory_limit", "8Gi"), + }, + "age": isvc.get("age", ""), + } + return json.dumps(result, indent=2) + + +@mcp.tool() +def get_model_endpoint(name: str, namespace: str) -> str: + """Get the inference endpoint URL for a deployed model.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + isvc = isvcs[name] + if not isvc["ready"]: + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": "", + "error": "InferenceService is not ready. Check conditions for details.", + }) + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": isvc["url"], + }) + + +# ── Workbench tools ────────────────────────────────────────────────────── + + +@mcp.tool() +def list_workbenches(namespace: str) -> str: + """List workbenches (Jupyter notebooks) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + return json.dumps(wbs, indent=2) + + +@mcp.tool() +def create_workbench( + namespace: str, + name: str, + display_name: str = "", + image: str = "jupyter-datascience-ubi9-python-3.9-2024.1", + cpu_request: str = "1", + memory_request: str = "4Gi", + gpu_count: int = 0, + pvc_size: str = "20Gi", +) -> str: + """Create a new workbench (Jupyter notebook) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + valid_images = [img["name"] for img in NOTEBOOK_IMAGES] + if image not in valid_images: + raise ValueError( + f"Image '{image}' not found. Available: {', '.join(valid_images)}" + ) + + wb = { + "name": name, + "display_name": display_name or name, + "image": image, + "status": "Running", + "cpu_request": cpu_request, + "memory_request": memory_request, + "gpu_count": gpu_count, + "pvc_name": f"{name}-pvc", + "pvc_size": pvc_size, + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-03-02T12:00:00Z", + } + WORKBENCHES.setdefault(namespace, []).append(wb) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "image": image, + "pvc": f"{name}-pvc", + }) + + +@mcp.tool() +def stop_workbench(namespace: str, name: str) -> str: + """Stop a running workbench (preserves data).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Stopped" + return json.dumps({"status": "stopped", "name": name, "namespace": namespace}) + + +@mcp.tool() +def start_workbench(namespace: str, name: str) -> str: + """Start a stopped workbench.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Running" + return json.dumps({"status": "running", "name": name, "namespace": namespace}) + + +@mcp.tool() +def delete_workbench(namespace: str, name: str) -> str: + """Delete a workbench. WARNING: PVC data may be lost if not backed up.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wbs.remove(wb) + return json.dumps({ + "status": "deleted", + "name": name, + "namespace": namespace, + "warning": "Associated PVC data has been deleted", + }) + + +@mcp.tool() +def list_notebook_images() -> str: + """List available notebook images for workbench creation.""" + return json.dumps(NOTEBOOK_IMAGES, indent=2) + + +# ── Pipeline server tools ─────────────────────────────────────────────── + + +@mcp.tool() +def configure_pipeline_server( + namespace: str, + data_connection: str, + database: str = "MariaDB", +) -> str: + """Configure a pipeline server for a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + if not any(dc["name"] == data_connection for dc in dcs): + available = [dc["name"] for dc in dcs] + raise ValueError( + f"Data connection '{data_connection}' not found. Available: {available}" + ) + + PIPELINE_SERVERS[namespace] = { + "configured": True, + "data_connection": data_connection, + "status": "Ready", + "database": database, + } + PROJECTS[namespace]["pipeline_server"] = True + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "data_connection": data_connection, + "database": database, + }) + + +@mcp.tool() +def get_pipeline_server_status(namespace: str) -> str: + """Get the status of the pipeline server in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + ps = PIPELINE_SERVERS.get(namespace) + if not ps: + return json.dumps({"namespace": namespace, "configured": False}) + return json.dumps({ + "namespace": namespace, + "configured": ps["configured"], + "data_connection": ps["data_connection"], + "status": ps["status"], + "database": ps["database"], + }) + + +# ── Serving runtime creation ──────────────────────────────────────────── + + +@mcp.tool() +def create_serving_runtime( + namespace: str, + name: str, + display_name: str = "", + model_formats: list = None, + container_image: str = "", + container_port: int = 8080, + multi_model: bool = False, + api_protocol: str = "REST", +) -> str: + """Create a custom ServingRuntime in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + if not model_formats: + raise ValueError("model_formats must specify at least one model format") + + runtime = { + "name": name, + "display_name": display_name or name, + "model_formats": model_formats, + "requires_instantiation": False, + "source": "custom", + "api_protocol": api_protocol, + "container_image": container_image, + "container_port": container_port, + "multi_model": multi_model, + } + SERVING_RUNTIMES.setdefault(namespace, []).append(runtime) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "model_formats": model_formats, + }) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/common-issues.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/serving-runtime-config/SKILL.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/serving-runtime-config/SKILL.md new file mode 100644 index 00000000..39ba97a1 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/serving-runtime-config/SKILL.md @@ -0,0 +1,278 @@ +--- +name: serving-runtime-config +description: | + Configure custom ServingRuntime CRs on OpenShift AI for model serving frameworks not covered by built-in runtimes. + + Use when: + - "Create a custom serving runtime" + - "I need a runtime for ONNX / Triton / custom framework" + - "Customize vLLM runtime parameters" + - "What serving runtimes are available?" + - "Add a custom container image for model serving" + + Handles listing existing runtimes, creating new ServingRuntime CRs, and validating compatibility with target models. + + NOT for deploying models (use /model-deploy after runtime is configured). + NOT for NIM platform setup (use /nim-setup). +model: inherit +color: blue +--- + +# /serving-runtime-config Skill + +Configure custom ServingRuntime custom resources on Red Hat OpenShift AI. Use when built-in runtimes (vLLM, NIM, Caikit+TGIS) do not support the target model framework, or when customizing an existing runtime's parameters (env vars, model format, container image). + +## Prerequisites + +**Required MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) + +**Required MCP Tools** (from rhoai): +- `list_serving_runtimes` - List available runtimes and platform templates with supported model formats +- `create_serving_runtime` - Instantiate a serving runtime from a platform template (no YAML needed) +- `list_data_science_projects` - Validate namespace is an RHOAI project + +**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools** (from openshift): +- `resources_get` (from openshift) - Inspect existing ServingRuntime CRs in detail +- `resources_create_or_update` (from openshift) - Create fully custom ServingRuntime CR (when not using templates) + +**Optional MCP Server**: `ai-observability` ([AI Observability MCP](https://github.com/rh-ai-quickstart/ai-observability-summarizer)) + +**Optional MCP Tools** (from ai-observability): +- `list_models` - Verify deployed models use the new runtime + +**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, KServe, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). + +## When to Use This Skill + +**Use this skill when you need to:** +- Create a custom ServingRuntime for a framework not covered by built-in runtimes +- Customize an existing runtime's parameters (env vars, container image, model format) +- Instantiate a platform template runtime into a namespace +- List and compare available serving runtimes and templates + +**Do NOT use this skill when:** +- You want to deploy a model using an existing runtime (use `/model-deploy`) +- You need NIM platform setup (use `/nim-setup`) +- You need to troubleshoot a deployment (use `/debug-inference`) + +## Workflow + +### Step 1: Gather Requirements + +**Ask the user for:** +- **Use case**: What framework/model needs serving? (e.g., "ONNX model", "custom TensorRT engine", "vLLM with custom args") +- **Namespace**: Target namespace for the ServingRuntime +- **Intent**: New runtime from scratch, or customize an existing one? + +**Document Consultation** (read before listing runtimes): +1. **Action**: Read [supported-runtimes.md](../../docs/references/supported-runtimes.md) using the Read tool to understand available runtimes and their capabilities +2. **Output to user**: "I consulted [supported-runtimes.md](../../docs/references/supported-runtimes.md) to understand available runtimes." + +**MCP Tool**: `list_serving_runtimes` (from rhoai) + +**Parameters**: +- `namespace`: user-specified namespace - REQUIRED +- `include_templates`: `true` - REQUIRED (shows both existing runtimes and platform templates) + +**Present findings** in a table: + +| Runtime Name | Model Format | Source | Requires Instantiation | +|--------------|-------------|--------|----------------------| +| [name] | [format] | namespace / template | [true/false] | + +The response distinguishes between: +- **Existing runtimes** (`source: "namespace"`) - ready to use with `/model-deploy` +- **Platform templates** (`source: "template"`, `requires_instantiation: true`) - must be instantiated first + +If an existing runtime fits the user's need, recommend using it directly with `/model-deploy`. If a platform template fits, offer to instantiate it (Step 4 alternative). Otherwise, proceed to Step 2 for custom runtime creation. + +Proceed automatically without waiting for user input. + +### Step 2: Determine Runtime Configuration + +Based on the user's framework and model requirements, determine the ServingRuntime spec. + +**If customizing an existing runtime:** + +**MCP Tool**: `resources_get` (from openshift) + +**Parameters**: +- `apiVersion`: `"serving.kserve.io/v1alpha1"` - REQUIRED +- `kind`: `"ServingRuntime"` - REQUIRED +- `namespace`: user-specified namespace - REQUIRED +- `name`: name of the existing runtime to customize - REQUIRED + +Extract the current spec as a starting point. Present the current configuration and ask what the user wants to change. + +**If the user requests a runtime for an unfamiliar framework -> Trigger live doc lookup:** +1. **Action**: Read [live-doc-lookup.md](../references/live-doc-lookup.md) using the Read tool for the lookup protocol +2. **Output to user**: "Framework [name] is not in my cached runtimes. I'll look up its serving requirements." +3. Use **WebFetch** to retrieve specs from Red Hat OpenShift AI documentation +4. Extract: container image, model format name, supported protocols, required env vars +5. **Output to user**: "I looked up [framework] on [source] to confirm its runtime requirements: [summary]" + +**Collect runtime parameters:** + +| Parameter | Value | Source | +|-----------|-------|--------| +| Runtime name | [name] | user input | +| Container image | [image:tag] | user input / doc lookup | +| Model format name | [format] | user input / doc lookup | +| Supported protocol versions | [v1, v2, grpc-v2] | user input / default | +| Multi-model serving | [true/false] | default: false (single-model) | +| Environment variables | [list] | user input | +| GPU resource requirements | [limits] | user input | + +Proceed automatically without waiting for user input. + +### Step 3: Generate ServingRuntime YAML + +Generate the ServingRuntime manifest using values from Steps 1-2. + +```yaml +apiVersion: serving.kserve.io/v1alpha1 +kind: ServingRuntime +metadata: + name: [runtime-name] + namespace: [namespace] + labels: + opendatahub.io/dashboard: "true" + annotations: + openshift.io/display-name: "[Display Name]" +spec: + supportedModelFormats: + - name: [model-format-name] + version: "[version]" + autoSelect: true + multiModel: false + containers: + - name: kserve-container + image: [container-image:tag] + ports: + - containerPort: 8080 + protocol: TCP + env: + - name: [ENV_VAR_NON_SECRET] + value: "[non-sensitive-value]" + - name: [SECRET_ENV_VAR] + valueFrom: + secretKeyRef: + name: [k8s-secret-name] + key: [secret-key-name] + resources: + limits: + nvidia.com/gpu: "[gpu-count]" + requests: + cpu: "[cpu]" + memory: "[memory]" +``` + +Display the ServingRuntime YAML to the user, **redacting any sensitive values**. + +**Ask**: "Proceed with creating this ServingRuntime? (yes/no/modify)" + +Proceed automatically without waiting for user input. + +- If **yes** -> Proceed to Step 4 +- If **no** -> Abort +- If **modify** -> Ask what to change, regenerate YAML, return to this step + +### Step 4: Create ServingRuntime + +**If instantiating from a platform template** (user chose a template from Step 1): + +**MCP Tool**: `create_serving_runtime` (from rhoai) + +**Parameters**: +- `namespace`: target namespace - REQUIRED +- `template_name`: name of the template to instantiate (e.g., `"vllm-cuda-runtime-template"`) - REQUIRED + +The response includes the created runtime name, display name, and supported model formats. + +**If creating a fully custom runtime** (custom container image, non-template configuration): + +**MCP Tool**: `resources_create_or_update` (from openshift) + +**Parameters**: +- `manifest`: full ServingRuntime manifest as JSON string - REQUIRED +- `namespace`: user-specified namespace - REQUIRED + +**Error Handling**: +- If namespace not found -> Report error, suggest creating namespace or using `/ds-project-setup` +- If runtime name already exists -> Ask user: "ServingRuntime `[name]` already exists. Update it? (yes/no)" +- If CRD not found -> Report: "ServingRuntime CRD not available. Ensure Red Hat OpenShift AI operator is installed." +- If RBAC error -> Report insufficient permissions + +### Step 5: Validate Runtime + +**MCP Tool**: `list_serving_runtimes` (from rhoai) + +**Parameters**: +- `namespace`: user-specified namespace - REQUIRED +- `include_templates`: `false` + +Verify the runtime appears in the namespace runtime list. + +For detailed inspection: + +**MCP Tool**: `resources_get` (from openshift) + +**Parameters**: +- `apiVersion`: `"serving.kserve.io/v1alpha1"` - REQUIRED +- `kind`: `"ServingRuntime"` - REQUIRED +- `namespace`: user-specified namespace - REQUIRED +- `name`: the created runtime name - REQUIRED + +**Report results** showing: runtime name, namespace, model format, container image, and next steps (`/model-deploy` to deploy a model using this runtime). + +## Common Issues + +For common issues (GPU scheduling, OOMKilled, image pull errors, RBAC), see [common-issues.md](../references/common-issues.md). + +### Issue 1: InferenceService Cannot Find Runtime + +**Error**: InferenceService status shows "Unknown" or runtime not matched + +**Cause**: The `modelFormat.name` in the InferenceService does not match any `supportedModelFormats[].name` in available ServingRuntimes. + +**Solution:** +1. Verify the model format name matches exactly (case-sensitive) +2. Check the runtime is in the same namespace as the InferenceService +3. Ensure the runtime has `opendatahub.io/dashboard: "true"` label + +### Issue 2: Runtime Port Mismatch + +**Error**: InferenceService created but health checks fail, endpoint returns connection refused + +**Cause**: The `containerPort` in the ServingRuntime does not match the port the serving framework actually listens on. + +**Solution:** +1. Check the framework's documentation for its default serving port +2. Update the `containerPort` in the ServingRuntime spec +3. Or set an environment variable to configure the framework's listen port to match + +## Dependencies + +### MCP Tools +See [Prerequisites](#prerequisites) for the complete list of required and optional MCP tools. + +### Related Skills +- `/model-deploy` - Deploy a model using the configured runtime +- `/nim-setup` - NIM platform setup (if NIM runtime is needed instead) +- `/debug-inference` - Troubleshoot InferenceService failures after deployment + +### Reference Documentation +- [supported-runtimes.md](../../docs/references/supported-runtimes.md) - Runtime capabilities and model format names +- [live-doc-lookup.md](../references/live-doc-lookup.md) - Protocol for fetching specs for unknown frameworks + +## Critical: Human-in-the-Loop Requirements + +See [skill-conventions.md](../references/skill-conventions.md) for general HITL and security conventions. + +**Skill-specific checkpoints:** +- After listing existing runtimes (Step 1): confirm whether to create new or customize existing +- After collecting parameters (Step 2): confirm runtime configuration +- Before creating ServingRuntime (Step 3): display full YAML, confirm +- **NEVER** overwrite an existing ServingRuntime without user confirmation diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/instruction.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/instruction.md new file mode 100644 index 00000000..d89e7c6a --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/instruction.md @@ -0,0 +1,19 @@ +# Serving Runtime Configuration Task + +You are an AI engineer on Red Hat OpenShift AI. Your team needs to serve a model using a custom inference engine that is not available as a default runtime on the platform. + +## Scenario +The existing platform-provided serving runtimes do not support the model format your team needs. You must create a custom runtime configuration that integrates properly with the platform and can be used to deploy models. + +## Requirements +- Examine the currently available serving runtimes and platform templates, distinguishing which are already instantiated versus which require instantiation before use +- Design a custom ServingRuntime CR that specifies the inference container, supported model formats, resource requirements, and API protocol +- Follow KServe container naming conventions so the runtime integrates correctly with the platform's model serving framework +- For runtimes supporting multiple model formats, explain how autoSelect should be configured to avoid format conflicts +- Explain where GPU resource allocation belongs (in the ServingRuntime vs in the InferenceService) and why +- Ensure the runtime will be visible and usable from the platform dashboard +- Document your design decisions and trade-offs + +Document your configuration plan and the complete runtime specification in `/root/report.md`. + +Use MCP tools to interact with the platform. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/solution/solve.sh b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/solution/solve.sh new file mode 100644 index 00000000..043771f9 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/solution/solve.sh @@ -0,0 +1,31 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# ServingRuntime Configuration + +## Custom Runtime: triton-onnx + +Platform templates: list_serving_runtimes with include_templates: true. Templates with requires_instantiation: true use create_serving_runtime. + +```yaml +apiVersion: serving.kserve.io/v1alpha1 +kind: ServingRuntime +metadata: + name: triton-onnx-runtime + labels: + opendatahub.io/dashboard: "true" +spec: + supportedModelFormats: + - name: onnx + version: "1" + autoSelect: true + multiModel: false + containers: + - name: kserve-container + image: nvcr.io/nvidia/tritonserver:latest + ports: + - containerPort: 8080 + protocol: TCP +``` + +### Key: supportedModelFormats.name must match InferenceService modelFormat.name +REPORT_EOF diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/task.toml b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/task.toml new file mode 100644 index 00000000..8ee93afa --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-ai-engineer__serving-runtime-config" +name = "rh-ai-engineer Serving Runtime Configuration Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-ai-engineer", "serving-runtime-config", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/tests/llm_judge.py b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/tests/llm_judge.py new file mode 100644 index 00000000..11fdec60 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "kserve_container_naming", + "file": "/root/report.md", + "question": "Does the ServingRuntime YAML in the report name the main container 'kserve-container' (the required KServe naming convention)?", + "reference": "A skilled report names the container kserve-container in the ServingRuntime spec, which is required by KServe for the model serving framework to function correctly. An unskilled report might use a framework-specific name like 'triton' or 'vllm', which would cause KServe integration issues." + }, + { + "id": "gpu_allocation_strategy", + "file": "/root/report.md", + "question": "Does the report explain that GPU resources should NOT be hardcoded in the ServingRuntime and instead should be allocated at the InferenceService level for flexibility?", + "reference": "A skilled report explains that GPU resources (nvidia.com/gpu) belong at the InferenceService deployment level because different models need 0, 1, or multiple GPUs. The ServingRuntime should remain GPU-agnostic. An unskilled report hardcodes nvidia.com/gpu: 1 directly in the ServingRuntime spec." + }, + { + "id": "autoselect_and_api_conventions", + "file": "/root/report.md", + "question": "Does the report configure autoSelect: false for non-primary model formats and use the correct ServingRuntime API version (v1alpha1)?", + "reference": "A skilled report uses autoSelect: true only for the primary format and false for secondary formats to prevent conflicts, and uses the serving.kserve.io/v1alpha1 API version for ServingRuntime (distinct from v1beta1 used for InferenceService). An unskilled report sets autoSelect: true for all formats or uses the wrong API version." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/tests/test.sh b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/tests/test_outputs.py b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/tests/test_outputs.py new file mode 100644 index 00000000..71257bf2 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/tests/test_outputs.py @@ -0,0 +1,97 @@ +""" +Tests for rh-ai-engineer__serving-runtime-config per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ["servingruntime", "serving runtime", "runtime"]), ( + "report should mention ServingRuntime" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 200, "report should have substantial content" + + +class TestSkillDependent: + def test_kserve_container_name(self): + """Skill teaches the main container MUST be named kserve-container for KServe + compatibility. Without skill, agents use framework-specific names like 'triton'.""" + c = read_report() + assert "kserve-container" in c, ( + "should name the main container 'kserve-container' (required by KServe)" + ) + + def test_serving_runtime_api_version(self): + """Skill teaches ServingRuntime uses serving.kserve.io/v1alpha1 API (alpha, + not beta like InferenceService). Without skill, agents use v1beta1 or omit + the apiVersion distinction between ServingRuntime and InferenceService.""" + c = read_report() + assert "v1alpha1" in c or ( + "alpha" in c.lower() and "serving" in c.lower() + ), "should use v1alpha1 API version for ServingRuntime" + + def test_autoselect_false_for_secondary(self): + """Skill teaches using autoSelect: true only for primary format and false for + secondary formats to avoid conflicts. Without skill, agents set true for all.""" + c = read_report().lower() + assert "autoselect: false" in c or "autoselect\":false" in c or "autoselect\": false" in c, ( + "should use autoSelect: false for non-primary model formats" + ) + + def test_gpu_at_inferenceservice_level(self): + """Skill teaches not hardcoding GPU in ServingRuntime; GPU allocation belongs + at the InferenceService level for flexibility. Without skill, agents hardcode + nvidia.com/gpu in the runtime spec.""" + c = read_report().lower() + assert any(t in c for t in [ + "inferenceservice level", "inferenceservice deployment", + "per inferenceservice", "not specified in the servingruntime", + "gpu allocation happens at", + ]), "should explain GPU allocation belongs at InferenceService level, not in the runtime" + + def test_model_format_matching(self): + """Skill teaches that supportedModelFormats must match InferenceService model + format for runtime selection.""" + c = read_report().lower() + assert any(t in c for t in [ + "model format", "supportedmodelformat", "supported model format", + "inferenceservice", "match", + ]), "should address model format matching for runtime selection" + + def test_dashboard_label(self): + """Skill teaches opendatahub.io/dashboard label for dashboard visibility.""" + c = read_report().lower() + assert any(t in c for t in [ + "opendatahub", "dashboard", "label", "visible", + "platform", "display", + ]), "should address dashboard/platform visibility via labels" + + def test_caikit_tgis_grpc(self): + """Docs teach Caikit+TGIS is gRPC-only (no REST API) and NIM uses + TensorRT-LLM with pre-compiled engines. Without docs, agents assume REST + for all runtimes.""" + c = read_report().lower() + assert any(t in c for t in [ + "grpc", "caikit", "tgis", "tensorrt", + ]) and ("runtime" in c or "serving" in c), ( + "should note Caikit+TGIS gRPC-only or NIM TensorRT-LLM characteristics" + ) diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/Dockerfile b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/Dockerfile new file mode 100644 index 00000000..d4978abe --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/Dockerfile @@ -0,0 +1,74 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "rhoai": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-rhoai-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/common-issues.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/model-deploy.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/model-deploy.md new file mode 100644 index 00000000..dcd79e44 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/model-deploy.md @@ -0,0 +1,160 @@ +# /model-deploy Examples + +Example interactions demonstrating the model-deploy skill workflow. + +## Example 1: Deploy Llama 3.1 8B with vLLM + +``` +User: "Deploy Llama 3.1 8B on my cluster" + +Agent: [Invokes /model-deploy skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand runtime capabilities. + +Step 1 of 10: Gathering deployment information. + +| Setting | Value | Source | +|---------|-------|--------| +| Model | Llama 3.1 8B Instruct | user input | +| Runtime | TBD | auto-detect | +| Namespace | ? | needs input | +| Model Source | ? | needs input | +| Deployment Mode | Serverless | default | + +What namespace should I deploy to, and where are the model weights stored? +(e.g., S3 bucket, OCI registry, PVC) + +User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" + +Agent: Step 2 of 10: Determining runtime. + +I consulted supported-runtimes.md to understand runtime capabilities. + +Based on Llama 3.1 8B (open-source HuggingFace model): + +Recommended runtime: vLLM +Reason: Default runtime for open-source LLMs. Broad compatibility, + OpenAI-compatible API, no additional setup required. + +Proceed with vLLM? (yes / no / suggest alternative) + +User: "yes" + +Agent: Step 3 of 10: Looking up hardware profile. + +I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. + +| Requirement | Value | +|-------------|-------| +| GPUs | 1x A10G/L4/A100/H100 | +| VRAM | 16GB | +| Key Args | --max-model-len=4096 --tool-call-parser hermes | + +Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). +Ensure your cluster has at least 1 GPU with 16GB VRAM. + +Step 5 of 10: Skipped (NIM platform check not needed for vLLM). + +Step 6-7 of 10: Generating InferenceService YAML. + +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: llama-3-1-8b + namespace: ai-models + annotations: + serving.kserve.io/deploymentMode: Serverless +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: vllm-runtime + storageUri: s3://my-bucket/llama-3.1-8b + resources: + limits: + nvidia.com/gpu: "1" + requests: + cpu: "4" + memory: "16Gi" + args: + - --max-model-len=4096 + - --tool-call-parser=hermes + +| Setting | Value | +|---------|-------| +| Model | llama-3-1-8b | +| Runtime | vllm-runtime | +| GPUs | 1 | +| Namespace | ai-models | +| Mode | Serverless | + +Proceed with creating this InferenceService? (yes/no/modify) + +User: "yes" + +Agent: Step 8 of 10: Creating InferenceService... + [Creates via resources_create_or_update] + + InferenceService created. Monitoring rollout... + +Step 9 of 10: Monitoring rollout. + +| Stage | Status | +|-------|--------| +| Pod Scheduled | Complete | +| Image Pulled | Complete | +| Container Started | Complete | +| Model Loaded | Complete | +| Ready | Complete | + +Step 10 of 10: Deployment complete! + +Model Deployment Successful + +Model: llama-3-1-8b +Runtime: vLLM +Namespace: ai-models +GPUs: 1 + +Inference Endpoint: + URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com + API: OpenAI-compatible REST + +Quick Test: + curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' + +Next Steps: + Monitor performance: "/ai-observability llama-3-1-8b" +``` + +## Example 2: Deploy a Model with NIM (NIM Not Set Up) + +``` +User: "Deploy Llama 3.1 70B with NIM" + +Agent: [Invokes /model-deploy skill] + ... + Step 2: User requested NIM runtime. + + Step 5: Checking NIM platform... + +NIM platform is not set up in namespace ai-models. + +The NIM runtime requires a one-time platform setup. Would you like to: +1. Run /nim-setup now to configure the NIM platform +2. Switch to vLLM runtime instead (no setup required) +3. Abort deployment + +User: "1" + +Agent: [Invokes /nim-setup skill — walks through NGC credential setup] + ... + NIM Platform Setup Complete. + + Resuming model deployment... + [Continues from Step 6 with NIM runtime] +``` diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/nim-setup.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/nim-setup.md new file mode 100644 index 00000000..ccfc1a5d --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/nim-setup.md @@ -0,0 +1,115 @@ +# /nim-setup Examples + +Example interactions demonstrating the nim-setup skill workflow. + +## Example 1: First-Time NIM Setup + +``` +User: "Set up NIM on my cluster" + +Agent: [Invokes /nim-setup skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand NIM platform requirements. + +Let me verify the prerequisites for NIM setup. + +Step 1 of 7: Checking GPU Operator and NFD Operator... + +GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) +NFD Operator: nfd.v4.19.0 (Succeeded) + +Both operators are installed and healthy. + +Step 2 of 7: I need your NGC credentials. + +To set up NIM, I need your NVIDIA NGC API key. +You can generate one at: https://ngc.nvidia.com/setup/api-key + +Please provide: +1. Your NGC API key +2. The target namespace for NIM resources + +User: "Here's my key: [user provides NGC API key], namespace: ai-models" + +Agent: Step 3 of 7: Creating NGC image pull secret... + +I'll create a docker-registry secret for pulling NIM images from nvcr.io: + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-image-pull-secret + namespace: ai-models + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] + +Should I create this image pull secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + Image pull secret created successfully. + +Step 4 of 7: Creating NGC API key secret... + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-api-key + namespace: ai-models + type: Opaque + stringData: + NGC_API_KEY: "[REDACTED]" + +Should I create this API key secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + API key secret created successfully. + +Step 5 of 7: Creating NIM Account CR... + + apiVersion: nim.opendatahub.io/v1 + kind: Account + metadata: + name: nim-account + namespace: ai-models + spec: + apiKeySecret: + name: ngc-api-key + imagePullSecret: + name: ngc-image-pull-secret + +Should I create this NIM Account CR in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates Account CR via resources_create_or_update] + Account CR created successfully. + +Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) + +User: "no" + +Agent: Step 7 of 7: Validating NIM platform readiness... + +NIM Platform Setup Complete + +Account CR: `nim-account` (namespace: ai-models) + Status: Ready + +NGC Credentials: + Image Pull Secret: ngc-image-pull-secret [created] + API Key Secret: ngc-api-key [created] + +NIM ServingRuntimes Available: + - nim-llama-3-1-8b-instruct + - nim-llama-3-1-70b-instruct + - nim-mistral-7b-instruct + +Next Steps: + To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" +``` diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/references/known-model-profiles.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/references/known-model-profiles.md new file mode 100644 index 00000000..40148981 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/references/known-model-profiles.md @@ -0,0 +1,83 @@ +--- +title: Known Model Hardware Profiles +category: references +tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] +semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] +use_cases: [model-deploy, debug-inference, ai-observability] +last_updated: 2026-03-10 +--- + +# Known Model Hardware Profiles + +Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). + +**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. + +## Model Source Conventions + +Each model lists a recommended `storageUri` with its authentication requirements: +- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** +- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). +- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. + +When the user does not specify a model source, use the `hf://` URI listed in the profile below. + +## Llama 3.x (Meta) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | +| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | +| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | +| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | + +- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) +- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly + +## Granite 3.x (IBM/Red Hat) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | +| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | + +- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) +- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser granite --chat-template` +- Red Hat-supported model family on RHOAI + +## Mixtral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | +| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) +- Mixture-of-Experts architecture: only ~13B/45B parameters active per token + +## Mistral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | +| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) + +## When a Model Is Not Listed + +If the requested model is not in this file, the agent MUST use the live doc lookup protocol: + +1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure +2. Fetch hardware specs from the appropriate source +3. Report findings to the user before proceeding with deployment + +Common cases requiring live lookup: +- Newly released models (after this file's last update) +- Domain-specific fine-tuned models +- Models with custom quantization +- NIM-specific optimized profiles diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/references/supported-runtimes.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/references/supported-runtimes.md new file mode 100644 index 00000000..9d018651 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/references/supported-runtimes.md @@ -0,0 +1,104 @@ +--- +title: Supported Serving Runtimes +category: references +tags: [runtimes, vllm, nim, caikit, tgis, serving] +semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] +use_cases: [model-deploy, serving-runtime-config, nim-setup] +last_updated: 2026-02-26 +--- + +# Supported Serving Runtimes + +This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. + +## Runtime Comparison + +| Runtime | API Type | Model Format | GPU Support | License | Setup Required | +|---------|----------|-------------|-------------|---------|----------------| +| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | +| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | +| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | +| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | + +## vLLM + +**Default runtime for most open-source models.** + +- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) +- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized +- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) +- **Model source**: S3-compatible storage, OCI registry, PVC, URI +- **Key features**: + - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) + - PagedAttention for efficient memory management + - Tensor parallelism for multi-GPU inference + - Tool/function calling support (`--tool-call-parser`) + - Continuous batching for high throughput +- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name + +## NVIDIA NIM + +**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** + +- **API**: OpenAI-compatible REST +- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog +- **GPU support**: NVIDIA only (requires specific GPU models per model profile) +- **Model source**: NGC model catalog (pulled automatically via NGC credentials) +- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) +- **Key features**: + - TensorRT-LLM optimization for lower latency + - Pre-compiled model engines (no compilation on first load) + - Optimized and generic profiles per GPU type + - Automatic model download from NGC +- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. +- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM +- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. +- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state + +## Caikit+TGIS + +**Red Hat's Caikit format with Text Generation Inference Server.** + +- **API**: gRPC (not REST) +- **Model formats**: Caikit format (requires conversion from HuggingFace) +- **GPU support**: NVIDIA +- **Model source**: S3-compatible storage +- **Key features**: + - Red Hat-supported runtime + - gRPC API for streaming inference + - Integrated with RHOAI model serving platform +- **When to choose**: When the model is already in Caikit format or when gRPC API is required. +- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name + +## Custom Runtimes + +**User-provided ServingRuntime CRs for unsupported frameworks.** + +- **API**: Defined by the custom runtime +- **Model formats**: Defined by the custom runtime +- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. +- **How to create**: Use `/serving-runtime-config` skill +- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility + +## Runtime Selection Decision Tree + +``` +Is the user's preferred runtime explicitly stated? +├── Yes → Use that runtime +└── No → Continue + +Is the model available in the NGC NIM catalog? +├── Yes → Suggest NIM (with vLLM as fallback) +│ Note: Requires /nim-setup and NVIDIA GPUs +└── No → Continue + +Is the model in Caikit format? +├── Yes → Caikit+TGIS +└── No → Continue + +Is the model a standard open-source LLM (HuggingFace-compatible)? +├── Yes → vLLM (default) +└── No → Custom runtime via /serving-runtime-config +``` diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..e7a4d11c --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +"""Mock OpenShift MCP server for SkillsBench rh-ai-engineer task. + +Simulates Kubernetes resource CRUD, pod management, logs, and events. + +Key scenario elements: +- LimitRange in namespaces: min CPU=100m, min memory=128Mi + (conflicts with KServe sidecar containers hardcoded at 10m CPU/15Mi memory) +- GPU node with custom taint ai-workload=true:NoSchedule +- NIM Account CR in ml-production: not ready (NGC credentials invalid) +- text-gen-legacy pods: OOMKilled (max-model-len=32768 on A10G) +- nim-llama-prod: no pods created (Account CR not ready) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + +# ── Cluster state ──────────────────────────────────────────────────────── + +GPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "gpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + "nvidia.com/gpu.present": "true", + "nvidia.com/gpu.product": "NVIDIA-A10G", + }, + }, + "spec": { + "taints": [ + { + "key": "ai-workload", + "value": "true", + "effect": "NoSchedule", + }, + ], + }, + "status": { + "allocatable": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "capacity": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "conditions": [ + {"type": "Ready", "status": "True"}, + ], + }, +} + +CPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "cpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + }, + }, + "spec": {"taints": []}, + "status": { + "allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "capacity": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +MASTER_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "master-1", + "labels": { + "node-role.kubernetes.io/master": "", + "node-role.kubernetes.io/control-plane": "", + }, + }, + "spec": { + "taints": [ + {"key": "node-role.kubernetes.io/master", "effect": "NoSchedule"}, + ], + }, + "status": { + "allocatable": {"cpu": "8", "memory": "32Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +ALL_NODES = [GPU_NODE, CPU_NODE, MASTER_NODE] + +# LimitRange applied by cluster policy to all DS project namespaces +NAMESPACE_LIMITRANGE = { + "apiVersion": "v1", + "kind": "LimitRange", + "metadata": { + "name": "default-limits", + }, + "spec": { + "limits": [ + { + "type": "Container", + "default": { + "cpu": "2", + "memory": "4Gi", + }, + "defaultRequest": { + "cpu": "500m", + "memory": "256Mi", + }, + "min": { + "cpu": "100m", + "memory": "128Mi", + }, + "max": { + "cpu": "32", + "memory": "128Gi", + }, + }, + ], + }, +} + +NIM_ACCOUNT_CR = { + "apiVersion": "nim.opendatahub.io/v1", + "kind": "Account", + "metadata": { + "name": "nim-account", + "namespace": "ml-production", + }, + "spec": { + "apiKeySecret": { + "name": "ngc-api-key", + }, + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "NGCCredentialsInvalid", + "message": "NGC API key validation failed: 401 Unauthorized. " + "The API key in secret 'ngc-api-key' is expired or invalid. " + "Re-create the secret with a valid NGC API key from " + "https://ngc.nvidia.com/setup/api-key and restart the " + "Account reconciliation.", + "lastTransitionTime": "2026-03-14T12:00:00Z", + }, + ], + "nimPullSecretStatus": "Failed", + "nimConfigStatus": "Pending", + }, +} + +SERVING_RUNTIME_VLLM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "vllm-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "vLLM", "version": "1", "autoSelect": True}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "quay.io/modh/vllm:rhoai-2.16", + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + }, + ], + }, +} + +SERVING_RUNTIME_NIM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "nim-serving-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "NIM", "version": "1"}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "nvcr.io/nim/meta/llama-3.1-8b-instruct:latest", + "ports": [{"containerPort": 8000, "protocol": "TCP"}], + "env": [ + {"name": "NGC_API_KEY", "valueFrom": { + "secretKeyRef": {"name": "ngc-api-key", "key": "api_key"}, + }}, + ], + }, + ], + }, +} + +PODS_BY_NAMESPACE = { + "ml-production": [ + { + "name": "text-gen-legacy-predictor-00001-abc12", + "namespace": "ml-production", + "status": "CrashLoopBackOff", + "restarts": 5, + "node": "gpu-worker-1", + "containers": [ + { + "name": "kserve-container", + "state": "waiting", + "reason": "CrashLoopBackOff", + "last_termination_reason": "OOMKilled", + "last_termination_exit_code": 137, + }, + ], + "labels": { + "serving.kserve.io/inferenceservice": "text-gen-legacy", + }, + "gpu": "1", + }, + # nim-llama-prod: NO pods created (Account CR not ready) + ], +} + +POD_LOGS = { + "text-gen-legacy-predictor-00001-abc12": ( + "INFO 2026-03-01 10:00:00 vllm_engine.py:125] vLLM engine starting...\n" + "INFO 2026-03-01 10:00:01 config.py:89] Model: mistralai/Mistral-7B-Instruct-v0.3\n" + "INFO 2026-03-01 10:00:01 config.py:92] max_model_len = 32768\n" + "INFO 2026-03-01 10:00:02 gpu_executor.py:45] GPU 0: NVIDIA A10G (24576 MiB)\n" + "INFO 2026-03-01 10:00:03 model_runner.py:88] Loading model weights...\n" + "INFO 2026-03-01 10:00:15 model_runner.py:112] Model weights loaded: 13.5 GiB\n" + "INFO 2026-03-01 10:00:15 worker.py:201] Allocating KV cache...\n" + "ERROR 2026-03-01 10:00:16 worker.py:215] torch.cuda.OutOfMemoryError: " + "CUDA out of memory. Tried to allocate 28.5 GiB for KV cache but only " + "10.1 GiB available after loading model weights (13.5 GiB).\n" + "ERROR 2026-03-01 10:00:16 vllm_engine.py:178] Engine failed to start\n" + "Traceback (most recent call last):\n" + " File \"/opt/vllm/vllm/engine/engine.py\", line 175, in start\n" + " self._init_kv_cache()\n" + " File \"/opt/vllm/vllm/worker/worker.py\", line 215, in _init_kv_cache\n" + " raise torch.cuda.OutOfMemoryError(msg)\n" + "torch.cuda.OutOfMemoryError: CUDA out of memory\n" + ), +} + +EVENTS_BY_NAMESPACE = { + "ml-production": [ + { + "type": "Warning", + "reason": "BackOff", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Back-off restarting failed container kserve-container in pod " + "text-gen-legacy-predictor-00001-abc12", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Warning", + "reason": "OOMKilled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Container kserve-container was OOMKilled (exit code 137). " + "GPU memory exhausted during KV cache allocation.", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Normal", + "reason": "Scheduled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Successfully assigned ml-production/" + "text-gen-legacy-predictor-00001-abc12 to gpu-worker-1", + "count": 1, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-02-28T08:00:00Z", + }, + { + "type": "Warning", + "reason": "NIMAccountNotReady", + "object": "InferenceService/nim-llama-prod", + "message": "NIM Account 'nim-account' in namespace 'ml-production' " + "is not ready", + "count": 12, + "first_timestamp": "2026-03-14T12:00:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + { + "type": "Warning", + "reason": "ImagePullBackOff", + "object": "InferenceService/nim-llama-prod", + "message": "Failed to pull image 'nvcr.io/nim/meta/llama-3.1-8b-instruct:" + "latest': unauthorized: authentication required", + "count": 8, + "first_timestamp": "2026-03-14T12:05:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + ], +} + + +# ── Resource tools ─────────────────────────────────────────────────────── + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: str = "", +) -> str: + """Get a single Kubernetes resource by apiVersion, kind, and name.""" + if kind == "Node": + for node in ALL_NODES: + if node["metadata"]["name"] == name: + return json.dumps(node, indent=2) + raise ValueError(f"Node '{name}' not found") + + if kind == "ServingRuntime": + if name == "vllm-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_VLLM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + if name == "nim-serving-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_NIM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + raise ValueError(f"ServingRuntime '{name}' not found in namespace '{namespace}'") + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps(lr, indent=2) + + if kind == "Account" and "nim" in apiVersion.lower(): + if namespace == "ml-production" and name == "nim-account": + return json.dumps(NIM_ACCOUNT_CR, indent=2) + raise ValueError( + f"Account '{name}' not found in namespace '{namespace}'" + ) + + if kind == "ClusterVersion" and apiVersion == "config.openshift.io/v1": + return json.dumps({ + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterVersion", + "metadata": {"name": "version"}, + "status": {"desired": {"version": "4.16.3"}}, + }) + + raise ValueError(f"Resource {apiVersion}/{kind}/{name} not found") + + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: str = "", + labelSelector: str = "", +) -> str: + """List Kubernetes resources by apiVersion and kind.""" + if kind == "Node": + nodes = ALL_NODES + if labelSelector: + parts = labelSelector.split("=", 1) + key = parts[0] + value = parts[1] if len(parts) > 1 else "" + nodes = [ + n for n in nodes + if n["metadata"]["labels"].get(key) == value + ] + return json.dumps(nodes, indent=2) + + if kind == "Service" and apiVersion == "serving.knative.dev/v1": + return json.dumps({ + "kind": "ServiceList", + "apiVersion": "serving.knative.dev/v1", + "items": [], + "metadata": {}, + }) + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps({ + "kind": "LimitRangeList", + "items": [lr], + }) + + if kind == "InferenceService": + return json.dumps({ + "kind": "InferenceServiceList", + "items": [], + }) + + raise ValueError(f"Unsupported list: {apiVersion}/{kind}") + + +@mcp.tool() +def pods_list( + namespace: str, + labelSelector: str = "", +) -> str: + """List pods in a namespace with optional label selector.""" + pods = PODS_BY_NAMESPACE.get(namespace, []) + + if labelSelector: + key, _, value = labelSelector.partition("=") + pods = [p for p in pods if p.get("labels", {}).get(key) == value] + + results = [] + for pod in pods: + results.append({ + "name": pod["name"], + "namespace": pod["namespace"], + "status": pod["status"], + "restarts": pod.get("restarts", 0), + "node": pod.get("node", ""), + "containers": pod.get("containers", []), + "gpu": pod.get("gpu", "0"), + }) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def pods_log( + namespace: str, + name: str, + container: str = "", +) -> str: + """Get logs from a pod container.""" + logs = POD_LOGS.get(name) + if logs is None: + raise ValueError(f"Pod '{name}' not found in namespace '{namespace}'") + return logs + + +@mcp.tool() +def events_list(namespace: str) -> str: + """List events in a namespace.""" + events = EVENTS_BY_NAMESPACE.get(namespace, []) + return json.dumps(events, indent=2) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/mcp-servers/mock-rhoai-mcp.py b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/mcp-servers/mock-rhoai-mcp.py new file mode 100644 index 00000000..12513127 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/mcp-servers/mock-rhoai-mcp.py @@ -0,0 +1,866 @@ +#!/usr/bin/env python3 +"""Mock RHOAI MCP server for SkillsBench rh-ai-engineer task. + +Simulates Red Hat OpenShift AI operations: Data Science Projects, +model serving, data connections, serving runtimes, inference services. + +Scenario: +- ml-production: existing project with two broken deployments + - text-gen-legacy: vLLM OOMKilled (max-model-len=32768 on A10G) + - nim-llama-prod: NIM failing (Account CR not ready, NGC creds invalid) +- fraud-detection: does not exist yet (agent creates it) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("rhoai") + +# ── In-memory state ────────────────────────────────────────────────────── + +PROJECTS = { + "ml-production": { + "name": "ml-production", + "display_name": "ML Production", + "description": "Production ML workloads", + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": "single", + "pipeline_server": True, + }, +} + +DATA_CONNECTIONS = { + "ml-production": [ + { + "name": "prod-model-store", + "type": "S3", + "bucket": "ml-models-prod", + "endpoint": "https://s3.us-east-1.amazonaws.com", + "region": "us-east-1", + }, + ], +} + +SERVING_RUNTIMES = { + "__platform_templates__": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "REST", + "supported_model_formats": [ + {"name": "vLLM", "version": "1", "autoSelect": True} + ], + }, + { + "name": "caikit-tgis-runtime", + "display_name": "Caikit+TGIS ServingRuntime", + "model_formats": ["caikit"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "gRPC", + }, + ], + "ml-production": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + { + "name": "nim-serving-runtime", + "display_name": "NVIDIA NIM ServingRuntime", + "model_formats": ["NIM"], + "requires_instantiation": False, + "source": "nim-account", + "api_protocol": "REST", + }, + { + "name": "ovms-1", + "display_name": "OpenVINO Model Server", + "model_formats": ["openvino_ir", "onnx"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + ], +} + +INFERENCE_SERVICES = { + "ml-production": { + "text-gen-legacy": { + "name": "text-gen-legacy", + "namespace": "ml-production", + "runtime": "vllm-runtime", + "model_format": "vLLM", + "storage_uri": "hf://mistralai/Mistral-7B-Instruct-v0.3", + "display_name": "Mistral 7B Legacy", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "16Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "PredictorFailed", + "message": "Predictor pod is not ready", + }, + { + "type": "PredictorReady", + "status": "False", + "reason": "ContainerCrashLoop", + "message": "Container kserve-container terminated: " + "OOMKilled (exit code 137). 5 restarts.", + }, + { + "type": "IngressReady", + "status": "True", + "reason": "IngressReady", + "message": "Ingress is ready", + }, + ], + "age": "3d", + }, + "nim-llama-prod": { + "name": "nim-llama-prod", + "namespace": "ml-production", + "runtime": "nim-serving-runtime", + "model_format": "NIM", + "storage_uri": "nim://meta/llama-3.1-8b-instruct", + "display_name": "Llama 3.1 8B (NIM)", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "32Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "RuntimeNotReady", + "message": "ServingRuntime 'nim-serving-runtime' " + "is not in ready state", + }, + { + "type": "PredictorReady", + "status": "Unknown", + "reason": "PodNotCreated", + "message": "Predictor pod has not been created. " + "Waiting for ServingRuntime to become ready.", + }, + { + "type": "IngressReady", + "status": "Unknown", + "reason": "PredictorNotReady", + "message": "Waiting for predictor to become ready", + }, + ], + "age": "1d", + }, + }, +} + +DEPLOYED_MODELS = {} + +WORKBENCHES = { + "ml-production": [ + { + "name": "data-exploration-nb", + "display_name": "Data Exploration", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Running", + "cpu_request": "1", + "memory_request": "8Gi", + "gpu_count": 0, + "pvc_name": "data-exploration-nb-pvc", + "pvc_size": "20Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-10T09:00:00Z", + }, + { + "name": "model-training-nb", + "display_name": "Model Training", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Stopped", + "cpu_request": "4", + "memory_request": "16Gi", + "gpu_count": 1, + "pvc_name": "model-training-nb-pvc", + "pvc_size": "50Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-15T14:00:00Z", + }, + ], +} + +PIPELINE_SERVERS = { + "ml-production": { + "configured": True, + "data_connection": "prod-model-store", + "status": "Ready", + "database": "MariaDB", + }, +} + +NOTEBOOK_IMAGES = [ + {"name": "jupyter-pytorch-ubi9-python-3.9-2024.1", "display_name": "PyTorch 2024.1", "packages": ["torch", "transformers"]}, + {"name": "jupyter-tensorflow-ubi9-python-3.9-2024.1", "display_name": "TensorFlow 2024.1", "packages": ["tensorflow"]}, + {"name": "jupyter-datascience-ubi9-python-3.9-2024.1", "display_name": "Standard Data Science", "packages": ["pandas", "scikit-learn"]}, + {"name": "jupyter-minimal-ubi9-python-3.9-2024.1", "display_name": "Minimal Python", "packages": []}, +] + + +# ── Project tools ──────────────────────────────────────────────────────── + + +@mcp.tool() +def list_data_science_projects() -> str: + """List all RHOAI Data Science Projects on the cluster.""" + projects = [] + for name, proj in PROJECTS.items(): + projects.append({ + "name": name, + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + }) + return json.dumps(projects, indent=2) + + +@mcp.tool() +def create_data_science_project( + name: str, + display_name: str, + description: str = "", +) -> str: + """Create a new RHOAI Data Science Project (namespace with dashboard labels).""" + if name in PROJECTS: + raise ValueError( + f"Project '{name}' already exists. Choose a different name " + "or configure the existing project." + ) + if not name.replace("-", "").replace("_", "").isalnum() or len(name) > 63: + raise ValueError( + f"Invalid project name '{name}'. Must be DNS-compatible: " + "lowercase alphanumeric and hyphens, max 63 chars." + ) + + PROJECTS[name] = { + "name": name, + "display_name": display_name, + "description": description, + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": None, + "pipeline_server": False, + } + DATA_CONNECTIONS[name] = [] + SERVING_RUNTIMES[name] = [] + INFERENCE_SERVICES[name] = {} + + return json.dumps({ + "status": "created", + "name": name, + "display_name": display_name, + "namespace": name, + "labels": {"opendatahub.io/dashboard": "true"}, + }) + + +@mcp.tool() +def get_project_details(name: str) -> str: + """Get detailed information about an RHOAI Data Science Project.""" + if name not in PROJECTS: + raise ValueError(f"Project '{name}' not found") + proj = PROJECTS[name] + dc_count = len(DATA_CONNECTIONS.get(name, [])) + isvc_count = len(INFERENCE_SERVICES.get(name, {})) + return json.dumps({ + "name": proj["name"], + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "labels": proj["labels"], + "data_connections": dc_count, + "inference_services": isvc_count, + "model_serving_mode": proj.get("model_serving_mode"), + "pipeline_server": proj.get("pipeline_server", False), + }) + + +@mcp.tool() +def get_project_status(namespace: str) -> str: + """Get comprehensive status of an RHOAI Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Project '{namespace}' not found") + proj = PROJECTS[namespace] + dcs = DATA_CONNECTIONS.get(namespace, []) + isvcs = INFERENCE_SERVICES.get(namespace, {}) + return json.dumps({ + "namespace": namespace, + "display_name": proj["display_name"], + "status": "Active", + "components": { + "data_connections": len(dcs), + "inference_services": len(isvcs), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + "pipeline_server": "configured" if proj.get("pipeline_server") else "not configured", + }, + }) + + +# ── Data connection tools ──────────────────────────────────────────────── + + +@mcp.tool() +def create_s3_data_connection( + namespace: str, + name: str, + bucket: str, + endpoint: str, + access_key: str, + secret_key: str, + region: str = "", +) -> str: + """Create an S3-compatible data connection in an RHOAI project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + existing = DATA_CONNECTIONS.get(namespace, []) + if any(dc["name"] == name for dc in existing): + raise ValueError( + f"Data connection '{name}' already exists in namespace '{namespace}'" + ) + + dc = { + "name": name, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + "region": region, + } + DATA_CONNECTIONS.setdefault(namespace, []).append(dc) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + }) + + +@mcp.tool() +def list_data_connections(namespace: str) -> str: + """List data connections in an RHOAI project namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + return json.dumps(dcs, indent=2) + + +# ── Model serving tools ───────────────────────────────────────────────── + + +@mcp.tool() +def set_model_serving_mode(namespace: str, mode: str) -> str: + """Enable model serving on a Data Science Project (single or multi mode).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + if mode not in ("single", "multi"): + raise ValueError(f"Invalid mode '{mode}'. Must be 'single' or 'multi'.") + + PROJECTS[namespace]["model_serving_mode"] = mode + + if not SERVING_RUNTIMES.get(namespace): + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + SERVING_RUNTIMES[namespace] = [ + {**t, "requires_instantiation": False, "source": "existing"} + for t in templates + ] + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "mode": mode, + }) + + +@mcp.tool() +def list_serving_runtimes( + namespace: str, + include_templates: bool = False, +) -> str: + """List available ServingRuntimes in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + runtimes = list(SERVING_RUNTIMES.get(namespace, [])) + if include_templates: + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + existing_names = {r["name"] for r in runtimes} + for t in templates: + if t["name"] not in existing_names: + runtimes.append(t) + + return json.dumps(runtimes, indent=2) + + +# ── Inference service tools ────────────────────────────────────────────── + + +@mcp.tool() +def deploy_model( + name: str, + namespace: str, + runtime: str, + model_format: str, + storage_uri: str, + display_name: str = "", + min_replicas: int = 1, + max_replicas: int = 1, + cpu_request: str = "1", + cpu_limit: str = "2", + memory_request: str = "4Gi", + memory_limit: str = "8Gi", + gpu_count: int = 0, +) -> str: + """Deploy an AI/ML model as a KServe InferenceService.""" + if namespace not in PROJECTS: + raise ValueError( + f"Namespace '{namespace}' is not a Data Science Project. " + "Create one via create_data_science_project first." + ) + + ns_runtimes = SERVING_RUNTIMES.get(namespace, []) + runtime_names = [r["name"] for r in ns_runtimes] + if runtime not in runtime_names: + available = ", ".join(runtime_names) or "none" + raise ValueError( + f"ServingRuntime '{runtime}' not found in namespace '{namespace}'. " + f"Available runtimes: {available}" + ) + + endpoint = f"https://{name}-{namespace}.apps.ocp-cluster.example.com" + isvc = { + "name": name, + "namespace": namespace, + "runtime": runtime, + "model_format": model_format, + "storage_uri": storage_uri, + "display_name": display_name or name, + "gpu_count": gpu_count, + "cpu_request": cpu_request, + "memory_request": memory_request, + "min_replicas": min_replicas, + "max_replicas": max_replicas, + "ready": True, + "url": endpoint, + "conditions": [ + {"type": "Ready", "status": "True", "reason": "Ready", "message": ""}, + {"type": "PredictorReady", "status": "True", "reason": "PodReady", "message": ""}, + {"type": "IngressReady", "status": "True", "reason": "IngressReady", "message": ""}, + ], + "age": "0s", + } + + INFERENCE_SERVICES.setdefault(namespace, {})[name] = isvc + DEPLOYED_MODELS[f"{namespace}/{name}"] = isvc + + return json.dumps({ + "status": "deployed", + "name": name, + "namespace": namespace, + "runtime": runtime, + "endpoint": endpoint, + "ready": True, + }) + + +@mcp.tool() +def list_inference_services( + namespace: str, + verbosity: str = "standard", +) -> str: + """List deployed InferenceServices in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + isvcs = INFERENCE_SERVICES.get(namespace, {}) + results = [] + for isvc_name, isvc in isvcs.items(): + entry = { + "name": isvc["name"], + "runtime": isvc["runtime"], + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "age": isvc.get("age", ""), + } + if verbosity == "full": + entry["conditions"] = isvc.get("conditions", []) + entry["storage_uri"] = isvc.get("storage_uri", "") + entry["gpu_count"] = isvc.get("gpu_count", 0) + results.append(entry) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def get_inference_service( + name: str, + namespace: str, + verbosity: str = "standard", +) -> str: + """Get detailed status of a specific InferenceService.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + + isvc = isvcs[name] + result = { + "name": isvc["name"], + "namespace": isvc["namespace"], + "runtime": isvc["runtime"], + "model_format": isvc.get("model_format", ""), + "storage_uri": isvc.get("storage_uri", ""), + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "conditions": isvc.get("conditions", []), + "gpu_count": isvc.get("gpu_count", 0), + "replicas": {"min": isvc.get("min_replicas", 1), "max": isvc.get("max_replicas", 1)}, + "resources": { + "cpu_request": isvc.get("cpu_request", "1"), + "memory_request": isvc.get("memory_request", "4Gi"), + "memory_limit": isvc.get("memory_limit", "8Gi"), + }, + "age": isvc.get("age", ""), + } + return json.dumps(result, indent=2) + + +@mcp.tool() +def get_model_endpoint(name: str, namespace: str) -> str: + """Get the inference endpoint URL for a deployed model.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + isvc = isvcs[name] + if not isvc["ready"]: + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": "", + "error": "InferenceService is not ready. Check conditions for details.", + }) + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": isvc["url"], + }) + + +# ── Workbench tools ────────────────────────────────────────────────────── + + +@mcp.tool() +def list_workbenches(namespace: str) -> str: + """List workbenches (Jupyter notebooks) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + return json.dumps(wbs, indent=2) + + +@mcp.tool() +def create_workbench( + namespace: str, + name: str, + display_name: str = "", + image: str = "jupyter-datascience-ubi9-python-3.9-2024.1", + cpu_request: str = "1", + memory_request: str = "4Gi", + gpu_count: int = 0, + pvc_size: str = "20Gi", +) -> str: + """Create a new workbench (Jupyter notebook) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + valid_images = [img["name"] for img in NOTEBOOK_IMAGES] + if image not in valid_images: + raise ValueError( + f"Image '{image}' not found. Available: {', '.join(valid_images)}" + ) + + wb = { + "name": name, + "display_name": display_name or name, + "image": image, + "status": "Running", + "cpu_request": cpu_request, + "memory_request": memory_request, + "gpu_count": gpu_count, + "pvc_name": f"{name}-pvc", + "pvc_size": pvc_size, + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-03-02T12:00:00Z", + } + WORKBENCHES.setdefault(namespace, []).append(wb) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "image": image, + "pvc": f"{name}-pvc", + }) + + +@mcp.tool() +def stop_workbench(namespace: str, name: str) -> str: + """Stop a running workbench (preserves data).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Stopped" + return json.dumps({"status": "stopped", "name": name, "namespace": namespace}) + + +@mcp.tool() +def start_workbench(namespace: str, name: str) -> str: + """Start a stopped workbench.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Running" + return json.dumps({"status": "running", "name": name, "namespace": namespace}) + + +@mcp.tool() +def get_workbench_url(namespace: str, name: str) -> str: + """Get the URL for accessing a running workbench.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + if wb["status"] != "Running": + return json.dumps({ + "namespace": namespace, + "name": name, + "url": "", + "error": f"Workbench is not running (status: {wb['status']}). Start it first.", + }) + url = f"https://{name}-{namespace}.apps.ocp-cluster.example.com" + return json.dumps({ + "namespace": namespace, + "name": name, + "url": url, + "status": wb["status"], + }) + + +@mcp.tool() +def list_workbench_storage(namespace: str, name: str) -> str: + """List PVC details for a workbench including size, usage, access mode.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + volumes = [ + { + "pvc_name": wb.get("pvc_name", f"{name}-pvc"), + "size": wb.get("pvc_size", "20Gi"), + "usage": "12Gi", # Mock usage + "access_mode": wb.get("pvc_access_mode", "ReadWriteOnce"), + "mount_path": "/opt/app-root/data", + }, + ] + # Include additional volumes if any + for extra in wb.get("extra_volumes", []): + volumes.append(extra) + return json.dumps({ + "namespace": namespace, + "workbench": name, + "volumes": volumes, + }, indent=2) + + +@mcp.tool() +def add_workbench_storage( + namespace: str, + workbench_name: str, + pvc_name: str, + mount_path: str, + size: str, +) -> str: + """Add additional storage volume to a workbench.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == workbench_name), None) + if not wb: + raise ValueError(f"Workbench '{workbench_name}' not found in '{namespace}'") + extra = wb.setdefault("extra_volumes", []) + extra.append({ + "pvc_name": pvc_name, + "size": size, + "usage": "0", + "access_mode": "ReadWriteOnce", + "mount_path": mount_path, + }) + return json.dumps({ + "status": "added", + "namespace": namespace, + "workbench": workbench_name, + "pvc_name": pvc_name, + "mount_path": mount_path, + "size": size, + }) + + +@mcp.tool() +def delete_workbench(namespace: str, name: str) -> str: + """Delete a workbench. WARNING: PVC data may be lost if not backed up.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wbs.remove(wb) + return json.dumps({ + "status": "deleted", + "name": name, + "namespace": namespace, + "warning": "Associated PVC data has been deleted", + }) + + +@mcp.tool() +def list_notebook_images() -> str: + """List available notebook images for workbench creation.""" + return json.dumps(NOTEBOOK_IMAGES, indent=2) + + +# ── Pipeline server tools ─────────────────────────────────────────────── + + +@mcp.tool() +def configure_pipeline_server( + namespace: str, + data_connection: str, + database: str = "MariaDB", +) -> str: + """Configure a pipeline server for a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + if not any(dc["name"] == data_connection for dc in dcs): + available = [dc["name"] for dc in dcs] + raise ValueError( + f"Data connection '{data_connection}' not found. Available: {available}" + ) + + PIPELINE_SERVERS[namespace] = { + "configured": True, + "data_connection": data_connection, + "status": "Ready", + "database": database, + } + PROJECTS[namespace]["pipeline_server"] = True + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "data_connection": data_connection, + "database": database, + }) + + +@mcp.tool() +def get_pipeline_server_status(namespace: str) -> str: + """Get the status of the pipeline server in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + ps = PIPELINE_SERVERS.get(namespace) + if not ps: + return json.dumps({"namespace": namespace, "configured": False}) + return json.dumps({ + "namespace": namespace, + "configured": ps["configured"], + "data_connection": ps["data_connection"], + "status": ps["status"], + "database": ps["database"], + }) + + +# ── Serving runtime creation ──────────────────────────────────────────── + + +@mcp.tool() +def create_serving_runtime( + namespace: str, + name: str, + display_name: str = "", + model_formats: list = None, + container_image: str = "", + container_port: int = 8080, + multi_model: bool = False, + api_protocol: str = "REST", +) -> str: + """Create a custom ServingRuntime in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + if not model_formats: + raise ValueError("model_formats must specify at least one model format") + + runtime = { + "name": name, + "display_name": display_name or name, + "model_formats": model_formats, + "requires_instantiation": False, + "source": "custom", + "api_protocol": api_protocol, + "container_image": container_image, + "container_port": container_port, + "multi_model": multi_model, + } + SERVING_RUNTIMES.setdefault(namespace, []).append(runtime) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "model_formats": model_formats, + }) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/common-issues.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/workbench-manage/SKILL.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/workbench-manage/SKILL.md new file mode 100644 index 00000000..5ad0fa58 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/workbench-manage/SKILL.md @@ -0,0 +1,396 @@ +--- +name: workbench-manage +description: | + Create and manage Jupyter notebook workbenches on OpenShift AI with image selection, resource configuration, PVC storage, and lifecycle management. + + Use when: + - "Create a notebook workbench" + - "Spin up a Jupyter environment for data science" + - "Start / stop my workbench" + - "What notebook images are available?" + - "Delete a workbench I no longer need" + + Handles Notebook CR lifecycle: create with configurable images and resources, start/stop, attach storage, and delete with data loss warnings. + + NOT for deploying models (use /model-deploy). + NOT for creating projects (use /ds-project-setup). + NOT for managing pipelines (use /pipeline-manage). +color: blue +model: inherit +metadata: + author: "Red Hat Ecosystem Engineering" + version: "1.0" +--- + +# /workbench-manage Skill + +Create and manage Jupyter notebook workbenches on Red Hat OpenShift AI. Handles the full workbench lifecycle: listing available notebook images, creating Notebook CRs with configurable CPU/memory/GPU resources, provisioning PVC storage, starting and stopping workbenches, and deleting them with proper data loss warnings. + +## Prerequisites + +**Required MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) + +**Required MCP Tools** (from rhoai): +- `list_data_science_projects` - Validate namespace is an RHOAI Data Science Project +- `list_notebook_images` - List available notebook container images (PyTorch, TensorFlow, Standard DS, etc.) +- `list_workbenches` - List existing workbenches in a project +- `get_workbench` - Get workbench details (status, image, resources, storage) +- `create_workbench` - Create a new Notebook CR with image, resources, and storage +- `start_workbench` - Start a stopped workbench +- `stop_workbench` - Stop a running workbench +- `delete_workbench` - Delete a workbench +- `get_workbench_url` - Get the OAuth-protected notebook URL +- `list_storage` - List PVCs in the project +- `create_storage` - Create a PVC for workbench storage +- `delete_storage` - Delete a PVC +- `list_data_connections` - List data connections available to attach + +**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools** (from openshift): +- `resources_get` (from openshift) - Inspect Notebook CR details, check node GPU availability +- `events_list` (from openshift) - Check pod events when workbench is stuck + +**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). + +**Additional cluster requirements**: +- Target namespace is an RHOAI Data Science Project (label: `opendatahub.io/dashboard: "true"`) + +## When to Use This Skill + +**Use this skill when you need to:** +- Create a new Jupyter notebook workbench for a data scientist +- List available notebook images (PyTorch, TensorFlow, Standard Data Science, etc.) +- Start or stop an existing workbench +- List workbenches in a project and check their status +- Delete a workbench and its associated storage +- Provision persistent storage for a workbench + +**Do NOT use this skill when:** +- You need to create a Data Science Project first (use `/ds-project-setup`) +- You want to deploy a model for inference (use `/model-deploy`) +- You need to manage data science pipelines (use `/pipeline-manage`) +- You need to troubleshoot a model deployment (use `/debug-inference`) + +## Workflow + +### Step 1: Determine Intent + +**Ask the user what they want to do:** +- **Create** a new workbench +- **Start / Stop** an existing workbench +- **List** workbenches in a project +- **Delete** a workbench + +**Ask for the target namespace** (required for all operations). + +**Validate namespace** is a Data Science Project: + +**MCP Tool**: `list_data_science_projects` (from rhoai) + +**Parameters**: none + +Verify the user-specified namespace appears in the project list. If not, report: "Namespace `[name]` is not an RHOAI Data Science Project. Use `/ds-project-setup` to create one." + +**Route to the appropriate sub-workflow:** +- Create -> Step 2 +- Start/Stop -> Step 5 +- List -> Use `list_workbenches`, display results, done +- Delete -> Step 6 + +### Step 2: Gather Configuration (Create) + +**List available notebook images:** + +**MCP Tool**: `list_notebook_images` (from rhoai) + +**Parameters**: none + +**Present available images** in a table: + +| Image Name | Description | +|------------|-------------| +| [name] | [description] | + +**Ask the user for workbench configuration:** +- **Workbench name**: DNS-compatible name (lowercase, hyphens, max 63 chars) +- **Image**: Selection from the available images list +- **CPU**: Number of CPU cores (default: 2) +- **Memory**: Memory allocation (default: 8Gi) +- **Storage size**: PVC size for persistent storage (default: 20Gi) +- **GPU** (optional): Number of GPUs to attach (e.g., 1) + +**Display configuration table:** + +| Setting | Value | +|---------|-------| +| Workbench name | [name] | +| Namespace | [namespace] | +| Image | [image_name] | +| CPU | [cpu] cores | +| Memory | [memory] | +| Storage | [storage_size] | +| GPU | [gpu_count or none] | + +Proceed automatically without waiting for user input. + +### Step 3: Provision Storage (Create) + +**Check existing storage:** + +**MCP Tool**: `list_storage` (from rhoai) + +**Parameters**: +- `namespace`: target namespace - REQUIRED + +If a suitable PVC already exists, ask user if they want to reuse it or create a new one. + +**Create PVC for workbench storage:** + +**MCP Tool**: `create_storage` (from rhoai) + +**Parameters**: +- `namespace`: target namespace - REQUIRED +- `name`: PVC name (default: `[workbench-name]-storage`) - REQUIRED +- `size`: storage size from Step 2 (e.g., `"20Gi"`) - REQUIRED +- `access_mode`: `"ReadWriteOnce"` - REQUIRED (default, single-pod access) + +**Verify creation:** + +**MCP Tool**: `list_storage` (from rhoai) + +**Parameters**: +- `namespace`: target namespace - REQUIRED + +Confirm the PVC appears and is in `Bound` or `Pending` state. + +**Error Handling**: +- If PVC name already exists -> Ask: "PVC `[name]` already exists. Reuse it or create with a different name?" +- If StorageClass not available -> Report: "Default StorageClass not configured. Contact your cluster administrator." +- If quota exceeded -> Report namespace storage quota limits + +### Step 4: Create Workbench (Create) + +**MCP Tool**: `create_workbench` (from rhoai) + +**Parameters**: +- `namespace`: target namespace - REQUIRED +- `name`: workbench name from Step 2 - REQUIRED +- `image`: selected notebook image name from Step 2 - REQUIRED +- `cpu`: CPU cores (e.g., `"2"`) - REQUIRED +- `memory`: memory allocation (e.g., `"8Gi"`) - REQUIRED +- `storage_size`: PVC storage size (e.g., `"20Gi"`) - REQUIRED + +**Monitor workbench startup** by polling status: + +**MCP Tool**: `get_workbench` (from rhoai) + +**Parameters**: +- `namespace`: target namespace - REQUIRED +- `name`: workbench name - REQUIRED + +Check until status shows the workbench is running. If status does not become ready within a reasonable polling window (3-4 checks), proceed to report current status and advise user to check back. + +**Get notebook URL:** + +**MCP Tool**: `get_workbench_url` (from rhoai) + +**Parameters**: +- `namespace`: target namespace - REQUIRED +- `name`: workbench name - REQUIRED + +**Error Handling**: +- If workbench name already exists -> Report: "Workbench `[name]` already exists. Choose a different name or manage the existing one." +- If image not found -> Re-run `list_notebook_images` and suggest available alternatives +- If RBAC error -> Report insufficient permissions to create Notebook CRs +- If GPU unavailable -> Report: "Requested GPU resources not available on cluster nodes. Reduce GPU count or wait for resources." + +**Report to user:** + +| Detail | Value | +|--------|-------| +| Workbench | [name] | +| Status | [Running / Starting] | +| Image | [image] | +| Resources | [cpu] CPU, [memory] RAM, [gpu] GPU | +| Storage | [storage_size] | +| URL | [notebook_url] | + +**Suggest next steps:** +- Access the notebook at the provided URL (OpenShift authentication required) +- Use `/ds-project-setup` to add data connections to the project +- Use `/model-deploy` when ready to deploy a trained model + +### Step 5: Manage Lifecycle (Start/Stop) + +**List workbenches to identify the target:** + +**MCP Tool**: `list_workbenches` (from rhoai) + +**Parameters**: +- `namespace`: target namespace - REQUIRED + +If user did not specify a workbench name, present the list and ask which one to manage. + +**For Start:** + +Confirm the workbench is currently stopped. If already running, report its URL and current status. + +**MCP Tool**: `start_workbench` (from rhoai) + +**Parameters**: +- `namespace`: target namespace - REQUIRED +- `name`: workbench name - REQUIRED + +**MCP Tool**: `get_workbench_url` (from rhoai) + +**Parameters**: +- `namespace`: target namespace - REQUIRED +- `name`: workbench name - REQUIRED + +**Output to user**: "Workbench `[name]` started. Access it at: [url]" + +**For Stop:** + +Proceed automatically without waiting for user input. + +**MCP Tool**: `stop_workbench` (from rhoai) + +**Parameters**: +- `namespace`: target namespace - REQUIRED +- `name`: workbench name - REQUIRED + +**Verify state change:** + +**MCP Tool**: `get_workbench` (from rhoai) + +**Parameters**: +- `namespace`: target namespace - REQUIRED +- `name`: workbench name - REQUIRED + +**Output to user**: "Workbench `[name]` stopped. Persistent storage is preserved. Use `/workbench-manage` to start it again." + +**Error Handling**: +- If workbench not found -> List available workbenches and ask user to select +- If already in target state -> Report current state (e.g., "Workbench is already running") + +### Step 6: Delete Workbench + +**Get workbench details:** + +**MCP Tool**: `get_workbench` (from rhoai) + +**Parameters**: +- `namespace`: target namespace - REQUIRED +- `name`: workbench name - REQUIRED + +**Display workbench details and data loss warning:** + +| Detail | Value | +|--------|-------| +| Workbench | [name] | +| Status | [Running / Stopped] | +| Image | [image] | +| Storage | [pvc_name] ([size]) | + +**WARNING**: Deleting this workbench will remove the Notebook CR. If the workbench is running, it will be stopped first. Any unsaved notebook work will be lost. + +**Ask**: "Delete workbench `[name]`? This action cannot be undone. (yes/no)" + +Proceed automatically without waiting for user input. + +**MCP Tool**: `delete_workbench` (from rhoai) + +**Parameters**: +- `namespace`: target namespace - REQUIRED +- `name`: workbench name - REQUIRED + +**Associated storage cleanup** (separate confirmation): + +**Ask**: "The PVC `[pvc_name]` ([size]) associated with this workbench still exists. Delete it too? WARNING: All data in this volume will be permanently lost. (yes/no)" + +Proceed automatically without waiting for user input. + +If user confirms PVC deletion: + +**MCP Tool**: `delete_storage` (from rhoai) + +**Parameters**: +- `namespace`: target namespace - REQUIRED +- `name`: PVC name - REQUIRED + +If user declines, report: "PVC `[pvc_name]` preserved. It can be reattached to a new workbench." + +**Output to user**: "Workbench `[name]` deleted. [PVC deleted / PVC preserved]." + +## Common Issues + +For common issues (GPU scheduling, OOMKilled, image pull errors, RBAC), see [common-issues.md](../references/common-issues.md). + +### Issue 1: Notebook Image Not Found + +**Error**: `create_workbench` fails with image not found or image reference is invalid + +**Cause**: The selected image name does not match any available notebook image, or the image registry is unreachable. + +**Solution:** +1. Run `list_notebook_images` to see current available images +2. Verify the exact image name (case-sensitive) +3. If no images are listed, the RHOAI operator may not have imported notebook images -- contact cluster administrator + +### Issue 2: PVC Binding Failure + +**Error**: PVC remains in `Pending` state, workbench cannot start + +**Cause**: The default StorageClass does not support the requested access mode, or no StorageClass is configured. + +**Solution:** +1. Check available StorageClasses via `resources_get` (from openshift) on `storageclasses.storage.k8s.io` +2. Use `ReadWriteOnce` access mode (most widely supported) +3. If `ReadWriteMany` is required, verify the StorageClass supports it (e.g., NFS, CephFS) +4. Contact cluster administrator if no StorageClass is available + +### Issue 3: Workbench Stuck in Starting + +**Error**: Workbench status remains in a starting/initializing state for an extended period + +**Cause**: Pod scheduling issues, image pull errors, or resource constraints. + +**Solution:** +1. Use `events_list` (from openshift) filtered by namespace to check for pod events +2. Common causes: + - `ImagePullBackOff`: Image registry unreachable or credentials missing + - `Insufficient cpu/memory`: Reduce resource requests or free up cluster resources + - `FailedScheduling`: Node taints or affinity rules preventing scheduling +3. If GPU is requested, verify GPU nodes have available capacity + +## Dependencies + +### MCP Tools +See [Prerequisites](#prerequisites) for the complete list of required and optional MCP tools. + +### Related Skills +- `/ds-project-setup` - Create a Data Science Project (prerequisite: namespace must exist) +- `/model-deploy` - Deploy a trained model from the workbench +- `/ai-observability` - Check GPU inventory before requesting GPU workbenches + +### Reference Documentation +- [skill-conventions.md](../references/skill-conventions.md) - Shared prerequisite, HITL, and security conventions + +## Example Usage + +**User**: "Create a PyTorch notebook workbench in my ml-team project with 4 CPUs and a GPU" + +**Skill response**: Validates `ml-team` is an RHOAI project, lists available notebook images, presents configuration table (PyTorch image, 4 CPU, 8Gi memory, 1 GPU, 20Gi storage), provisions PVC storage, creates workbench, monitors startup, and returns the notebook URL. + +## Critical: Human-in-the-Loop Requirements + +See [skill-conventions.md](../references/skill-conventions.md) for general HITL and security conventions. + +**Skill-specific checkpoints:** +- Before creating workbench (Step 4): display full configuration table, confirm +- Before stopping a workbench (Step 5): warn about unsaved work, confirm +- Before deleting a workbench (Step 6): display details, warn about data loss, confirm +- Before deleting associated PVC (Step 6): separate confirmation with permanent data loss warning +- **NEVER** auto-delete workbenches or storage +- **NEVER** stop a running workbench without confirmation (user may have unsaved notebook work) diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/instruction.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/instruction.md new file mode 100644 index 00000000..39b97c27 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/instruction.md @@ -0,0 +1,13 @@ +# Workbench Management Task + +You are an AI engineer on Red Hat OpenShift AI. Your data science team needs workbenches set up for model development, and some existing workbenches need cleanup. + +## Requirements +- Review existing workbenches in the project: their status, resource usage, and notebook images +- Plan a new workbench for a data scientist who needs PyTorch with 4 CPUs, 16Gi memory, and 50Gi persistent storage +- Identify any stopped or unused workbenches that should be cleaned up to free resources +- Document the lifecycle procedures: how to stop a workbench to save resources, restart it, and safely delete one + +Document your workbench assessment, creation plan, and cleanup recommendations in `/root/report.md`. + +Use MCP tools to interact with the platform. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/solution/solve.sh b/evaluation/with_skills/rh-ai-engineer__workbench-manage/solution/solve.sh new file mode 100644 index 00000000..49e5cc92 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/solution/solve.sh @@ -0,0 +1,25 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Workbench Creation Plan + +## Workbench: fraud-analysis +Project/Namespace: fraud-detection + +### Storage (create_storage) +- PVC: 20Gi, access mode: ReadWriteOnce +- Namespace validated via list_data_science_projects + +### Configuration (create_workbench) +- Image: Jupyter Data Science Notebook (from list_notebook_images) +- CPU: 2 +- Memory: 8Gi +- Storage: 20Gi + +### Lifecycle +- start_workbench / stop_workbench for running/stopped state +- get_workbench_url: OAuth-protected notebook URL for access + +### Delete Warnings +- delete_workbench: Data loss warning — unsaved work lost, action cannot be undone +- delete_storage: Separate confirmation for PVC deletion — permanent data loss +REPORT_EOF diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/task.toml b/evaluation/with_skills/rh-ai-engineer__workbench-manage/task.toml new file mode 100644 index 00000000..6c538b09 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-ai-engineer__workbench-manage" +name = "rh-ai-engineer Workbench Management Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-ai-engineer", "workbench-manage", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/tests/llm_judge.py b/evaluation/with_skills/rh-ai-engineer__workbench-manage/tests/llm_judge.py new file mode 100644 index 00000000..b7792ec1 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "stop_vs_delete_data", "file": "/root/report.md", "question": "Does the report explain that stopping a workbench preserves PVC data while deleting requires separate storage decision?", "reference": "A skilled report distinguishes stop (preserves) from delete (data loss risk). An unskilled report treats stop and delete equivalently."}, + {"id": "notebook_image_discovery", "file": "/root/report.md", "question": "Does the report describe discovering or listing available notebook images before creating a workbench?", "reference": "A skilled report lists available notebook images (via list_notebook_images or equivalent) to guide workbench creation. An unskilled report skips image discovery and assumes a default."}, + {"id": "storage_access_mode_awareness", "file": "/root/report.md", "question": "Does the report mention the PVC access mode (ReadWriteOnce or RWO) when describing workbench storage configuration or provisioning?", "reference": "A skilled report specifies the storage access mode (ReadWriteOnce) for PVC provisioning, showing awareness of storage class constraints. An unskilled report describes storage size but omits access mode details."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/tests/test.sh b/evaluation/with_skills/rh-ai-engineer__workbench-manage/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/tests/test_outputs.py b/evaluation/with_skills/rh-ai-engineer__workbench-manage/tests/test_outputs.py new file mode 100644 index 00000000..59f74eec --- /dev/null +++ b/evaluation/with_skills/rh-ai-engineer__workbench-manage/tests/test_outputs.py @@ -0,0 +1,73 @@ +""" +Tests for rh-ai-engineer__workbench-manage per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ["workbench", "notebook"]), ( + "report should mention workbench or notebook" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 200, "report should have substantial content" + + +class TestSkillDependent: + def test_stop_preserves_data(self): + """Skill teaches: stopping a workbench preserves PVC data; only delete removes it.""" + c = read_report().lower() + assert any(t in c for t in [ + "stop", "preserve", "data", "pvc", "storage", + "stopped", "restart", "start again", + ]), "should explain that stop preserves data vs delete" + + def test_delete_pvc_warning(self): + """Skill teaches: deleting workbench requires separate confirmation for PVC; warn about permanent data loss.""" + c = read_report().lower() + assert any(t in c for t in [ + "pvc", "delete", "data loss", "permanent", "warning", + "volume", "storage", "backup", "cannot be undone", + ]), "should warn about PVC/data loss on deletion" + + def test_lifecycle_operations(self): + """Skill teaches: create, start, stop, delete with distinct implications.""" + c = read_report().lower() + ops = sum(1 for t in ["start", "stop", "delet", "creat"] if t in c) + assert ops >= 2, "should describe lifecycle operations (create, start, stop, delete)" + + def test_list_notebook_images_tool(self): + """Skill teaches: list_notebook_images MCP tool to discover available notebook images.""" + c = read_report().lower() + assert any(t in c for t in ["list_notebook_images", "notebook images", "available images"]), ( + "should reference list_notebook_images tool (skill)" + ) + + def test_gpu_tuning_awareness(self): + """Docs teach GPU scheduling triage and OOM mitigation using + model/context-size controls for workbenches with GPU resources. + Without docs, agents don't address GPU resource tuning.""" + c = read_report().lower() + assert any(t in c for t in [ + "gpu", "oom", "context size", "max-model-len", "memory", + ]) and any(t in c for t in ["workbench", "notebook", "resource", "gpu"]), ( + "should address GPU/OOM tuning for workbench resources" + ) diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/Dockerfile b/evaluation/with_skills/rh-developer__containerize-deploy/environment/Dockerfile new file mode 100644 index 00000000..1cbfefcf --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-developer__containerize-deploy/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/skills/containerize-deploy/SKILL.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/skills/containerize-deploy/SKILL.md new file mode 100644 index 00000000..67487f6d --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/skills/containerize-deploy/SKILL.md @@ -0,0 +1,477 @@ +--- +name: containerize-deploy +description: | + Complete end-to-end workflow for containerizing and deploying applications to OpenShift or standalone RHEL systems. Orchestrates /detect-project, /s2i-build, /deploy, /helm-deploy, and /rhel-deploy skills with user confirmation checkpoints at each phase. Supports S2I, Podman, Helm deployment strategies for OpenShift, and Podman/native deployments for RHEL hosts. Use this skill when user wants to go from source code to running application in one guided workflow. Supports resume after interruption and rollback on failure. Triggers on /containerize-deploy command. +model: inherit +color: green +metadata: + user_invocable: "true" +--- + +# /containerize-deploy Skill + +Provide a complete, guided workflow from local source code to running application on OpenShift or standalone RHEL systems. This skill orchestrates `/detect-project`, `/s2i-build`, `/deploy`, `/helm-deploy`, and `/rhel-deploy` with clear user checkpoints at each phase. + +## Overview + +``` +[Intro] → [Detect] → [Target] → [Strategy] ──┬─→ [OpenShift: S2I/Podman/Helm] ──┬─→ [Complete] + └─→ [RHEL: /rhel-deploy] ──────────┘ +``` + +## When to Use This Skill + +Use `/containerize-deploy` when a user wants a complete guided workflow from source code to running application on OpenShift or standalone RHEL systems. This skill orchestrates project detection, build strategy selection, and deployment with user confirmation at each phase. + +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +## Workflow + +### Phase 0: Introduction + +Present the workflow overview and available deployment targets/strategies. Ask: **Ready to begin?** (yes/no) + +Proceed automatically without waiting for user input. + +### Phase 1: Project Detection + +Execute the `/detect-project` workflow. + +**If Remote URL provided:** +Follow the "Remote Repository Strategy" path in `/detect-project`. +- Ask user to choose: Remote S2I, Remote Podman, or Clone. + +**If Local Files:** +Proceed with standard detection. + +```markdown +## Phase 1: Analyzing Your Project + +[If Local] +Scanning project directory for language indicators... + +[If Remote] +Analyzing remote repository options... + +... +``` + +Store confirmed values in session state, including `BUILD_STRATEGY` and `HELM_CHART_DETECTED`. + +### Phase 1.4: Deployment Target Selection + +```markdown +## Deployment Target + +Where would you like to deploy this application? + +| Target | Description | Requirements | +|--------|-------------|--------------| +| **OpenShift** | Deploy to OpenShift/Kubernetes cluster | `oc login` access | +| **RHEL Host** | Deploy directly to a standalone RHEL system | SSH access to RHEL 8+ | + +**Which target would you like to use?** +1. OpenShift - Deploy to current cluster +2. RHEL - Deploy to a RHEL host via SSH +``` + +Store `DEPLOYMENT_TARGET` in session state. + +Proceed automatically without waiting for user input. + +**If user selects "RHEL":** +- Store `DEPLOYMENT_TARGET = "rhel"` in session state +- Delegate to `/rhel-deploy` skill with detected project info +- Pass: `APP_NAME`, `LANGUAGE`, `FRAMEWORK`, `VERSION`, `BUILDER_IMAGE`, `CONTAINER_PORT` +- The `/rhel-deploy` skill handles SSH connection, deployment strategy, and service creation +- After `/rhel-deploy` completes → Go to **Phase 8 (Completion)** + +**If user selects "OpenShift":** +- Store `DEPLOYMENT_TARGET = "openshift"` in session state +- Continue to Phase 1.5 (Strategy Selection) + +### Phase 1.5: Strategy Selection + +If multiple deployment options are available (Helm chart detected, Dockerfile present, or standard project): + +```markdown +## Deployment Strategy + +Based on my analysis, you have these options: + +| Strategy | Use When | Detected | +|----------|----------|----------| +| **S2I** | Standard apps, no Dockerfile needed | [Yes/No] | +| **Podman** | Custom Containerfile/Dockerfile exists | [Yes/No] | +| **Helm** | Helm chart exists or complex deployments | [Yes/No] | + +**Detected in your project:** +[List what was found: language indicators, Dockerfile, Helm chart at ./chart] + +**Which deployment strategy would you like to use?** +1. S2I - Build with Source-to-Image +2. Podman - Build from Containerfile/Dockerfile +3. Helm - Use existing Helm chart +4. Create Helm chart - Generate a new Helm chart for your project (if no chart exists) +``` + +Store `DEPLOYMENT_STRATEGY` in session state. + +Proceed automatically without waiting for user input. + +### Phase 1.6: Image Selection (S2I/Podman only) + +If user selected S2I or Podman deployment strategy, offer image selection options: + +```markdown +## Image Selection + +**Current recommendation:** `[builder-image]` +(Based on: [language] [version]) + +**Image Selection Options:** +- **quick** - Use the recommended image (good for most cases) +- **smart** - Run `/recommend-image` for tailored selection (production vs dev, security, performance) + +Which option would you prefer? +``` + +**If user selects "smart":** +- Invoke `/recommend-image` skill with detected `LANGUAGE`, `FRAMEWORK`, `VERSION` +- Store the result in `BUILDER_IMAGE` and `IMAGE_VARIANT` session state +- Continue to Phase 2 + +**If user selects "quick":** +- Use the already-detected `BUILDER_IMAGE` +- Continue to Phase 2 + +**BRANCHING LOGIC:** +- If `DEPLOYMENT_STRATEGY` is **"S2I"** or **"Podman"** → After Phase 2, continue to **Phase 3 (S2I/Podman Path)** +- If `DEPLOYMENT_STRATEGY` is **"Helm"** → After Phase 2, go to **Phase 2-H (Helm Path)** + +### Phase 1.7: Configuration Review (MANDATORY) + +**This phase MUST NOT be skipped regardless of how the user responded to previous phases.** + +```markdown +## Configuration Review + +Before I proceed, let me confirm the deployment configuration: + +**Environment Type:** +| Type | Characteristics | +|------|-----------------| +| **Development** | `latest` tags, lower resources, quick iteration | +| **Staging** | Version tags, moderate resources, testing | +| **Production** | Pinned versions, higher resources, HA-ready | + +**Which environment is this deployment for?** +1. Development +2. Staging +3. Production + +--- + +**Configuration Approach:** +| Approach | When to Use | +|----------|-------------| +| **Runtime config** | Need to change settings without rebuilding (Recommended for prod) | +| **Build-time config** | Simpler, settings baked into image (OK for dev) | + +**How should environment variables be handled?** +1. Runtime (ConfigMap mount) +2. Build-time (baked into image) + +--- + +**Resource Settings:** +| Setting | Dev Default | Prod Default | +|---------|-------------|--------------| +| Replicas | 1 | 2+ | +| CPU limit | 200m | 400m+ | +| Memory limit | 256Mi | 512Mi+ | + +**Use defaults for your environment, or customize?** +1. Use defaults +2. Customize resources +``` + +Proceed automatically without waiting for user input. + +Store: `ENVIRONMENT_TYPE`, `CONFIG_APPROACH`, `RESOURCE_PROFILE` in session state. + +### Phase 2: OpenShift Connection + +```markdown +## Phase 2: Connecting to OpenShift + +Checking cluster connection... + +**Current Context:** +| Setting | Value | +|---|---| +| Cluster | [cluster-api-url] | +| User | [username] | +| Namespace | [current-namespace] | + +**Is this the correct cluster and namespace?** +- yes - Continue to build +- no - I need to change this + +[If no] +**To change context:** +1. Run `oc login ` in your terminal +2. Or run `oc project ` to switch namespace +3. Then tell me to continue + +**Available namespaces you have access to:** +[List first 10 namespaces/projects] + +Which namespace should I deploy to? +``` + +Store confirmed `NAMESPACE` in session state. + +--- + +## S2I/PODMAN PATH (If DEPLOYMENT_STRATEGY is "S2I" or "Podman") + +### Phase 3: Git Repository Check + +```markdown +## Git Repository + +I need a Git URL for the S2I build. + +**Detected from .git/config:** +- Remote: `[git-url]` +- Branch: `[current-branch]` + +**Is this correct?** (yes/no) + +[If no git config found] +**Please provide:** +1. Git repository URL (e.g., https://github.com/user/repo.git) +2. Branch name (default: main) +``` + +Store `GIT_URL` and `GIT_BRANCH` in session state. + +### Phase 4: Pre-Build Summary + +```markdown +## Phase 3: Build Configuration + +Here's what I'll create on OpenShift: + +**Target:** +- Cluster: [cluster] +- Namespace: [namespace] + +**Resources to Create:** + +1. **ImageStream** `[app-name]` + - Stores built container images + +2. **BuildConfig** `[app-name]` + - Source: [git-url] (branch: [branch]) + - Builder: [builder-image] + - Output: [app-name]:latest + +--- + +**Would you like to see the full YAML?** (yes/no) + +[If yes, show both YAML manifests] + +--- + +**Proceed with creating these resources and starting the build?** +- yes - Create resources and start build +- modify - I need to change something +- cancel - Stop here +``` + +### Phase 5: Execute Build + +```markdown +## Creating Build Resources... + +[x] ImageStream created: [app-name] +[x] BuildConfig created: [app-name] + +## Starting Build... + +**Build:** [app-name]-1 +**Status:** Running + +--- +**Build Logs:** +``` +[Stream S2I build output] +``` +--- + +[When complete] + +## Build Successful! + +**Build:** [app-name]-1 +**Duration:** [X]m [Y]s +**Image:** [image-reference] + +**CRITICAL: Wait for the build to reach 'Complete' status before proceeding.** + +Continue to deployment? (yes/no) +``` + +### Phase 6: Pre-Deploy Summary + +```markdown +## Phase 4: Deployment Configuration + +**Image ready!** Now let's deploy it. + +**Resources to Create:** + +1. **Deployment** `[app-name]` + - Image: [app-name]:latest + - Replicas: 1 + - Port: [detected-port] + +2. **Service** `[app-name]` + - Internal load balancer + - Port: [port] + +3. **Route** `[app-name]` + - External HTTPS access + - URL: https://[app-name]-[namespace].[domain] + +--- + +**Would you like to see the full YAML?** (yes/no) + +[If yes, show all three YAML manifests] + +--- + +**Proceed with deployment?** +- yes - Deploy the application +- modify - I need to change something +- cancel - Stop here (build artifacts preserved) +``` + +### Phase 7: Execute Deployment + +```markdown +## Deploying Application... + +[x] Deployment created: [app-name] +[x] Service created: [app-name] +[x] Route created: [app-name] + +## Waiting for Rollout... + +**Pod Status:** +| Pod | Status | Ready | +|-----|-----|---| +| [app-name]-xxx-yyy | Running | 1/1 | + +Rollout complete! +``` + +**If rollout fails** (pods not ready, CrashLoopBackOff, ImagePullBackOff, etc.): + +```markdown +## Deployment Failed + +The deployment did not complete successfully. + +**Pod Status:** +| Pod | Status | Ready | Restarts | +|-----|--------|-------|----------| +| [app-name]-xxx-yyy | [status] | 0/1 | [count] | + +--- + +**Would you like me to diagnose the issue?** + +1. **Debug Pod** (`/debug-pod`) - Investigate pod failures +2. **Debug Network** (`/debug-network`) - Check service/route connectivity +3. **Debug Build** (`/debug-build`) - Re-check build if image issues +4. **View logs manually** +5. **Rollback and stop** + +Select an option: +``` + +- If user selects a debug option → Invoke the corresponding skill +- After debugging → Offer to retry deployment + +--- + +## HELM PATH (If DEPLOYMENT_STRATEGY is "Helm") + +### Phase 2-H: Helm Deployment + +If user selected Helm in Phase 1.5, execute this path instead of Phases 3-7. + +```markdown +## Helm Deployment + +Switching to Helm deployment workflow... + +The `/helm-deploy` skill will handle: +1. Validate the Helm chart +2. Review and customize values +3. Install/upgrade the release +4. Monitor deployment +5. Present results + +Proceeding with Helm deployment... +``` + +**Delegate to `/helm-deploy` skill:** +- Pass `APP_NAME`, `NAMESPACE`, `HELM_CHART_PATH` from session state +- The helm-deploy skill handles chart detection, values review, and installation +- After helm-deploy completes → Go to **Phase 8 (Completion)** + +**If user chose "Create Helm chart":** +- Generate chart using templates from templates/helm/ +- Replace `${APP_NAME}` placeholders with detected app name +- Set `${CONTAINER_PORT}` based on detected port +- Then proceed with helm-deploy workflow + +--- + +## COMPLETION (Both paths converge here) + +### Phase 8: Completion + +Present a summary including: +- Application name, namespace, language, framework +- Access URLs (external route, internal service DNS) +- Resources created with status (ImageStream, BuildConfig, Deployment, Service, Route) +- Quick commands: view logs, scale, rebuild, delete +- Next steps: open app URL, set up webhooks, add env vars, configure autoscaling + +## Dependencies + +### Required MCP Servers +- `openshift` - cluster resource management for OpenShift deployments + +### Related Skills +- `/debug-pod` - Pod failures (CrashLoopBackOff, OOMKilled, ImagePullBackOff) +- `/debug-build` - S2I or Podman build failures +- `/debug-network` - Service connectivity issues (no endpoints, 503 errors) +- `/debug-rhel` - RHEL deployment failures (systemd, SELinux, firewall) + +### Reference Documentation +- [docs/builder-images.md](../../docs/builder-images.md) - Language detection, S2I builder images +- [docs/image-selection-criteria.md](../../docs/image-selection-criteria.md) - Image variant selection, LTS timelines +- [docs/python-s2i-entrypoints.md](../../docs/python-s2i-entrypoints.md) - Python S2I configuration +- [docs/rhel-deployment.md](../../docs/rhel-deployment.md) - RHEL host deployment +- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common error patterns and troubleshooting +- [docs/prerequisites.md](../../docs/prerequisites.md) - All required tools by skill diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/buildconfig.yaml.template b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/buildconfig.yaml.template new file mode 100644 index 00000000..b3294eb2 --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/buildconfig.yaml.template @@ -0,0 +1,38 @@ +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: build + app.kubernetes.io/part-of: ${APP_NAME} +spec: + source: + type: Git + git: + uri: ${GIT_URL} + ref: ${GIT_BRANCH} + strategy: + type: Source + sourceStrategy: + from: + kind: DockerImage + name: ${BUILDER_IMAGE} + env: [] + output: + to: + kind: ImageStreamTag + name: ${APP_NAME}:latest + triggers: + - type: ConfigChange + - type: ImageChange + runPolicy: Serial + resources: + limits: + memory: "1Gi" + cpu: "1" + requests: + memory: "512Mi" + cpu: "500m" diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/deployment.yaml.template b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/deployment.yaml.template new file mode 100644 index 00000000..eb3b481a --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/deployment.yaml.template @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: application + app.kubernetes.io/part-of: ${APP_NAME} + annotations: + image.openshift.io/triggers: | + [{"from":{"kind":"ImageStreamTag","name":"${APP_NAME}:latest"},"fieldPath":"spec.template.spec.containers[0].image"}] +spec: + replicas: ${REPLICAS} + selector: + matchLabels: + app: ${APP_NAME} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + template: + metadata: + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + spec: + containers: + - name: ${APP_NAME} + image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/${APP_NAME}:latest + ports: + - containerPort: ${CONTAINER_PORT} + protocol: TCP + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + livenessProbe: + httpGet: + path: / + port: ${CONTAINER_PORT} + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: / + port: ${CONTAINER_PORT} + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + env: [] + restartPolicy: Always + terminationGracePeriodSeconds: 30 diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/Chart.yaml.template b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/Chart.yaml.template new file mode 100644 index 00000000..1aa22dd1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/Chart.yaml.template @@ -0,0 +1,13 @@ +apiVersion: v2 +name: ${APP_NAME} +description: ${APP_DESCRIPTION} +type: application +version: 0.1.0 +appVersion: "${APP_VERSION}" +keywords: + - ${LANGUAGE} + - ${FRAMEWORK} + - openshift +maintainers: + - name: ${MAINTAINER_NAME} + email: ${MAINTAINER_EMAIL} diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/NOTES.txt.template b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/NOTES.txt.template new file mode 100644 index 00000000..154e628d --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/NOTES.txt.template @@ -0,0 +1,32 @@ +Congratulations! Your application {{ include "${APP_NAME}.fullname" . }} has been deployed. + +{{- if .Values.route.enabled }} + +Access your application at: +{{- if .Values.route.host }} + https://{{ .Values.route.host }} +{{- else }} + Run: oc get route {{ include "${APP_NAME}.fullname" . }} -o jsonpath='{.spec.host}' +{{- end }} + +{{- else }} + +Your application is available internally at: + {{ include "${APP_NAME}.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }} + +To expose it externally, create a Route or set route.enabled=true. + +{{- end }} + +Useful commands: + # View pods + oc get pods -l app.kubernetes.io/name={{ include "${APP_NAME}.name" . }} + + # View logs + oc logs -l app.kubernetes.io/name={{ include "${APP_NAME}.name" . }} -f + + # Upgrade release + helm upgrade {{ .Release.Name }} ./{{ .Chart.Name }} -f values.yaml + + # Uninstall release + helm uninstall {{ .Release.Name }} diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/_helpers.tpl.template b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/_helpers.tpl.template new file mode 100644 index 00000000..15873b10 --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/_helpers.tpl.template @@ -0,0 +1,60 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "${APP_NAME}.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "${APP_NAME}.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "${APP_NAME}.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "${APP_NAME}.labels" -}} +helm.sh/chart: {{ include "${APP_NAME}.chart" . }} +{{ include "${APP_NAME}.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "${APP_NAME}.selectorLabels" -}} +app.kubernetes.io/name: {{ include "${APP_NAME}.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "${APP_NAME}.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "${APP_NAME}.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/deployment.yaml.template b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/deployment.yaml.template new file mode 100644 index 00000000..a6cbd868 --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/deployment.yaml.template @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "${APP_NAME}.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "${APP_NAME}.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "${APP_NAME}.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/route.yaml.template b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/route.yaml.template new file mode 100644 index 00000000..e2bab29a --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/route.yaml.template @@ -0,0 +1,24 @@ +{{- if .Values.route.enabled }} +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + {{- if .Values.route.host }} + host: {{ .Values.route.host }} + {{- end }} + to: + kind: Service + name: {{ include "${APP_NAME}.fullname" . }} + weight: 100 + port: + targetPort: http + {{- with .Values.route.tls }} + tls: + termination: {{ .termination }} + insecureEdgeTerminationPolicy: {{ .insecureEdgeTerminationPolicy }} + {{- end }} + wildcardPolicy: None +{{- end }} diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/service.yaml.template b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/service.yaml.template new file mode 100644 index 00000000..837bc888 --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/templates/service.yaml.template @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "${APP_NAME}.selectorLabels" . | nindent 4 }} diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/values.yaml.template b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/values.yaml.template new file mode 100644 index 00000000..1cca6017 --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/helm/values.yaml.template @@ -0,0 +1,67 @@ +# Default values for ${APP_NAME} +replicaCount: 1 + +image: + repository: ${IMAGE_REPOSITORY} + pullPolicy: IfNotPresent + tag: "${IMAGE_TAG}" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: "" + +podAnnotations: {} +podSecurityContext: {} +securityContext: {} + +service: + type: ClusterIP + port: ${CONTAINER_PORT} + +route: + enabled: true + host: "" + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + +resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + +livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + +readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + +nodeSelector: {} +tolerations: [] +affinity: {} + +env: [] +# - name: MY_VAR +# value: "my-value" diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/imagestream.yaml.template b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/imagestream.yaml.template new file mode 100644 index 00000000..46572193 --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/imagestream.yaml.template @@ -0,0 +1,13 @@ +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: image + app.kubernetes.io/part-of: ${APP_NAME} +spec: + lookupPolicy: + local: false diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/route.yaml.template b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/route.yaml.template new file mode 100644 index 00000000..7c53d2e7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/route.yaml.template @@ -0,0 +1,21 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: route + app.kubernetes.io/part-of: ${APP_NAME} +spec: + to: + kind: Service + name: ${APP_NAME} + weight: 100 + port: + targetPort: http + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + wildcardPolicy: None diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/service.yaml.template b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/service.yaml.template new file mode 100644 index 00000000..7e1cf371 --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/service.yaml.template @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: service + app.kubernetes.io/part-of: ${APP_NAME} +spec: + selector: + app: ${APP_NAME} + ports: + - name: http + port: ${CONTAINER_PORT} + targetPort: ${CONTAINER_PORT} + protocol: TCP + type: ClusterIP + sessionAffinity: None diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/systemd/systemd-container-rootful.service b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/systemd/systemd-container-rootful.service new file mode 100644 index 00000000..c1e8fe8f --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/systemd/systemd-container-rootful.service @@ -0,0 +1,27 @@ +# Rootful Podman container managed by systemd (system service) +# Location: /etc/systemd/system/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${PORT} - Port number (used for both host and container binding) +# ${IMAGE} - Container image reference + +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run --name ${APP_NAME} \ + -p ${PORT}:${PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/systemd/systemd-container-rootless.service b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/systemd/systemd-container-rootless.service new file mode 100644 index 00000000..ca9dc371 --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/systemd/systemd-container-rootless.service @@ -0,0 +1,27 @@ +# Rootless Podman container managed by systemd (user service) +# Location: ~/.config/systemd/user/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${PORT} - Port number (used for both host and container binding) +# ${IMAGE} - Container image reference + +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run --name ${APP_NAME} \ + -p ${PORT}:${PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/systemd/systemd-native.service b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/systemd/systemd-native.service new file mode 100644 index 00000000..c55cfc07 --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/environment/templates/systemd/systemd-native.service @@ -0,0 +1,39 @@ +# Native application managed by systemd (system service) +# Location: /etc/systemd/system/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${SERVICE_USER} - User to run the service as +# ${APP_PATH} - Application install path (e.g., /opt/app-name) +# ${PORT} - Application listen port +# ${START_COMMAND} - Application start command +# +# Start command examples by language: +# Node.js: /usr/bin/node ${APP_PATH}/server.js +# Python: /usr/bin/python3 ${APP_PATH}/app.py +# Java: /usr/bin/java -jar ${APP_PATH}/app.jar +# Go: ${APP_PATH}/binary-name + +[Unit] +Description=${APP_NAME} Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=${APP_PATH} +Environment=PORT=${PORT} +ExecStart=${START_COMMAND} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=${APP_PATH} + +[Install] +WantedBy=multi-user.target diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/instruction.md b/evaluation/with_skills/rh-developer__containerize-deploy/instruction.md new file mode 100644 index 00000000..42797f3c --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/instruction.md @@ -0,0 +1,15 @@ +# Containerization and Deployment Task + +You are a Red Hat developer. Your team has a Python web application that needs to be containerized and deployed to OpenShift. You need to evaluate the available approaches and recommend the best one. + +## Requirements +- Examine the application source and determine its language, dependencies, and build requirements +- Compare containerization strategies (e.g., S2I, Dockerfile, Helm chart) and explain the trade-offs of each for this application +- If a multi-stage Dockerfile approach is viable, include a working example showing build and runtime stages +- Recommend the best approach with a clear justification +- Define the deployment configuration including: resource requests/limits, all three probe types (startup, liveness, readiness), autoscaling (HPA), and how external traffic will reach the application +- Address application-specific concerns like database connection pooling configuration + +Document your strategy evaluation, recommendation, and deployment plan in `/root/report.md`. + +Use MCP tools to examine the environment. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/solution/solve.sh b/evaluation/with_skills/rh-developer__containerize-deploy/solution/solve.sh new file mode 100644 index 00000000..713efa82 --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/solution/solve.sh @@ -0,0 +1,23 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Containerize and Deploy Plan + +## Phase 1: Detect +- Language: Python +- Framework: Django +- Entry point: manage.py + +## Phase 2: Strategy +- Target: OpenShift +- Strategy: S2I (recommended for Python on OpenShift) +- Alternative: Dockerfile with multi-stage build + +## Phase 3: Build +- Builder image: ubi9/python-311 +- APP_MODULE: myproject.wsgi:application + +## Phase 4: Deploy +- Deployment + Service + Route +- Port: 8000 (Django default) +- On failure: /debug-pod, /debug-build, /debug-network +REPORT_EOF diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/task.toml b/evaluation/with_skills/rh-developer__containerize-deploy/task.toml new file mode 100644 index 00000000..9022cd22 --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__containerize-deploy" +name = "rh-developer End-to-End Containerize and Deploy Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "containerize-deploy", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/tests/llm_judge.py b/evaluation/with_skills/rh-developer__containerize-deploy/tests/llm_judge.py new file mode 100644 index 00000000..0dc24c7f --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "multistage_and_startup_probe", + "file": "/root/report.md", + "question": "Does the report include both a multi-stage Dockerfile example (with COPY --from=builder or AS builder) AND a startup probe configuration?", + "reference": "A skilled report includes a multi-stage Dockerfile showing build and runtime stages with COPY --from=builder, and configures a startupProbe in addition to liveness/readiness probes. An unskilled report provides only a single-stage Dockerfile and only liveness/readiness probes without startup probe." + }, + { + "id": "hpa_and_pool_config", + "file": "/root/report.md", + "question": "Does the report include a HorizontalPodAutoscaler manifest (with autoscaling/v2 API) AND database connection pool configuration (SQLALCHEMY_POOL or equivalent)?", + "reference": "A skilled report includes a complete HPA YAML with kind: HorizontalPodAutoscaler and autoscaling/v2 API, plus SQLAlchemy connection pool settings (pool_size, pool_recycle). An unskilled report mentions autoscaling conceptually without the manifest, and skips connection pool configuration." + }, + { + "id": "strategy_comparison_depth", + "file": "/root/report.md", + "question": "Does the report compare at least 3 containerization strategies (S2I, Dockerfile, Helm) with specific trade-offs and a justified recommendation?", + "reference": "A skilled report provides a detailed comparison table of S2I, Dockerfile, and Helm with pros/cons/trade-offs for each, leading to a justified recommendation. An unskilled report may compare strategies superficially without detailed trade-offs." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/tests/test.sh b/evaluation/with_skills/rh-developer__containerize-deploy/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/tests/test_outputs.py b/evaluation/with_skills/rh-developer__containerize-deploy/tests/test_outputs.py new file mode 100644 index 00000000..5f7eec38 --- /dev/null +++ b/evaluation/with_skills/rh-developer__containerize-deploy/tests/test_outputs.py @@ -0,0 +1,110 @@ +""" +Tests for rh-developer__containerize-deploy per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_containerization(self): + content = read_report().lower() + assert any(t in content for t in ["container", "deploy", "image"]), ( + "report should mention containerization or deployment" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_startup_probe(self): + """Skill docs teach startup probe in addition to liveness/readiness. + Without skill, agents typically only include liveness and readiness probes.""" + c = read_report() + assert "startupProbe" in c or "startup probe" in c.lower() or "startupprobe" in c.lower(), ( + "should include startup probe configuration (startupProbe YAML key)" + ) + + def test_multistage_dockerfile_example(self): + """Skill docs teach multi-stage Dockerfile with COPY --from=builder pattern. + Without skill, agents mention multi-stage conceptually but don't provide the example.""" + c = read_report() + assert "COPY --from=" in c or "AS builder" in c or "copy --from=" in c.lower(), ( + "should include a multi-stage Dockerfile example with COPY --from= or AS builder syntax" + ) + + def test_hpa_autoscaling_config(self): + """Skill docs teach complete HPA configuration with autoscaling API. + Without skill, agents mention autoscaling conceptually but skip the manifest.""" + c = read_report() + assert "HorizontalPodAutoscaler" in c or "autoscaling/v2" in c, ( + "should include HorizontalPodAutoscaler manifest or autoscaling/v2 API reference" + ) + + def test_connection_pool_config(self): + """Skill docs teach application-specific database connection pooling with + SQLAlchemy settings. Without skill, agents skip pool configuration details.""" + c = read_report() + assert any(t in c for t in [ + "SQLALCHEMY_POOL", "pool_size", "POOL_SIZE", + "pool_recycle", "POOL_RECYCLE", + ]), "should include SQLAlchemy connection pool settings (pool_size, pool_recycle)" + + def test_strategy_comparison(self): + """Skill teaches comparing at least 2 containerization strategies with trade-offs.""" + c = read_report().lower() + strategies = ["s2i", "dockerfile", "helm", "podman", "source-to-image"] + mentioned = sum(1 for s in strategies if s in c) + assert mentioned >= 2, "should compare at least 2 containerization strategies" + + def test_session_affinity_config(self): + """Skill docs teach explicit sessionAffinity configuration in Service spec. + Without skill, agents skip this detail in the Service definition.""" + c = read_report().lower() + assert "sessionaffinity" in c or "session affinity" in c, ( + "should specify sessionAffinity in Service configuration" + ) + + def test_app_module_s2i_entrypoint(self): + """Skill teaches APP_MODULE environment variable for S2I Python startup + (e.g., app:app). Without skill, agents don't know this S2I-specific + configuration for WSGI entry point discovery.""" + c = read_report() + assert "APP_MODULE" in c or "app:app" in c or "APP_SCRIPT" in c, ( + "should reference APP_MODULE or app:app S2I entrypoint configuration" + ) + + def test_gunicorn_worker_formula(self): + """Skill teaches Gunicorn worker count formula: (2 × CPU cores) + 1. + Without skill, agents hardcode worker count without the sizing formula.""" + c = read_report() + assert any(t in c for t in [ + "2 * cores", "2 × CPU", "(2 * cores) + 1", "2 × cores", + "2*cores", "2 * cpu", "2x CPU", "2 x cores", + ]) or ("worker" in c.lower() and ("formula" in c.lower() or "cores" in c.lower())), ( + "should include Gunicorn worker count formula based on CPU cores" + ) + + def test_sqlalchemy_engine_options(self): + """Skill teaches SQLALCHEMY_ENGINE_OPTIONS configuration for advanced + pool tuning. Without skill, agents configure individual pool parameters + but miss the unified engine options dict.""" + c = read_report() + assert "SQLALCHEMY_ENGINE_OPTIONS" in c or "engine_options" in c, ( + "should include SQLALCHEMY_ENGINE_OPTIONS for advanced pool configuration" + ) diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/Dockerfile b/evaluation/with_skills/rh-developer__debug-build/environment/Dockerfile new file mode 100644 index 00000000..1cbfefcf --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-build/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__debug-build/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-build/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__debug-build/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-build/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__debug-build/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-build/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__debug-build/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-build/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__debug-build/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-build/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__debug-build/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-build/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__debug-build/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-build/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__debug-build/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-build/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__debug-build/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-build/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-developer__debug-build/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..5f7e49b1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-build/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,755 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + { + "name": "api-service-2", + "namespace": "api-platform", + "status": "Failed", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "1m48s", + "reason": "AssembleFailed", + "message": "Assemble script failed with exit code 1", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "api-service-2": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.110.0\n" + "Collecting uvicorn==0.27.1\n" + "Collecting pydantic==2.6.0\n" + "Collecting psycopg2==2.9.9\n" + " ERROR: Could not build wheels for psycopg2, which is required to install pyproject.toml-based projects\n" + " error: subprocess-exited-with-error\n" + " × Running setup.py install for psycopg2 did not run successfully.\n" + " │ exit code: 1\n" + " ╰─> [25 lines of output]\n" + " Error: pg_config executable not found.\n" + " pg_config is required to build psycopg2 from source.\n" + " Please add the directory containing pg_config to the $PATH\n" + " or specify the full executable path with the option:\n" + " python setup.py build_ext --pg-config /path/to/pg_config\n" + " note: This error originates from a subprocess, and is likely not a problem with pip.\n" + "error: legacy-install-failure\n" + "---> Assemble script FAILED with exit code 1\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/skills/debug-build/SKILL.md b/evaluation/with_skills/rh-developer__debug-build/environment/skills/debug-build/SKILL.md new file mode 100644 index 00000000..0a5eebf2 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-build/environment/skills/debug-build/SKILL.md @@ -0,0 +1,315 @@ +--- +name: debug-build +description: | + Diagnose OpenShift build failures including S2I builds, Docker/Podman builds, and BuildConfig issues. Automates multi-step diagnosis: BuildConfig validation, build pod logs, registry authentication, and source repository access. Use this skill when builds fail, hang, or produce unexpected results. Triggers on /debug-build command or phrases like "build failed", "S2I error", "can't pull builder image", "can't push to registry", "build timeout". +model: inherit +color: cyan +metadata: + user_invocable: "true" +--- + +# /debug-build Skill + +Diagnose OpenShift build failures by automatically gathering BuildConfig, Build status, build pod logs, and related resources. + +## Prerequisites + +Before running this skill: +1. User is logged into OpenShift cluster +2. User has access to the target namespace +3. Build or BuildConfig name is known (or can be identified from recent builds) + +## When to Use This Skill + +Use this skill when OpenShift builds fail, hang, or produce unexpected results. It diagnoses S2I builds, Docker/Podman builds, and BuildConfig issues by analyzing build pod logs, registry authentication, and source repository access. + +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +## Workflow + +### Step 1: Identify Target Build + +```markdown +## Build Debugging + +**Current OpenShift Context:** +- Cluster: [cluster] +- Namespace: [namespace] + +Which build would you like me to debug? + +1. **Specify build name** - Enter the build name directly (e.g., myapp-1) +2. **List failed builds** - Show recent failed builds in current namespace +3. **From BuildConfig** - Debug latest build from a specific BuildConfig + +Select an option or enter a build name: +``` + +Proceed automatically without waiting for user input. + +If user selects "List failed builds": +Use kubernetes MCP `resources_list` for builds, filter by Failed phase: + +```markdown +## Recent Failed Builds in [namespace] + +| Build | BuildConfig | Status | Started | Duration | +|-------|-------------|--------|---------|----------| +| [app-1] | [app] | Failed | [timestamp] | [duration] | +| [app-2] | [app] | Cancelled | [timestamp] | [duration] | +| [other-1] | [other] | Failed | [timestamp] | [duration] | + +Which build would you like me to debug? +``` + +Proceed automatically without waiting for user input. + +### Step 2: Get Build Status Overview + +Use kubernetes MCP `resources_get` to get Build details: + +```markdown +## Build Status: [build-name] + +**Build Info:** +| Field | Value | +|-------|-------| +| BuildConfig | [buildconfig-name] | +| Strategy | [Source/Docker/JenkinsPipeline] | +| Phase | [New/Pending/Running/Complete/Failed/Cancelled] | +| Started | [timestamp] | +| Completed | [timestamp or "Still running"] | +| Duration | [duration] | + +**Build Configuration:** +| Setting | Value | +|---------|-------| +| Source Type | [Git/Binary/Dockerfile] | +| Git URL | [url] | +| Git Ref | [branch/tag] | +| Builder Image | [image:tag] | +| Output Image | [imagestream:tag] | + +**Build Status:** +- Phase: [phase] +- Reason: [reason if failed] +- Message: [message if available] + +**Quick Assessment:** +[Based on status, provide initial assessment - e.g., "Build failed during assemble phase - likely dependency installation issue"] + +Continue with detailed analysis? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Step 3: Analyze BuildConfig + +Use kubernetes MCP `resources_get` to get BuildConfig: + +```markdown +## BuildConfig Analysis: [buildconfig-name] + +**Source Configuration:** +| Setting | Value | Status | +|---------|-------|--------| +| Git URL | [url] | [OK/WARN: check access] | +| Git Ref | [ref] | [OK/WARN: branch not found] | +| Context Dir | [dir or "/"] | [OK] | +| Source Secret | [secret-name or "None"] | [OK/MISSING] | + +**Builder Image:** +| Setting | Value | Status | +|---------|-------|--------| +| Image | [image:tag] | [OK/WARN: check exists] | +| Pull Secret | [secret-name or "None"] | [OK/MISSING] | + +**Output Configuration:** +| Setting | Value | Status | +|---------|-------|--------| +| Output To | [ImageStreamTag] | [OK] | +| Push Secret | [secret-name or "None"] | [OK/MISSING] | + +**Environment Variables:** +| Name | Value | Source | +|------|-------|--------| +| [VAR] | [value or "***"] | [Direct/ConfigMap/Secret] | + +**Issues Found:** +- [Issue 1 - e.g., "Source secret 'github-creds' referenced but not found"] +- [Issue 2 - e.g., "Builder image uses older tag, may have compatibility issues"] + +Continue to view build logs? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Step 4: Get Build Pod Logs + +Use kubernetes MCP `pod_logs` for the builder pod: + +```markdown +## Build Logs: [build-name] + +**Build Phases:** +| Phase | Status | Duration | +|-------|--------|----------| +| Fetching source | [Complete/Failed] | [duration] | +| Pulling builder image | [Complete/Failed] | [duration] | +| Assemble | [Complete/Failed] | [duration] | +| Commit | [Complete/Failed] | [duration] | +| Push | [Complete/Failed] | [duration] | + +**Failed Phase: [phase-name]** + +``` +[Last 100 lines of build logs, focused on the failing phase] +``` + +**Log Analysis:** + +[Analyze logs and identify errors:] + +**Errors Found:** +- Line [X]: [error description - e.g., "npm ERR! 404 Not Found - package 'nonexistent@1.0.0'"] +- Line [Y]: [error description - e.g., "error: unable to resolve 'github.com/private/repo'"] + +**S2I Phase Explanation:** + +[For S2I builds, explain what the failed phase does:] +- **assemble**: Installs dependencies and builds application +- **commit**: Creates the final container image layer +- **push**: Pushes image to internal registry + +Continue to check related resources? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Step 5: Check Related Resources + +Check secrets, imagestreams, and source access: + +```markdown +## Related Resources Analysis + +**ImageStreams:** +| ImageStream | Tags | Last Updated | Status | +|-------------|------|--------------|--------| +| [app] | [latest, v1.0] | [timestamp] | [OK] | +| [builder] | [imported] | [timestamp] | [OK/MISSING] | + +**Secrets:** +| Secret | Type | Used By | Status | +|--------|------|---------|--------| +| [source-secret] | kubernetes.io/basic-auth | Source | [OK/MISSING] | +| [push-secret] | kubernetes.io/dockerconfigjson | Output | [OK/MISSING] | + +**Source Repository Access:** +[If GitHub MCP available, check if source URL is accessible] +- URL: [git-url] +- Status: [Accessible/401 Unauthorized/404 Not Found/Timeout] + +**Registry Access:** +[Check if internal registry is accessible] +- Registry: image-registry.openshift-image-registry.svc:5000 +- Status: [OK/Unreachable] + +**Issues Found:** +- [Issue 1 - e.g., "Secret 'github-token' missing - cannot authenticate to private repo"] +- [Issue 2 - e.g., "Builder ImageStreamTag 'nodejs:18' not imported"] + +Continue to full diagnosis summary? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Step 6: Present Diagnosis Summary + +```markdown +## Diagnosis Summary: [build-name] + +### Root Cause + +**Primary Issue:** [Categorized root cause] + +| Category | Status | Details | +|----------|--------|---------| +| Source Access | [OK/FAIL] | [details] | +| Builder Image | [OK/FAIL] | [details] | +| Dependencies | [OK/FAIL] | [details] | +| Build Script | [OK/FAIL] | [details] | +| Registry Push | [OK/FAIL] | [details] | + +### Detailed Findings + +**[Category 1: e.g., Dependency Installation]** +- Problem: [specific problem - e.g., "npm package 'lodash@99.0.0' does not exist"] +- Evidence: [from build logs] +- Impact: [build fails during assemble phase] + +**[Category 2: e.g., Source Authentication]** +- Problem: [specific problem] +- Evidence: [from events/logs] +- Impact: [cannot clone repository] + +### Recommended Actions + +1. **[Action 1]** - [description] + ```bash + [command to fix - e.g., oc create secret generic github-token --from-literal=...] + ``` + +2. **[Action 2]** - [description] + ```bash + [command to fix - e.g., oc import-image nodejs:18 --from=registry.access.redhat.com/ubi9/nodejs-18] + ``` + +3. **[Action 3]** - [description] + +### Retry Build + +After fixing the issue: +```bash +# Start a new build +oc start-build [buildconfig-name] -n [namespace] + +# Or start build with follow +oc start-build [buildconfig-name] -n [namespace] --follow +``` + +--- + +Would you like me to: +1. Execute one of the recommended fixes +2. Retry the build +3. Compare with last successful build +4. Debug the build pod (/debug-pod) +5. Exit debugging + +Select an option: +``` + +Proceed automatically without waiting for user input. + +## Build Failure Categories + +For S2I build phase failures, common error patterns (Node.js, Python, Java), and troubleshooting decision trees, see [docs/debugging-patterns.md](../../docs/debugging-patterns.md). + +## Dependencies + +### Required MCP Servers +- `openshift` - Kubernetes/OpenShift resource access for builds, BuildConfigs, and build pod logs + +### Related Skills +- `/s2i-build` - To retry build after fixing issues +- `/debug-pod` - To debug the builder pod directly +- `/deploy` - To deploy after a successful build + +### Reference Documentation +- [docs/builder-images.md](../../docs/builder-images.md) - S2I builder image selection, version mapping +- [docs/python-s2i-entrypoints.md](../../docs/python-s2i-entrypoints.md) - Python APP_MODULE configuration +- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common error patterns +- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (oc), cluster access verification diff --git a/evaluation/with_skills/rh-developer__debug-build/instruction.md b/evaluation/with_skills/rh-developer__debug-build/instruction.md new file mode 100644 index 00000000..2cfea7f9 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-build/instruction.md @@ -0,0 +1,14 @@ +# Build Debugging Task + +You are a Red Hat developer. An OpenShift Source-to-Image (S2I) build is failing. Investigate the build process to identify and fix the issue. + +## Requirements +- Examine the build configuration and logs +- Identify which S2I build phase is failing (fetch, pull, assemble, commit, push) +- If the fix involves S2I customization, explain how S2I assemble scripts can be extended or overridden +- Provide multiple fix options with concrete commands or file changes, using the appropriate package manager for UBI-based builder images +- Recommend a fix + +Use MCP tools to examine the cluster. Document your methodology, findings, and recommendations in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-developer__debug-build/solution/solve.sh b/evaluation/with_skills/rh-developer__debug-build/solution/solve.sh new file mode 100644 index 00000000..1e0579ec --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-build/solution/solve.sh @@ -0,0 +1,21 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Build Debug Report + +## Build Failure Analysis + +### S2I Build Phases +1. Fetching source ✓ +2. Pulling builder image ✓ +3. **Assemble** ✗ (FAILED) +4. Commit (not reached) +5. Push (not reached) + +### Root Cause +Assemble phase failed — likely dependency installation error in pip install. + +### Fix +- Check requirements.txt for version conflicts (gunicorn, APP_MODULE) +- Verify builder image compatibility (python:3.11-ubi9) +- Retry: `oc start-build flask-app -n myproject --follow` +REPORT_EOF diff --git a/evaluation/with_skills/rh-developer__debug-build/task.toml b/evaluation/with_skills/rh-developer__debug-build/task.toml new file mode 100644 index 00000000..af5ff817 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-build/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__debug-build" +name = "rh-developer Build Debugging Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "debug-build", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-developer__debug-build/tests/llm_judge.py b/evaluation/with_skills/rh-developer__debug-build/tests/llm_judge.py new file mode 100644 index 00000000..7bfd7911 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-build/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "s2i_custom_assemble", + "file": "/root/report.md", + "question": "Does the report mention .s2i/bin/assemble as a way to customize the S2I build process, and reference the default assemble script path at /usr/libexec/s2i/assemble?", + "reference": "A skilled report shows creating a .s2i/bin/assemble script that installs missing packages and then calls /usr/libexec/s2i/assemble (the default assemble script). An unskilled report recommends a custom Dockerfile or builder image instead of using S2I customization hooks." + }, + { + "id": "phase_diagnosis_and_remediation", + "file": "/root/report.md", + "question": "Does the report identify which S2I phase (fetch, assemble, commit, push) failed and provide concrete oc commands for remediation?", + "reference": "A skilled report breaks down the build into phases, identifies the failing phase, and provides actionable commands like 'oc start-build' to retry. An unskilled report gives a generic build failure description." + }, + { + "id": "systematic_build_analysis", + "file": "/root/report.md", + "question": "Does the report follow a systematic approach: inspecting the BuildConfig, analyzing build logs by phase, checking related resources (secrets, imagestreams), and providing structured findings with concrete remediation?", + "reference": "A skilled report follows a structured debugging workflow: BuildConfig analysis, phase-by-phase log analysis, related resource checks, and categorized findings with concrete remediation commands. An unskilled report gives ad-hoc observations without systematic investigation." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-developer__debug-build/tests/test.sh b/evaluation/with_skills/rh-developer__debug-build/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-build/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-developer__debug-build/tests/test_outputs.py b/evaluation/with_skills/rh-developer__debug-build/tests/test_outputs.py new file mode 100644 index 00000000..c3ac3895 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-build/tests/test_outputs.py @@ -0,0 +1,77 @@ +""" +Tests for rh-developer__debug-build per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_build(self): + content = read_report().lower() + assert "build" in content, "report should mention builds" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_s2i_custom_assemble_script(self): + """Skill teaches creating .s2i/bin/assemble to extend the S2I build process. + Without skill, agents recommend Dockerfile or custom builder image instead.""" + c = read_report() + assert ".s2i/bin/assemble" in c or ".s2i/bin" in c, ( + "should mention .s2i/bin/assemble as a way to customize the S2I build" + ) + + def test_default_assemble_path(self): + """Skill teaches invoking the default S2I assemble script at /usr/libexec/s2i/assemble. + Without skill, agents don't know the default script path.""" + c = read_report() + assert "/usr/libexec/s2i/" in c or "libexec/s2i" in c, ( + "should reference the default S2I assemble script at /usr/libexec/s2i/" + ) + + def test_package_manager_awareness(self): + """Report should mention package installation approach for the builder image.""" + c = read_report().lower() + assert any(t in c for t in ["microdnf", "dnf", "yum", "package manager", "install package"]), ( + "should mention package installation approach for the builder image" + ) + + def test_s2i_phase_breakdown(self): + """Skill teaches S2I phases (fetch, pull, assemble, commit, push).""" + c = read_report().lower() + phases = ["assemble", "fetch", "pull", "push", "commit"] + mentioned = sum(1 for p in phases if p in c) + assert mentioned >= 2, ( + "should identify S2I build phases (skill teaches phase-by-phase diagnosis)" + ) + + def test_concrete_remediation_command(self): + """Skill teaches providing concrete oc/command remediation.""" + c = read_report().lower() + assert any(t in c for t in ["oc ", "oc start-build", "oc create", "oc import", "retry"]) or ( + "```" in read_report() and ("oc" in c or "bash" in c) + ), "should include concrete remediation commands" + + def test_dependency_fix_suggestion(self): + """Report should suggest concrete dependency fixes for the failing build.""" + c = read_report().lower() + assert any(t in c for t in [ + "psycopg", "pip install", "requirements", "dependency", "package" + ]), "should suggest concrete dependency fixes for the failing build" diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/Dockerfile b/evaluation/with_skills/rh-developer__debug-container/environment/Dockerfile new file mode 100644 index 00000000..a4c2cd43 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-container/environment/Dockerfile @@ -0,0 +1,74 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "podman": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-podman-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__debug-container/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-container/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__debug-container/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-container/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__debug-container/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-container/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__debug-container/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-container/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__debug-container/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-container/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__debug-container/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-container/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__debug-container/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-container/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__debug-container/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-container/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__debug-container/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-container/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-developer__debug-container/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-container/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/mcp-servers/mock-podman-mcp.py b/evaluation/with_skills/rh-developer__debug-container/environment/mcp-servers/mock-podman-mcp.py new file mode 100644 index 00000000..3d86ba08 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-container/environment/mcp-servers/mock-podman-mcp.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python3 +"""Mock Podman MCP Server for container debugging evaluation. + +Simulates a local Podman environment with several containers, including +one that is crashing (OOMKilled) and one that has an entrypoint error. + +Scenario: + - myapp-web: Exited (137) - OOMKilled, memory limit 256m too low + - myapp-worker: Exited (1) - missing Python dependency 'celery' + - nginx-proxy: Running, healthy + - postgres-db: Running, healthy +""" + +import json +from typing import Optional + +from fastmcp import FastMCP + +mcp = FastMCP("podman") + +NOW = "2026-03-02T12:00:00Z" + +CONTAINERS = { + "a1b2c3d4e5f6": { + "Id": "a1b2c3d4e5f67890abcdef1234567890abcdef1234567890abcdef1234567890", + "Names": ["myapp-web"], + "Image": "myapp:latest", + "ImageID": "sha256:abc123def456789012345678901234567890abcdef1234567890abcdef123456", + "Created": "2026-03-01T10:00:00Z", + "State": { + "Status": "exited", + "Running": False, + "Paused": False, + "Restarting": False, + "OOMKilled": True, + "Dead": False, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-03-01T10:00:05Z", + "FinishedAt": "2026-03-02T08:45:12Z", + }, + "Config": { + "Entrypoint": ["python3"], + "Cmd": ["-m", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"], + "WorkingDir": "/app", + "User": "1001", + "Env": [ + "APP_ENV=production", + "DATABASE_URL=postgresql://db:5432/myapp", + "WORKERS=4", + "MAX_REQUESTS=1000", + ], + "ExposedPorts": {"8080/tcp": {}}, + }, + "HostConfig": { + "Memory": 268435456, + "MemorySwap": 268435456, + "CpuQuota": 100000, + "CpuPeriod": 100000, + "PortBindings": {"8080/tcp": [{"HostIp": "0.0.0.0", "HostPort": "8080"}]}, + "Binds": ["/data/myapp:/app/data:rw"], + }, + "Mounts": [ + {"Type": "bind", "Source": "/data/myapp", "Destination": "/app/data", "Mode": "rw"}, + ], + }, + "b2c3d4e5f6a7": { + "Id": "b2c3d4e5f6a7890123456789abcdef1234567890abcdef1234567890abcdef12", + "Names": ["myapp-worker"], + "Image": "myapp:latest", + "ImageID": "sha256:abc123def456789012345678901234567890abcdef1234567890abcdef123456", + "Created": "2026-03-01T10:00:00Z", + "State": { + "Status": "exited", + "Running": False, + "Paused": False, + "Restarting": False, + "OOMKilled": False, + "Dead": False, + "Pid": 0, + "ExitCode": 1, + "Error": "", + "StartedAt": "2026-03-01T10:00:08Z", + "FinishedAt": "2026-03-01T10:00:12Z", + }, + "Config": { + "Entrypoint": ["python3"], + "Cmd": ["-m", "celery", "-A", "tasks", "worker", "--loglevel=info"], + "WorkingDir": "/app", + "User": "1001", + "Env": [ + "APP_ENV=production", + "DATABASE_URL=postgresql://db:5432/myapp", + "CELERY_BROKER_URL=redis://redis:6379/0", + ], + }, + "HostConfig": { + "Memory": 536870912, + "MemorySwap": 1073741824, + "CpuQuota": 0, + "CpuPeriod": 0, + }, + "Mounts": [], + }, + "c3d4e5f6a7b8": { + "Id": "c3d4e5f6a7b8901234567890abcdef1234567890abcdef1234567890abcdef12", + "Names": ["nginx-proxy"], + "Image": "nginx:1.25", + "ImageID": "sha256:def456789012345678901234567890abcdef1234567890abcdef1234567890ab", + "Created": "2026-02-28T08:00:00Z", + "State": { + "Status": "running", + "Running": True, + "Paused": False, + "Restarting": False, + "OOMKilled": False, + "Dead": False, + "Pid": 12345, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-02-28T08:00:05Z", + "FinishedAt": "0001-01-01T00:00:00Z", + }, + "Config": { + "Entrypoint": ["/docker-entrypoint.sh"], + "Cmd": ["nginx", "-g", "daemon off;"], + "WorkingDir": "", + "User": "", + "Env": ["NGINX_PORT=80"], + "ExposedPorts": {"80/tcp": {}, "443/tcp": {}}, + }, + "HostConfig": { + "Memory": 0, + "MemorySwap": 0, + "CpuQuota": 0, + "CpuPeriod": 0, + "PortBindings": { + "80/tcp": [{"HostIp": "0.0.0.0", "HostPort": "80"}], + "443/tcp": [{"HostIp": "0.0.0.0", "HostPort": "443"}], + }, + }, + "Mounts": [ + {"Type": "bind", "Source": "/etc/nginx/conf.d", "Destination": "/etc/nginx/conf.d", "Mode": "ro"}, + ], + }, + "d4e5f6a7b8c9": { + "Id": "d4e5f6a7b8c9012345678901abcdef1234567890abcdef1234567890abcdef12", + "Names": ["postgres-db"], + "Image": "postgres:15", + "ImageID": "sha256:789012345678901234567890abcdef1234567890abcdef1234567890abcdef12", + "Created": "2026-02-25T12:00:00Z", + "State": { + "Status": "running", + "Running": True, + "Paused": False, + "Restarting": False, + "OOMKilled": False, + "Dead": False, + "Pid": 23456, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-02-25T12:00:10Z", + "FinishedAt": "0001-01-01T00:00:00Z", + }, + "Config": { + "Entrypoint": ["docker-entrypoint.sh"], + "Cmd": ["postgres"], + "WorkingDir": "", + "User": "postgres", + "Env": [ + "POSTGRES_DB=myapp", + "POSTGRES_USER=app", + "PGDATA=/var/lib/postgresql/data", + ], + "ExposedPorts": {"5432/tcp": {}}, + }, + "HostConfig": { + "Memory": 1073741824, + "MemorySwap": 2147483648, + "CpuQuota": 0, + "CpuPeriod": 0, + "PortBindings": {"5432/tcp": [{"HostIp": "127.0.0.1", "HostPort": "5432"}]}, + }, + "Mounts": [ + {"Type": "volume", "Source": "pgdata", "Destination": "/var/lib/postgresql/data", "Mode": "rw"}, + ], + }, +} + +LOGS = { + "myapp-web": ( + "INFO: Started server process [1]\n" + "INFO: Waiting for application startup.\n" + "INFO: Application startup complete.\n" + "INFO: Uvicorn running on http://0.0.0.0:8080\n" + "INFO: Loading ML model into memory...\n" + "INFO: Model size: 1.2GB\n" + "WARNING: Memory usage at 89% of limit (237MB/256MB)\n" + "INFO: Processing request batch (32 items)\n" + "WARNING: Memory usage at 95% of limit (248MB/256MB)\n" + "WARNING: Memory pressure detected, attempting GC\n" + "INFO: GC freed 12MB, usage now at 92%\n" + "INFO: Processing request batch (64 items)\n" + "CRITICAL: Memory usage exceeded limit\n" + "Killed\n" + ), + "myapp-worker": ( + "Traceback (most recent call last):\n" + ' File "/usr/lib/python3.11/runpy.py", line 198, in _run_module_as_main\n' + ' return _run_code(code, main_globals, None,\n' + ' File "/usr/lib/python3.11/runpy.py", line 88, in _run_code\n' + ' exec(code, run_globals)\n' + "ModuleNotFoundError: No module named 'celery'\n" + ), + "nginx-proxy": ( + "2026/02/28 08:00:05 [notice] 1#1: nginx/1.25.4\n" + "2026/02/28 08:00:05 [notice] 1#1: built by gcc 12.2.0\n" + "2026/02/28 08:00:05 [notice] 1#1: OS: Linux 5.14.0-362.el9.x86_64\n" + "2026/02/28 08:00:05 [notice] 1#1: start worker processes\n" + "2026/02/28 08:00:05 [notice] 1#1: start worker process 29\n" + "2026/02/28 08:00:05 [notice] 1#1: start worker process 30\n" + ), + "postgres-db": ( + "PostgreSQL init process complete; ready for start up.\n" + '2026-02-25 12:00:10.123 UTC [1] LOG: starting PostgreSQL 15.5\n' + '2026-02-25 12:00:10.456 UTC [1] LOG: listening on IPv4 address "0.0.0.0", port 5432\n' + '2026-02-25 12:00:10.789 UTC [1] LOG: database system is ready to accept connections\n' + ), +} + +IMAGES = [ + { + "Id": "sha256:abc123def456789012345678901234567890abcdef1234567890abcdef123456", + "RepoTags": ["myapp:latest"], + "Created": "2026-02-28T15:30:00Z", + "Size": 1345678901, + "VirtualSize": 1345678901, + "Labels": {"maintainer": "dev@myapp.io", "version": "2.1.0"}, + "Config": { + "Entrypoint": ["python3"], + "Cmd": ["-m", "uvicorn", "main:app"], + "WorkingDir": "/app", + "ExposedPorts": {"8080/tcp": {}}, + "Env": ["PYTHONDONTWRITEBYTECODE=1", "PYTHONUNBUFFERED=1"], + }, + }, + { + "Id": "sha256:def456789012345678901234567890abcdef1234567890abcdef1234567890ab", + "RepoTags": ["nginx:1.25"], + "Created": "2026-01-15T10:00:00Z", + "Size": 187654321, + "VirtualSize": 187654321, + "Labels": {"maintainer": "NGINX Docker Maintainers"}, + "Config": { + "Entrypoint": ["/docker-entrypoint.sh"], + "Cmd": ["nginx", "-g", "daemon off;"], + "ExposedPorts": {"80/tcp": {}}, + }, + }, + { + "Id": "sha256:789012345678901234567890abcdef1234567890abcdef1234567890abcdef12", + "RepoTags": ["postgres:15"], + "Created": "2026-01-20T12:00:00Z", + "Size": 412345678, + "VirtualSize": 412345678, + "Labels": {"maintainer": "PostgreSQL Docker Maintainers"}, + "Config": { + "Entrypoint": ["docker-entrypoint.sh"], + "Cmd": ["postgres"], + "ExposedPorts": {"5432/tcp": {}}, + }, + }, +] + + +def _find_container(name_or_id: str): + for cid, c in CONTAINERS.items(): + if name_or_id in (cid, c["Id"]): + return c + if name_or_id in c["Names"]: + return c + return None + + +@mcp.tool() +def container_list(all: bool = True) -> str: + """List containers. Set all=True to include stopped containers.""" + results = [] + for cid, c in CONTAINERS.items(): + if not all and not c["State"]["Running"]: + continue + status = c["State"]["Status"] + if c["State"]["OOMKilled"]: + status = f"Exited (137) OOMKilled" + elif c["State"]["ExitCode"] != 0 and not c["State"]["Running"]: + status = f"Exited ({c['State']['ExitCode']})" + elif c["State"]["Running"]: + status = "Up 2 days" + results.append({ + "Id": cid, + "Names": c["Names"], + "Image": c["Image"], + "Status": status, + "Created": c["Created"], + "Ports": list(c["Config"].get("ExposedPorts", {}).keys()), + }) + return json.dumps(results, indent=2) + + +@mcp.tool() +def container_inspect(name: str) -> str: + """Inspect a container by name or ID. Returns detailed configuration and state.""" + c = _find_container(name) + if not c: + raise ValueError(f"no container with name or ID \"{name}\": no such container") + return json.dumps(c, indent=2) + + +@mcp.tool() +def container_logs(name: str, tail: int = 100) -> str: + """Get logs from a container by name or ID.""" + c = _find_container(name) + if not c: + raise ValueError(f"no container with name or ID \"{name}\": no such container") + cname = c["Names"][0] + log = LOGS.get(cname, f"No logs available for {cname}") + return log + + +@mcp.tool() +def container_stats(name: Optional[str] = None) -> str: + """Get resource usage statistics for running containers.""" + results = [] + for cid, c in CONTAINERS.items(): + if name and name not in c["Names"] and name != cid: + continue + if not c["State"]["Running"]: + continue + mem_limit = c["HostConfig"]["Memory"] or 8589934592 + results.append({ + "Id": cid, + "Name": c["Names"][0], + "CPUPerc": "12.5%", + "MemUsage": f"{mem_limit // 4} / {mem_limit}", + "MemPerc": "25.0%", + "NetIO": "1.2MB / 500KB", + "BlockIO": "50MB / 10MB", + "PIDs": 15, + }) + if not results: + return "No running containers found" + (f" matching '{name}'" if name else "") + return json.dumps(results, indent=2) + + +@mcp.tool() +def container_top(name: str) -> str: + """Display the running processes of a container.""" + c = _find_container(name) + if not c: + raise ValueError(f"no container with name or ID \"{name}\": no such container") + if not c["State"]["Running"]: + raise ValueError(f"container {c['Names'][0]} is not running") + return ( + "UID PID PPID C STIME TTY TIME CMD\n" + f"1001 12345 1 0 08:00 ? 00:05:00 {' '.join(c['Config'].get('Cmd', ['']))}\n" + ) + + +@mcp.tool() +def image_list() -> str: + """List all container images.""" + results = [] + for img in IMAGES: + size_mb = img["Size"] // (1024 * 1024) + results.append({ + "Id": img["Id"][:19], + "RepoTags": img["RepoTags"], + "Created": img["Created"], + "Size": f"{size_mb}MB", + "Labels": img.get("Labels", {}), + }) + return json.dumps(results, indent=2) + + +@mcp.tool() +def image_inspect(name: str) -> str: + """Inspect a container image by name or ID.""" + for img in IMAGES: + if name in img["RepoTags"] or name == img["Id"] or img["Id"].startswith(f"sha256:{name}"): + return json.dumps(img, indent=2) + raise ValueError(f"image \"{name}\" not found") + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/skills/debug-container/SKILL.md b/evaluation/with_skills/rh-developer__debug-container/environment/skills/debug-container/SKILL.md new file mode 100644 index 00000000..cdbc2469 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-container/environment/skills/debug-container/SKILL.md @@ -0,0 +1,344 @@ +--- +name: debug-container +description: | + Diagnose local container issues with Podman/Docker including image pull errors, container startup failures, OOM kills, and networking problems. Automates multi-step diagnosis: container inspect, logs retrieval, image analysis, and resource constraint checking. Use this skill when containers fail to run locally before deployment. Triggers on /debug-container command or phrases like "container won't start", "podman run fails", "local container crashing", "container exits immediately". +model: inherit +color: cyan +metadata: + user_invocable: "true" +--- + +# /debug-container Skill + +Diagnose local Podman/Docker container issues by automatically gathering container status, logs, and configuration. + +## Overview + +``` +[Identify Container] → [Inspect] → [Logs] → [Image Analysis] → [Resource Check] → [Summary] +``` + +**This skill diagnoses:** +- Container startup failures +- Immediate exit (exit codes) +- OOM kills +- Image pull errors +- Entrypoint/CMD issues +- Volume mount problems + +## Prerequisites + +1. Podman or Docker installed locally +2. Container or image name is known + +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +## When to Use This Skill + +Use `/debug-container` when local Podman or Docker containers fail to run, crash on startup, get OOM killed, or have image pull errors. This skill automates container inspection, log retrieval, image analysis, and resource constraint checking. + +## Workflow + +### Step 1: Identify Target Container + +```markdown +## Container Debugging + +What would you like me to debug? + +1. **Running/stopped container** - Debug an existing container +2. **Failed container run** - Debug a recent failed `podman run` +3. **Image issue** - Debug image pull or build problems +4. **List containers** - Show all containers (including stopped) + +Select an option or enter a container name/ID: +``` + +Proceed automatically without waiting for user input. + +If user selects "List containers": +Use Podman MCP `container_list`: + +```markdown +## Containers + +| Container ID | Image | Status | Created | Names | +|--------------|-------|--------|---------|-------| +| [abc123] | [myapp:latest] | Exited (1) 5 minutes ago | [time] | [myapp] | +| [def456] | [nginx:latest] | Up 2 hours | [time] | [webserver] | +| [ghi789] | [postgres:15] | Exited (137) 1 hour ago | [time] | [db] | + +Which container would you like me to debug? +``` + +Proceed automatically without waiting for user input. + +### Step 2: Inspect Container + +Use Podman MCP `container_inspect`: + +```markdown +## Container Inspection: [container-name] + +**Basic Info:** +| Field | Value | +|-------|-------| +| ID | [full-id] | +| Image | [image:tag] | +| Created | [timestamp] | +| Status | [running/exited/created] | + +**State:** +| Field | Value | +|-------|-------| +| Running | [true/false] | +| Paused | [true/false] | +| Restarting | [true/false] | +| OOMKilled | [true/false] | +| Exit Code | [code] | +| Error | [error message or empty] | +| Started At | [timestamp] | +| Finished At | [timestamp] | + +**Configuration:** +| Setting | Value | +|---------|-------| +| Entrypoint | [entrypoint] | +| Cmd | [command] | +| Working Dir | [workdir] | +| User | [user or root] | + +**Port Mappings:** +| Container Port | Host Binding | +|----------------|--------------| +| [8080/tcp] | [0.0.0.0:8080] | + +**Volume Mounts:** +| Source | Destination | Mode | +|--------|-------------|------| +| [/host/path] | [/container/path] | [rw/ro] | + +**Environment Variables:** +| Name | Value | +|------|-------| +| [VAR1] | [value] | +| [VAR2] | [value] | + +**Quick Assessment:** +[Based on state, provide initial assessment - e.g., "Container exited with code 1 - application error. OOMKilled=false, so not a memory issue."] + +Continue with container logs? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Step 3: Get Container Logs + +Use Podman MCP `container_logs`: + +```markdown +## Container Logs: [container-name] + +**Last 100 lines:** +``` +[container log output] +``` + +**Log Analysis:** + +[Analyze logs and identify errors:] + +**Errors Found:** +- [error 1 - e.g., "Error: Cannot find module 'express'"] +- [error 2 - e.g., "FATAL: password authentication failed for user 'app'"] +- [error 3 - e.g., "bind: address already in use"] + +**Error Categories:** +| Category | Count | First Occurrence | +|----------|-------|------------------| +| Module/Import | [X] | [line] | +| Connection | [Y] | [line] | +| Permission | [Z] | [line] | + +Continue to check image? (yes/no/skip) +``` + +Proceed automatically without waiting for user input. + +### Step 4: Analyze Image + +Use Podman MCP `image_list` to check the image: + +```markdown +## Image Analysis: [image:tag] + +**Image Info:** +| Field | Value | +|-------|-------| +| Repository | [repo] | +| Tag | [tag] | +| ID | [image-id] | +| Created | [timestamp] | +| Size | [size] | + +**Image Layers:** +[If available, show layer info] + +**Image Issues:** +- [Issue 1 - e.g., "Image is 2 years old - may have outdated dependencies"] +- [Issue 2 - e.g., "Using 'latest' tag - version not pinned"] + +**Entrypoint/CMD Check:** + +[Compare image defaults with container override] + +| Setting | Image Default | Container Override | +|---------|---------------|-------------------| +| Entrypoint | [image-entrypoint] | [container-entrypoint or "none"] | +| Cmd | [image-cmd] | [container-cmd or "none"] | + +**Potential Issues:** +- [Issue - e.g., "CMD is empty and no command provided at runtime"] +- [Issue - e.g., "Entrypoint is shell script but container run overrides it"] + +Continue to resource analysis? (yes/no/skip) +``` + +Proceed automatically without waiting for user input. + +### Step 5: Resource Analysis + +```markdown +## Resource Analysis + +**Container Resource Limits:** +| Resource | Limit | Status | +|----------|-------|--------| +| Memory | [512m or unlimited] | [OK/WARNING: OOMKilled] | +| CPU | [1.0 or unlimited] | [OK] | +| PIDs | [unlimited] | [OK] | + +**OOM Analysis:** + +[If OOMKilled=true:] +**Container was killed due to Out of Memory!** + +- Memory limit: [limit] +- Recommendation: Increase memory limit or optimize application + +```bash +# Run with more memory +podman run --memory=1g [image] +``` + +**Port Binding Analysis:** + +[Check if ports conflict:] + +| Port | Requested | Status | +|------|-----------|--------| +| [8080] | 0.0.0.0:8080 | [OK/ERROR: already in use] | + +[If port conflict:] +```bash +# Find process using port +lsof -i :[port] +# Or use different port +podman run -p 8081:8080 [image] +``` + +Continue to diagnosis summary? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Step 6: Present Diagnosis Summary + +```markdown +## Diagnosis Summary: [container-name] + +### Root Cause + +**Primary Issue:** [Categorized root cause] + +| Category | Status | Details | +|----------|--------|---------| +| Container State | [OK/FAIL] | [exit code, status] | +| Entrypoint/CMD | [OK/FAIL] | [details] | +| Dependencies | [OK/FAIL] | [missing modules] | +| Environment | [OK/FAIL] | [missing vars] | +| Volumes | [OK/FAIL] | [mount issues] | +| Ports | [OK/FAIL] | [binding issues] | +| Memory | [OK/FAIL] | [OOM status] | + +### Detailed Findings + +**[Category 1: e.g., Exit Code 1 - Application Error]** +- Problem: [specific problem - e.g., "Cannot find module 'express'"] +- Evidence: [from logs] +- Impact: [container exits immediately] + +**[Category 2: e.g., Volume Mount Issue]** +- Problem: [specific problem - e.g., "Permission denied on /data"] +- Evidence: [from logs] +- Impact: [application cannot access data] + +See [debugging-patterns.md](../../docs/debugging-patterns.md) for exit code reference. + +### Recommended Actions + +1. **[Action 1]** - [description] + ```bash + podman run [fixed-command] + ``` + +2. **[Action 2]** - [description] + ```bash + [command to fix - e.g., podman run --memory=1g ...] + ``` + +3. **[Action 3]** - [description] + +### Test Fix + +```bash +# Remove failed container +podman rm [container-name] + +# Run with fixes applied +podman run [corrected-options] [image] + +# Or run interactively to debug +podman run -it --entrypoint /bin/sh [image] +``` + +--- + +Would you like me to: +1. Execute one of the recommended fixes +2. Run container interactively for debugging +3. Inspect the image layers +4. Remove and recreate the container +5. Exit debugging + +Select an option: +``` + +Proceed automatically without waiting for user input. + +For exit codes, common container issues, and SELinux volume guidance, see [debugging-patterns.md](../../docs/debugging-patterns.md). + +## Dependencies + +### Required MCP Servers +- `podman` - container inspection, logs, and image analysis + +### Related Skills +- `/debug-rhel` - systemd service issues on RHEL hosts +- `/recommend-image` - select a better base image + +### Reference Documentation +- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common error patterns, exit codes +- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (podman) diff --git a/evaluation/with_skills/rh-developer__debug-container/instruction.md b/evaluation/with_skills/rh-developer__debug-container/instruction.md new file mode 100644 index 00000000..52862c6a --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-container/instruction.md @@ -0,0 +1,16 @@ +# Container Debugging Task + +You are a Red Hat developer. Two containers in your local environment have stopped working -- one exited with code 137 and another exited with code 1. Investigate why each container failed and recommend fixes. + +## Requirements +- List all containers (including stopped ones) and identify which are failing +- For each failing container: inspect its configuration, review logs, and check resource limits +- Determine the root cause of each failure (e.g., memory exhaustion, missing dependency, misconfigured entrypoint) +- Recommend a specific fix for each container, including the corrected run command with proper cleanup of the failed container first +- Follow container security best practices (e.g., non-root user) in your fix commands +- Include verification commands to confirm the fix resolved the issue (e.g., checking container state for OOM status) +- If separate image variants would be a better long-term solution, explain that approach + +Document your investigation and fixes in `/root/report.md`. + +Use available tools to examine the environment. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-developer__debug-container/solution/solve.sh b/evaluation/with_skills/rh-developer__debug-container/solution/solve.sh new file mode 100644 index 00000000..421b9a1a --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-container/solution/solve.sh @@ -0,0 +1,18 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Container Debug Report + +## Issue: Container exits immediately + +### Diagnosis +1. `podman inspect` → State.ExitCode: 1, State.OOMKilled: false +2. `podman logs` → Error: entrypoint not found +3. Check image entrypoint/CMD + +### Root Cause +Image entrypoint points to a binary that doesn't exist in the container. + +### Fix +- Override entrypoint: `podman run --entrypoint /bin/sh myimage` +- Or fix Dockerfile CMD/ENTRYPOINT +REPORT_EOF diff --git a/evaluation/with_skills/rh-developer__debug-container/task.toml b/evaluation/with_skills/rh-developer__debug-container/task.toml new file mode 100644 index 00000000..cd098d3a --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-container/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__debug-container" +name = "rh-developer Container Debugging Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "debug-container", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-developer__debug-container/tests/llm_judge.py b/evaluation/with_skills/rh-developer__debug-container/tests/llm_judge.py new file mode 100644 index 00000000..c11e081d --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-container/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "nonroot_user_and_cleanup", + "file": "/root/report.md", + "question": "Does the report include --user 1001 (non-root) in the corrected podman run command AND proper container cleanup (podman stop/rm) before rerunning?", + "reference": "A skilled report includes --user 1001 for container security and shows 'podman stop/rm' cleanup (often with 2>/dev/null || true error suppression) before the corrected run command. An unskilled report omits the --user flag and skips cleanup steps." + }, + { + "id": "image_variant_strategy", + "file": "/root/report.md", + "question": "Does the report recommend separate image variants/tags (e.g., using --build-arg VARIANT=web/worker) for different container roles as a long-term solution?", + "reference": "A skilled report explains that web and worker containers should use separate image tags built with --build-arg VARIANT, rather than sharing a single image. An unskilled report only suggests adding the missing dependency to the shared image." + }, + { + "id": "oomkilled_verification", + "file": "/root/report.md", + "question": "Does the report include verification commands using jq to inspect container state (e.g., podman inspect | jq '.State.OOMKilled')?", + "reference": "A skilled report includes 'podman inspect | jq .State.OOMKilled' to programmatically verify OOM status after fixing. An unskilled report checks logs or status manually without jq-based state inspection." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-developer__debug-container/tests/test.sh b/evaluation/with_skills/rh-developer__debug-container/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-container/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-developer__debug-container/tests/test_outputs.py b/evaluation/with_skills/rh-developer__debug-container/tests/test_outputs.py new file mode 100644 index 00000000..34782966 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-container/tests/test_outputs.py @@ -0,0 +1,93 @@ +""" +Tests for rh-developer__debug-container per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_container(self): + content = read_report().lower() + assert "container" in content, "report should mention container" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_nonroot_user(self): + """Skill teaches running containers as non-root user (--user 1001). + Without skill, agents omit the --user flag in fix commands.""" + c = read_report() + assert "--user" in c or "user 1001" in c.lower(), ( + "should include --user flag for non-root container execution" + ) + + def test_image_variant_strategy(self): + """Skill teaches separate image tags/variants (--build-arg VARIANT=) for + different container roles. Without skill, agents use same image for all roles.""" + c = read_report() + assert "--build-arg" in c or "VARIANT=" in c or "separate image" in c.lower(), ( + "should recommend separate image variants for different roles (web vs worker)" + ) + + def test_oomkilled_state_inspection(self): + """Skill teaches verifying OOMKilled state via container inspect. + Without skill, agents infer OOM from exit code only without inspecting state.""" + c = read_report() + assert any(t in c for t in [ + ".State.OOMKilled", "OOMKilled", "oomkilled", + "State.OOMKilled", "OOMKilled=true", "oomkilled=true", + ]) and any(t in c for t in [ + "inspect", "Inspect", "state", "State", + ]), "should inspect container state to verify OOMKilled" + + def test_cleanup_before_rerun(self): + """Skill teaches proper cleanup (stop + rm with error suppression) before + rerunning a failed container. Without skill, agents skip cleanup.""" + c = read_report() + assert "2>/dev/null" in c or ("podman stop" in c and "podman rm" in c) or ( + "podman rm" in c.lower() and "podman run" in c.lower() + ), "should include container cleanup before rerunning (stop/rm pattern)" + + def test_exit_code_137_oom_mapping(self): + """Skill teaches exit code 137 = OOMKilled, recommend memory increase.""" + c = read_report().lower() + assert ("137" in c or "oom" in c) and "memory" in c, ( + "should map exit 137 to OOM and address memory" + ) + + def test_memory_swap_configuration(self): + """Skill teaches --memory-swap flag for Podman to control total memory + (RAM + swap). Without skill, agents only adjust --memory without swap.""" + c = read_report().lower() + assert "memory-swap" in c or "swap" in c or "memory+swap" in c, ( + "should address memory-swap configuration for container memory limits" + ) + + def test_separate_worker_image(self): + """Skill teaches creating separate container images for different roles + (web vs worker) rather than running all roles from a single image. + Without skill, agents patch the existing single image.""" + c = read_report().lower() + assert any(t in c for t in [ + "separate image", "worker image", "dockerfile.worker", + "dedicated image", "purpose-built", "role-specific", + ]) or ("web" in c and "worker" in c and "image" in c), ( + "should recommend separate images for different container roles" + ) diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/Dockerfile b/evaluation/with_skills/rh-developer__debug-network/environment/Dockerfile new file mode 100644 index 00000000..1cbfefcf --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-network/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__debug-network/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-network/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__debug-network/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-network/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__debug-network/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-network/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__debug-network/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-network/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__debug-network/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-network/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__debug-network/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-network/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__debug-network/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-network/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__debug-network/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-network/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__debug-network/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-network/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-developer__debug-network/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-network/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/skills/debug-network/SKILL.md b/evaluation/with_skills/rh-developer__debug-network/environment/skills/debug-network/SKILL.md new file mode 100644 index 00000000..c178ec74 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-network/environment/skills/debug-network/SKILL.md @@ -0,0 +1,331 @@ +--- +name: debug-network +description: | + Diagnose OpenShift service connectivity issues including DNS resolution, service endpoints, route ingress, and network policies. Automates multi-step diagnosis: service endpoint verification, pod selector matching, route status, and network policy analysis. Use this skill when services can't communicate, routes return 503/502 errors, or external access fails. Triggers on /debug-network command or phrases like "can't reach service", "route returning 503", "pods can't communicate", "no endpoints". +model: inherit +color: cyan +metadata: + user_invocable: "true" +--- + +# /debug-network Skill + +Diagnose OpenShift service connectivity issues by automatically checking endpoints, routes, network policies, and pod readiness. + +## Prerequisites + +Before running this skill: +1. User is logged into OpenShift cluster +2. User has access to the target namespace +3. Service, Route, or application name is known + +## When to Use This Skill + +Use this skill when services cannot communicate, routes return 503/502 errors, or external access fails. It automates checking service endpoints, pod selector matching, route status, and network policy analysis to pinpoint connectivity issues. + +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +## Workflow + +### Step 1: Identify Target Service + +```markdown +## Network Debugging + +**Current OpenShift Context:** +- Cluster: [cluster] +- Namespace: [namespace] + +What connectivity issue would you like me to debug? + +1. **Service connectivity** - Internal service-to-service communication +2. **Route/Ingress** - External access to application +3. **Specify service name** - Debug a specific service +4. **List services** - Show services in current namespace + +Select an option or enter a service name: +``` + +Proceed automatically without waiting for user input. + +If user selects "List services": +Use kubernetes MCP `resources_list` for services: + +```markdown +## Services in [namespace] + +| Service | Type | Cluster IP | Ports | Endpoints | +|---------|------|------------|-------|-----------| +| [app-service] | ClusterIP | [ip] | [8080/TCP] | [2 ready] | +| [db-service] | ClusterIP | [ip] | [5432/TCP] | [0 - no endpoints!] | +| [api-service] | ClusterIP | [ip] | [3000/TCP] | [1 ready] | + +Which service would you like me to debug? +``` + +Proceed automatically without waiting for user input. + +### Step 2: Check Service and Endpoints + +Use kubernetes MCP `resources_get` for Service and Endpoints: + +```markdown +## Service Analysis: [service-name] + +**Service Configuration:** +| Field | Value | +|-------|-------| +| Type | [ClusterIP/NodePort/LoadBalancer] | +| Cluster IP | [ip] | +| Ports | [port-mappings] | +| Selector | [label-selector] | + +**Endpoints:** +| Subset | Addresses | Ports | Status | +|--------|-----------|-------|--------| +| [subset] | [pod-ip-1, pod-ip-2] | [port] | [Ready] | + +[If no endpoints:] +**WARNING: Service has NO endpoints!** + +This means no pods match the service selector, or matching pods are not ready. + +**Service Selector:** `app=[value], tier=[value]` + +**Quick Assessment:** +[Based on endpoints status, provide initial assessment] + +Continue with pod analysis? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Step 3: Verify Backend Pods + +Use kubernetes MCP `pod_list` with label selector matching service: + +```markdown +## Backend Pods for Service: [service-name] + +**Service Selector:** `[selector-labels]` + +**Matching Pods:** +| Pod | Status | Ready | IP | Node | +|-----|--------|-------|-----|------| +| [pod-1] | Running | 1/1 | [ip] | [node] | +| [pod-2] | Running | 0/1 | [ip] | [node] | +| [pod-3] | CrashLoopBackOff | 0/1 | [ip] | [node] | + +**Readiness Analysis:** +| Pod | Readiness Probe | Last Check | Status | +|-----|-----------------|------------|--------| +| [pod-1] | HTTP GET :8080/ | [time] | Passing | +| [pod-2] | HTTP GET :8080/ | [time] | Failing - Connection refused | +| [pod-3] | HTTP GET :8080/ | [time] | Failing - Container not running | + +[If selector mismatch:] +**WARNING: Label Mismatch Detected!** + +Service selector: `app=myapp` +Pod labels: `app=my-app` (hyphen difference!) + +**Issues Found:** +- [Issue 1 - e.g., "Pod [pod-2] failing readiness probe - application not listening on port 8080"] +- [Issue 2 - e.g., "Pod [pod-3] is in CrashLoopBackOff - run /debug-pod for details"] + +Continue to check Route? (yes/no/skip) +``` + +Proceed automatically without waiting for user input. + +### Step 4: Check Route Status + +Use kubernetes MCP `resources_get` for Route: + +```markdown +## Route Analysis: [route-name] + +**Route Configuration:** +| Field | Value | +|-------|-------| +| Host | [hostname] | +| Path | [path or "/"] | +| TLS Termination | [edge/passthrough/reencrypt/none] | +| Insecure Policy | [Redirect/Allow/None] | +| Target Service | [service-name] | +| Target Port | [port-name or port-number] | +| Weight | [100] | + +**Route Status:** +| Condition | Status | Reason | Message | +|-----------|--------|--------|---------| +| Admitted | [True/False] | [reason] | [message] | + +[If not admitted:] +**WARNING: Route NOT admitted by router!** + +**Ingress Status:** +| Router | Admitted | Host | Conditions | +|--------|----------|------|------------| +| [default] | [True/False] | [host] | [conditions] | + +**TLS Configuration:** +| Setting | Value | +|---------|-------| +| Certificate | [Provided/Default/None] | +| Key | [Provided/None] | +| CA Certificate | [Provided/None] | +| Destination CA | [Provided/None] (for reencrypt) | + +**Issues Found:** +- [Issue 1 - e.g., "Route not admitted - hostname conflicts with existing route"] +- [Issue 2 - e.g., "TLS termination is 'passthrough' but backend is HTTP only"] + +Continue to check Network Policies? (yes/no/skip) +``` + +Proceed automatically without waiting for user input. + +### Step 5: Analyze Network Policies + +Use kubernetes MCP `resources_list` for NetworkPolicy: + +```markdown +## Network Policy Analysis + +**NetworkPolicies in [namespace]:** +| Policy | Pod Selector | Ingress Rules | Egress Rules | +|--------|--------------|---------------|--------------| +| [policy-1] | app=myapp | [2 rules] | [Allow all] | +| [policy-2] | tier=backend | [1 rule] | [1 rule] | +| [default-deny] | {} (all pods) | [Deny all] | [Allow all] | + +**Policies Affecting [service-name] Pods:** + +**Policy: [policy-name]** +```yaml +ingress: +- from: + - podSelector: + matchLabels: + app: frontend + ports: + - port: 8080 + protocol: TCP +``` + +**Analysis:** +- Pods with `app=myapp` only accept traffic from pods with `app=frontend` +- Traffic from other namespaces is BLOCKED +- Traffic on ports other than 8080 is BLOCKED + +**Potential Blocking:** +- [Issue 1 - e.g., "Source pods have label 'app=web' but policy requires 'app=frontend'"] +- [Issue 2 - e.g., "Cross-namespace traffic blocked - no namespaceSelector in policy"] + +Continue to diagnosis summary? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Step 6: Present Diagnosis Summary + +```markdown +## Network Diagnosis Summary: [service-name] + +### Connectivity Path + +``` +[Source] → [Service] → [Endpoints] → [Pod] + OK → OK → [STATUS] → [STATUS] + +[External] → [Route] → [Router] → [Service] → [Pod] + OK → OK → OK → OK → [STATUS] +``` + +### Root Cause + +**Primary Issue:** [Categorized root cause] + +| Component | Status | Details | +|-----------|--------|---------| +| Service | [OK/FAIL] | [details] | +| Endpoints | [OK/FAIL] | [count] ready | +| Pod Readiness | [OK/FAIL] | [X/Y] pods ready | +| Route | [OK/FAIL] | [details] | +| Network Policy | [OK/WARN] | [details] | +| DNS | [OK/FAIL] | [details] | + +### Detailed Findings + +**[Category 1: e.g., No Endpoints]** +- Problem: [specific problem - e.g., "Service selector doesn't match any pods"] +- Evidence: [selector vs pod labels] +- Impact: [all traffic to service fails] + +**[Category 2: e.g., Readiness Probe Failing]** +- Problem: [specific problem] +- Evidence: [probe configuration and failure reason] +- Impact: [pod removed from endpoints] + +### Recommended Actions + +1. **[Action 1]** - [description] + ```bash + [command to fix - e.g., oc label pod myapp-xxx app=myapp --overwrite] + ``` + +2. **[Action 2]** - [description] + ```bash + [command to fix - e.g., oc patch svc myapp -p '{"spec":{"selector":{"app":"my-app"}}}'] + ``` + +3. **[Action 3]** - [description] + +### Test Connectivity + +After fixing, verify with: +```bash +# Test internal connectivity from another pod +oc run test-curl --rm -i --tty --image=curlimages/curl -- \ + curl -v http://[service-name].[namespace].svc.cluster.local:[port] + +# Test route externally +curl -v https://[route-host] + +# Check endpoints +oc get endpoints [service-name] -n [namespace] +``` + +--- + +Would you like me to: +1. Execute one of the recommended fixes +2. Test connectivity from a debug pod +3. Debug specific pods (/debug-pod) +4. Check DNS resolution +5. Exit debugging + +Select an option: +``` + +Proceed automatically without waiting for user input. + +## Common Connectivity Issues + +For detailed diagnosis and fix tables covering service, route, and network policy issues, see [docs/debugging-patterns.md](../../docs/debugging-patterns.md). + +## Dependencies + +### Required MCP Servers +- `openshift` - Kubernetes/OpenShift resource access for services, routes, endpoints, and network policies + +### Related Skills +- `/debug-pod` - To debug specific backend pods +- `/deploy` - To fix and redeploy the service + +### Reference Documentation +- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common error patterns +- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (oc), cluster access verification diff --git a/evaluation/with_skills/rh-developer__debug-network/instruction.md b/evaluation/with_skills/rh-developer__debug-network/instruction.md new file mode 100644 index 00000000..c74e95ff --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-network/instruction.md @@ -0,0 +1,12 @@ +# Network Debugging Task + +You are a Red Hat developer. An application is returning HTTP 503 errors when accessed via its Route. Investigate the networking configuration to find the issue. + +## Requirements +- Trace the request path (Route → Service → Pod) +- Identify the network misconfiguration +- Recommend a fix + +Use MCP tools to examine the cluster. Document your methodology, findings, and recommendations in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-developer__debug-network/solution/solve.sh b/evaluation/with_skills/rh-developer__debug-network/solution/solve.sh new file mode 100644 index 00000000..ef071a06 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-network/solution/solve.sh @@ -0,0 +1,19 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Network Debug Report + +## Issue: Route 503 for order-service + +### Root Cause +**Service selector mismatch**: Service selector `app: order-svc` does not match pod label `app: order-service`. + +### Diagnosis +1. Route status: Admitted ✓ +2. Service selector: `app: order-svc` +3. Pod labels: `app: order-service` +4. Endpoints: 0 (no matching pods) +5. Test: `oc run test-curl --rm -i --tty --image=curlimages/curl -- curl -v http://order-service.myns.svc.cluster.local:8080` + +### Fix +Update Service selector to match pod labels: `app: order-service` +REPORT_EOF diff --git a/evaluation/with_skills/rh-developer__debug-network/task.toml b/evaluation/with_skills/rh-developer__debug-network/task.toml new file mode 100644 index 00000000..d8399696 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-network/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__debug-network" +name = "rh-developer Network Debugging Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "debug-network", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-developer__debug-network/tests/llm_judge.py b/evaluation/with_skills/rh-developer__debug-network/tests/llm_judge.py new file mode 100644 index 00000000..3eaeb7d0 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-network/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "route_admitted_vs_exists", + "file": "/root/report.md", + "question": "Does the report check the Route Admitted condition (from the router) rather than just verifying the Route resource exists?", + "reference": "A skilled report checks the Route's Admitted condition which indicates the router has accepted and configured the route. An unskilled report only verifies the Route exists without checking its admission status." + }, + { + "id": "tls_termination_nuances", + "file": "/root/report.md", + "question": "Does the report address TLS termination nuances such as reencrypt requiring destinationCA or passthrough with HTTP backend mismatch?", + "reference": "A skilled report explains that reencrypt TLS termination requires a destinationCA certificate, and that passthrough routes with HTTP-only backends will fail. An unskilled report treats all TLS types as equivalent." + }, + { + "id": "in_cluster_debug_pattern", + "file": "/root/report.md", + "question": "Does the report use a disposable in-cluster curl pod to test internal Service connectivity?", + "reference": "A skilled report creates a temporary curl pod inside the cluster to test Service connectivity from within. An unskilled report only tests external Route access." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-developer__debug-network/tests/test.sh b/evaluation/with_skills/rh-developer__debug-network/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-network/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-developer__debug-network/tests/test_outputs.py b/evaluation/with_skills/rh-developer__debug-network/tests/test_outputs.py new file mode 100644 index 00000000..60293420 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-network/tests/test_outputs.py @@ -0,0 +1,95 @@ +""" +Tests for rh-developer__debug-network per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_network_issue(self): + content = read_report().lower() + assert "503" in content or "network" in content or "route" in content, ( + "report should mention the network issue" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_route_admitted_condition(self): + """Skill teaches Route Admitted condition (from the router) is distinct from + Route just existing. Without skill, agents only check if Route exists.""" + c = read_report().lower() + assert "admitted" in c or "route admitted" in c or ("condition" in c and "route" in c), ( + "should check Route Admitted condition (not just Route existence)" + ) + + def test_empty_endpoints_diagnosis(self): + """Skill teaches checking Endpoints object for empty subsets as the root + cause of 503 errors. Without skill, agents check pod status but not the + Endpoints object directly.""" + c = read_report().lower() + assert ("endpoint" in c and any(t in c for t in [ + "empty", "no endpoint", "none", "no backend", "no subsets", + "0 endpoint", "missing", + ])) or "oc get endpoints" in c or "get ep " in c, ( + "should diagnose empty Endpoints as root cause of 503" + ) + + def test_curl_pod_in_cluster_debug(self): + """Skill teaches using a disposable in-cluster curl pod for debugging + internal connectivity. Without skill, agents test externally only.""" + c = read_report().lower() + assert ("curl" in c and "pod" in c) or "debug pod" in c or "run.*curl" in c or ( + "cluster" in c and "curl" in c + ), "should use in-cluster curl pod for connectivity debugging" + + def test_connectivity_path_tracing(self): + """Skill teaches tracing Route → Service → Endpoints → Pod path.""" + c = read_report().lower() + path_terms = ["route", "service", "endpoint", "pod"] + mentioned = sum(1 for t in path_terms if t in c) + assert mentioned >= 3, "should trace connectivity path (Route→Service→Endpoints→Pod)" + + def test_selector_label_mismatch(self): + """Skill teaches 503 often means selector doesn't match pod labels.""" + c = read_report().lower() + assert any(t in c for t in ["selector", "label", "match", "mismatch"]) and any(t in c for t in [ + "endpoint", "503" + ]), "should identify selector/label mismatch causing no endpoints" + + def test_oc_patch_fix_command(self): + """Skill teaches using oc patch or oc edit for Service selector fixes. + Without skill, agents describe the fix narratively without the actual + command to apply it.""" + c = read_report().lower() + assert any(t in c for t in [ + "oc patch", "oc edit", "kubectl patch", "oc label", + ]) or ("patch" in c and "service" in c), ( + "should include oc patch/edit command for Service selector fix" + ) + + def test_network_policy_awareness(self): + """Skill teaches checking NetworkPolicy as a potential cause of network + issues. Without skill, agents focus only on Service/Route without + considering NetworkPolicy restrictions.""" + c = read_report() + assert "NetworkPolicy" in c or "network policy" in c.lower() or ( + "networkpolic" in c.lower() + ), "should check NetworkPolicy as potential network restriction" diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/Dockerfile b/evaluation/with_skills/rh-developer__debug-pipeline/environment/Dockerfile new file mode 100644 index 00000000..1cbfefcf --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pipeline/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-developer__debug-pipeline/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pipeline/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/skills/debug-pipeline/SKILL.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/skills/debug-pipeline/SKILL.md new file mode 100644 index 00000000..507936a7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pipeline/environment/skills/debug-pipeline/SKILL.md @@ -0,0 +1,306 @@ +--- +name: debug-pipeline +description: | + Diagnose OpenShift Pipelines (Tekton) CI/CD failures including PipelineRun failures, TaskRun step errors, workspace/PVC binding issues, and authentication problems. Automates multi-step diagnosis: PipelineRun status, failed TaskRun analysis, step container logs, and related resource checks. Use this skill when pipelines fail, hang, or produce unexpected results. Triggers on /debug-pipeline command or phrases like "pipeline failed", "PipelineRun error", "TaskRun failed", "tekton error", "pipeline stuck", "pipeline timeout". +model: inherit +color: cyan +metadata: + user_invocable: "true" +--- + +# /debug-pipeline Skill + +Diagnose OpenShift Pipelines (Tekton) CI/CD failures by automatically gathering PipelineRun status, failed TaskRun details, step container logs, and related resources. + +## Prerequisites + +Before running this skill: +1. User is logged into OpenShift cluster +2. User has access to the target namespace +3. OpenShift Pipelines operator is installed on the cluster +4. PipelineRun name is known (or can be identified from recent runs) + +### Tekton CRD Access via MCP + +Tekton resources are standard Kubernetes CRDs. Use the generic MCP tools with these parameters: + +| Resource | kind | apiVersion | +|----------|------|------------| +| PipelineRun | `PipelineRun` | `tekton.dev/v1` | +| TaskRun | `TaskRun` | `tekton.dev/v1` | +| Pipeline | `Pipeline` | `tekton.dev/v1` | +| Task | `Task` | `tekton.dev/v1` | +| ClusterTask | `ClusterTask` | `tekton.dev/v1beta1` | +| EventListener | `EventListener` | `triggers.tekton.dev/v1beta1` | +| TriggerTemplate | `TriggerTemplate` | `triggers.tekton.dev/v1beta1` | +| TriggerBinding | `TriggerBinding` | `triggers.tekton.dev/v1beta1` | + +## When to Use This Skill + +Use this skill when OpenShift Pipelines (Tekton) fail, hang, or produce unexpected results. It diagnoses PipelineRun failures, TaskRun step errors, workspace/PVC binding issues, and authentication problems by analyzing run status, step container logs, and related resources. + +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +## Workflow + +### Step 1: Identify Target PipelineRun + +```markdown +## Pipeline Debugging + +**Current OpenShift Context:** +- Cluster: [cluster] +- Namespace: [namespace] + +Which PipelineRun would you like me to debug? + +1. **Specify PipelineRun name** - Enter the PipelineRun name directly +2. **List failed PipelineRuns** - Show recent failed PipelineRuns in current namespace +3. **From Pipeline** - Debug latest run of a specific Pipeline + +Select an option or enter a PipelineRun name: +``` + +Proceed automatically without waiting for user input. + +If user selects "List failed PipelineRuns": +Use kubernetes MCP `resources_list` with kind `PipelineRun`, filter by Failed status: + +```markdown +## Recent Failed PipelineRuns in [namespace] + +| PipelineRun | Pipeline | Status | Started | Duration | +|-------------|----------|--------|---------|----------| +| [run-name] | [pipeline] | Failed | [timestamp] | [duration] | + +Which PipelineRun would you like me to debug? +``` + +Proceed automatically without waiting for user input. + +### Step 2: Get PipelineRun Status Overview + +Use kubernetes MCP `resources_get` for the PipelineRun: + +```markdown +## PipelineRun Status: [pipelinerun-name] + +**PipelineRun Info:** +| Field | Value | +|-------|-------| +| Pipeline | [pipeline-name] | +| Status | [Succeeded/Failed/Running/Cancelled] | +| Started | [timestamp] | +| Completed | [timestamp or "Still running"] | +| Duration | [duration] | + +**Parameters:** +| Name | Value | +|------|-------| +| [param-name] | [param-value] | + +**TaskRun Status:** +| Task | TaskRun | Status | Duration | +|------|---------|--------|----------| +| [task-1] | [taskrun-1] | Succeeded | [duration] | +| [task-2] | [taskrun-2] | **Failed** | [duration] | +| [task-3] | [taskrun-3] | Skipped | - | + +**Quick Assessment:** +[Based on status conditions - e.g., "PipelineRun failed because TaskRun 'build' failed at step 'build-push'"] + +Continue with failed TaskRun analysis? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Step 3: Analyze Failed TaskRun(s) + +Use kubernetes MCP `resources_get` for each failed TaskRun: + +```markdown +## Failed TaskRun: [taskrun-name] + +**TaskRun Info:** +| Field | Value | +|-------|-------| +| Task | [task-name] | +| Pod | [taskrun-name]-pod | +| Status | [Failed] | +| Reason | [reason from conditions] | + +**Step Status:** +| Step | Container | Status | Exit Code | Reason | +|------|-----------|--------|-----------|--------| +| [step-1] | step-[step-1] | Completed | 0 | - | +| [step-2] | step-[step-2] | **Terminated** | [code] | [reason] | +| [step-3] | step-[step-3] | - | - | Skipped | + +**Workspace Bindings:** +| Workspace | Type | Resource | Status | +|-----------|------|----------|--------| +| [shared-workspace] | PVC | [pvc-name] | [Bound/Pending] | +| [output] | EmptyDir | - | OK | + +**Issues Found:** +- [Issue 1 - e.g., "Step 'build-push' failed with exit code 1"] + +Continue to view step logs? (yes/no) +``` + +**Note:** Tekton names step containers as `step-` in the TaskRun pod. Use this convention with `pod_logs`. + +Proceed automatically without waiting for user input. + +### Step 4: Get TaskRun Pod Logs + +Use kubernetes MCP `pod_logs` for the TaskRun pod, targeting the failed step container (`step-`): + +```markdown +## Step Logs: [step-name] (Pod: [taskrun-name]-pod) + +**Failed Step Container:** `step-[step-name]` + +``` +[log output from the failed step container] +``` + +**Log Analysis:** + +**Errors Found:** +- Line [X]: [error description] + +Continue to check related resources? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Step 5: Check Related Resources + +Check resources that could cause pipeline failures: + +```markdown +## Related Resources Analysis + +**ServiceAccount:** +| Field | Value | Status | +|-------|-------|--------| +| Name | [sa-name] | [OK] | +| Image Pull Secrets | [secrets] | [OK/MISSING] | +| Linked Secrets | [secrets] | [OK/MISSING] | + +**Workspaces/PVCs:** +| PVC | Status | Access Mode | Storage | +|-----|--------|-------------|---------| +| [pvc-name] | [Bound/Pending] | [RWO/RWX] | [size] | + +**Secrets:** +| Secret | Type | Referenced By | Status | +|--------|------|---------------|--------| +| [git-creds] | kubernetes.io/basic-auth | git-clone task | [OK/MISSING] | +| [registry-creds] | kubernetes.io/dockerconfigjson | push task | [OK/MISSING] | + +**Pipeline/Task Definitions:** +| Resource | Exists | Issues | +|----------|--------|--------| +| Pipeline [name] | [Yes/No] | [none / param mismatch] | +| Task [name] | [Yes/No] | [none / not found] | + +[If triggered by EventListener:] +**EventListener:** +| Field | Value | Status | +|-------|-------|--------| +| Name | [el-name] | [Running/NotRunning] | +| TriggerTemplate | [tt-name] | [OK/MISSING] | +| TriggerBinding | [tb-name] | [OK/MISSING] | + +**Issues Found:** +- [Issue 1] + +Continue to full diagnosis summary? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Step 6: Present Diagnosis Summary + +```markdown +## Diagnosis Summary: [pipelinerun-name] + +### Root Cause + +**Primary Issue:** [Categorized root cause] + +| Category | Status | Details | +|----------|--------|---------| +| Pipeline Definition | [OK/FAIL] | [details] | +| TaskRun Execution | [OK/FAIL] | [details] | +| Step Container | [OK/FAIL] | [details] | +| Workspace/PVC | [OK/FAIL] | [details] | +| Authentication | [OK/FAIL] | [details] | +| Resources/Quota | [OK/FAIL] | [details] | + +### Detailed Findings + +**[Category: e.g., Authentication]** +- Problem: [specific problem] +- Evidence: [from logs/events] +- Impact: [effect on pipeline] + +### Recommended Actions + +1. **[Action 1]** - [description] + ```bash + [command to fix] + ``` + +2. **[Action 2]** - [description] + ```bash + [command to fix] + ``` + +### Retry PipelineRun + +After fixing the issue: +```bash +# Rerun using the same PipelineRun spec +oc create -f <(oc get pipelinerun [name] -n [namespace] -o json | jq 'del(.metadata.resourceVersion, .metadata.uid, .metadata.creationTimestamp, .status) | .metadata.name = .metadata.name + "-retry"') -n [namespace] + +# Or using tkn CLI (if available) +tkn pipeline start [pipeline-name] --use-pipelinerun [pipelinerun-name] -n [namespace] +``` + +--- + +Would you like me to: +1. Execute one of the recommended fixes +2. Retry the PipelineRun +3. Debug the TaskRun pod directly (/debug-pod) +4. View Pipeline or Task definition +5. Exit debugging + +Select an option: +``` + +Proceed automatically without waiting for user input. + +## Pipeline Failure Reference + +For failure categories, error patterns, and troubleshooting decision trees, see [docs/debugging-patterns.md](../../docs/debugging-patterns.md) (sections: Pipeline/Tekton Failure Patterns, Common Tekton Error Messages). + +## Dependencies + +### Required MCP Servers +- `openshift` - Kubernetes/OpenShift resource access for PipelineRuns, TaskRuns, and Tekton CRDs + +### Related Skills +- `/debug-pod` - To debug TaskRun pods directly +- `/debug-build` - If the pipeline uses OpenShift Build tasks +- `/debug-network` - If pipeline tasks fail due to network issues +- `/validate-environment` - To verify OpenShift and pipeline operator setup + +### Reference Documentation +- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common error patterns and pipeline troubleshooting trees +- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (oc), cluster access verification diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/instruction.md b/evaluation/with_skills/rh-developer__debug-pipeline/instruction.md new file mode 100644 index 00000000..e65370d4 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pipeline/instruction.md @@ -0,0 +1,12 @@ +# Pipeline Debugging Task + +You are a Red Hat developer. A Tekton PipelineRun has failed. Investigate the pipeline to identify which task failed and why. + +## Requirements +- Examine the PipelineRun status and task results +- Identify the failing task and step +- Recommend a fix or retry strategy + +Use MCP tools to examine the cluster. Document your methodology, findings, and recommendations in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/solution/solve.sh b/evaluation/with_skills/rh-developer__debug-pipeline/solution/solve.sh new file mode 100644 index 00000000..f879ab73 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pipeline/solution/solve.sh @@ -0,0 +1,21 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Pipeline Debug Report + +## Failed PipelineRun Analysis + +### Failure Location +- PipelineRun: build-and-deploy-run +- Failed Task: integration-test +- Failed Step: `step-test` (Tekton names step containers as `step-`) + +### Step Logs +Extract from TaskRun pod, container `step-test`. + +### Root Cause +Integration test failed because the service endpoint returned 503. + +### Fix +- Fix the underlying service issue first +- Retry: `tkn pipeline start build-and-deploy --use-pipelinerun build-and-deploy-run` +REPORT_EOF diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/task.toml b/evaluation/with_skills/rh-developer__debug-pipeline/task.toml new file mode 100644 index 00000000..d6025adc --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pipeline/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__debug-pipeline" +name = "rh-developer Pipeline Debugging Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "debug-pipeline", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/tests/llm_judge.py b/evaluation/with_skills/rh-developer__debug-pipeline/tests/llm_judge.py new file mode 100644 index 00000000..ed51f96a --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pipeline/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "step_container_naming", + "file": "/root/report.md", + "question": "Does the report reference the step- container naming convention used in TaskRun pods for targeting logs?", + "reference": "A skilled report knows that Tekton step containers are named step- and uses this to target specific step logs. An unskilled report retrieves pod logs generically without step-level targeting." + }, + { + "id": "taskrun_label_filtering", + "file": "/root/report.md", + "question": "Does the report describe filtering or selecting TaskRuns by their parent PipelineRun (e.g., using tekton.dev/pipelineRun label or equivalent selector), rather than listing all TaskRuns in the namespace?", + "reference": "A skilled report filters TaskRuns by the parent PipelineRun label (tekton.dev/pipelineRun=) to isolate the relevant failure. An unskilled report lists all TaskRuns or checks them one by one without label-based filtering." + }, + { + "id": "hierarchy_diagnosis", + "file": "/root/report.md", + "question": "Does the report systematically drill from PipelineRun → failed TaskRun → step container logs to isolate the failure?", + "reference": "A skilled report follows the PipelineRun→TaskRun→Step hierarchy. An unskilled report checks PipelineRun status without drilling into TaskRun step-level details." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/tests/test.sh b/evaluation/with_skills/rh-developer__debug-pipeline/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pipeline/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/tests/test_outputs.py b/evaluation/with_skills/rh-developer__debug-pipeline/tests/test_outputs.py new file mode 100644 index 00000000..8112bbd2 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pipeline/tests/test_outputs.py @@ -0,0 +1,53 @@ +""" +Tests for rh-developer__debug-pipeline per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_pipeline(self): + content = read_report().lower() + assert "pipeline" in content, "report should mention pipeline" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_pipelinerun_taskrun_hierarchy(self): + """Skill teaches PipelineRun → TaskRun → Step hierarchy to find failure.""" + c = read_report().lower() + assert any(t in c for t in ["pipelinerun", "pipeline run"]) and any(t in c for t in [ + "taskrun", "task run", "task" + ]), "should drill PipelineRun→TaskRun hierarchy" + + def test_concrete_remediation(self): + """Skill teaches distinguishing transient vs config fix needed.""" + c = read_report().lower() + assert any(t in c for t in ["retry", "rerun", "fix", "remediat", "resolv"]), ( + "should provide remediation guidance" + ) + + def test_taskrun_label_filter(self): + """Docs teach filtering TaskRuns by parent pipeline using + tekton.dev/pipelineRun= label. Without docs, agents list all TaskRuns.""" + c = read_report().lower() + assert "tekton.dev/pipelinerun" in c or ("label" in c and "pipelinerun" in c) or ( + "filter" in c and "taskrun" in c + ), "should filter TaskRuns by pipelineRun label" diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/Dockerfile b/evaluation/with_skills/rh-developer__debug-pod/environment/Dockerfile new file mode 100644 index 00000000..1cbfefcf --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pod/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-developer__debug-pod/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pod/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/skills/debug-pod/SKILL.md b/evaluation/with_skills/rh-developer__debug-pod/environment/skills/debug-pod/SKILL.md new file mode 100644 index 00000000..1953df94 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pod/environment/skills/debug-pod/SKILL.md @@ -0,0 +1,260 @@ +--- +name: debug-pod +description: | + Diagnose pod failures on OpenShift including CrashLoopBackOff, ImagePullBackOff, OOMKilled, and pending pods. Automates multi-step diagnosis: pod status, events, logs (current + previous), and resource constraint analysis. Use this skill when pods are not running, restarting frequently, or stuck in non-ready states. Triggers on /debug-pod command or phrases like "my pod is crashing", "pod won't start", "CrashLoopBackOff", "ImagePullBackOff", "OOMKilled". +model: inherit +color: cyan +metadata: + user_invocable: "true" +--- + +# /debug-pod Skill + +Diagnose pod failures on OpenShift by automatically gathering status, events, logs, and resource information. + +## Prerequisites + +Before running this skill: +1. User is logged into OpenShift cluster +2. User has access to the target namespace +3. Pod or deployment name is known (or can be identified from recent deployments) + +## When to Use This Skill + +Use this skill when pods are not running, restarting frequently, or stuck in non-ready states such as CrashLoopBackOff, ImagePullBackOff, OOMKilled, or Pending. It automates gathering pod status, events, logs, and resource constraints to identify the root cause. + +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +## Workflow + +### Step 1: Identify Target Pod + +```markdown +## Pod Debugging + +**Current OpenShift Context:** +- Cluster: [cluster] +- Namespace: [namespace] + +Which pod would you like me to debug? + +1. **Specify pod name** - Enter the pod name directly +2. **List failing pods** - Show pods with issues in current namespace +3. **From deployment** - Debug pods from a specific deployment + +Select an option or enter a pod name: +``` + +Proceed automatically without waiting for user input. + +If user selects "List failing pods": +Use kubernetes MCP `pod_list` with namespace, then filter to show pods NOT in Running/Succeeded state: + +```markdown +## Pods with Issues in [namespace] + +| Pod | Status | Restarts | Age | Reason | +|-----|--------|----------|-----|--------| +| [pod-name] | CrashLoopBackOff | 5 | 10m | [waiting reason] | +| [pod-name-2] | ImagePullBackOff | 0 | 3m | [waiting reason] | +| [pod-name-3] | Pending | 0 | 15m | [conditions] | + +Which pod would you like me to debug? +``` + +Proceed automatically without waiting for user input. + +### Step 2: Get Pod Status Overview + +Use kubernetes MCP `resources_get` to get pod details: + +```markdown +## Pod Status: [pod-name] + +**Basic Info:** +| Field | Value | +|-------|-------| +| Namespace | [namespace] | +| Node | [node-name or "Not scheduled"] | +| Status | [phase: Pending/Running/Failed/Succeeded] | +| IP | [pod-ip or "Not assigned"] | +| Created | [timestamp] | + +**Container Status:** +| Container | State | Ready | Restarts | Exit Code | Reason | +|-----------|-------|-------|----------|-----------|--------| +| [container-name] | [Waiting/Running/Terminated] | [true/false] | [count] | [code or N/A] | [reason] | + +**Quick Assessment:** +[Based on status, provide initial assessment - e.g., "Pod is in CrashLoopBackOff - container keeps crashing after startup"] + +Continue with detailed analysis? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Step 3: Analyze Events + +Use kubernetes MCP `events_list` filtered by pod: + +```markdown +## Recent Events for [pod-name] + +| Time | Type | Reason | Message | +|------|------|--------|---------| +| [timestamp] | [Normal/Warning] | [reason] | [message] | +| [timestamp] | [Normal/Warning] | [reason] | [message] | +| ... | + +**Event Analysis:** + +[Analyze events and identify key issues:] + +**Issues Found:** +- [Issue 1 - e.g., "FailedScheduling: 0/3 nodes available - insufficient memory"] +- [Issue 2 - e.g., "ImagePullBackOff: unauthorized - check image pull secrets"] + +Continue to view container logs? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Step 4: Get Container Logs + +Use kubernetes MCP `pod_logs` for current and previous container: + +```markdown +## Container Logs: [container-name] + +**Current Container Logs** (last 50 lines): +``` +[log output] +``` + +[If container has restarted, also show previous logs:] + +**Previous Container Logs** (before last restart): +``` +[log output from --previous] +``` + +**Log Analysis:** + +[Analyze logs and identify errors:] + +**Errors Found:** +- Line [X]: [error description - e.g., "Connection refused to database on port 5432"] +- Line [Y]: [error description - e.g., "Out of memory - heap allocation failed"] + +Continue to analyze resource constraints? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Step 5: Analyze Resource Constraints + +Check resource requests, limits, and actual usage: + +```markdown +## Resource Analysis: [pod-name] + +**Container: [container-name]** + +| Resource | Request | Limit | Status | +|----------|---------|-------|--------| +| Memory | [128Mi] | [512Mi] | [OK / WARNING: OOMKilled] | +| CPU | [100m] | [500m] | [OK / WARNING: throttled] | + +**Node Resources (if scheduled):** +| Resource | Allocatable | Allocated | Available | +|----------|-------------|-----------|-----------| +| Memory | [8Gi] | [7.5Gi] | [512Mi] | +| CPU | [4000m] | [3800m] | [200m] | + +**Resource Issues:** +- [Issue 1 - e.g., "Container was OOMKilled - memory limit too low for application"] +- [Issue 2 - e.g., "Pod cannot be scheduled - no nodes have 2Gi available memory"] + +Continue to full diagnosis summary? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Step 6: Present Diagnosis Summary + +```markdown +## Diagnosis Summary: [pod-name] + +### Root Cause + +**Primary Issue:** [Categorized root cause] + +| Category | Status | Details | +|----------|--------|---------| +| Container Start | [OK/FAIL] | [details] | +| Image Pull | [OK/FAIL] | [details] | +| Resource Scheduling | [OK/FAIL] | [details] | +| Application Health | [OK/FAIL] | [details] | +| Volume Mounts | [OK/FAIL] | [details] | + +### Detailed Findings + +**[Category 1: e.g., Image Pull Issues]** +- Problem: [specific problem] +- Evidence: [from events/logs] +- Impact: [how this affects the pod] + +**[Category 2: e.g., Application Crash]** +- Problem: [specific problem] +- Evidence: [from logs] +- Impact: [how this affects the pod] + +### Recommended Actions + +1. **[Action 1]** - [description] + ```bash + [command to fix - e.g., oc create secret docker-registry...] + ``` + +2. **[Action 2]** - [description] + ```bash + [command to fix - e.g., oc set resources deployment/app --limits=memory=1Gi] + ``` + +3. **[Action 3]** - [description] + +### Related Documentation + +- [Link to relevant Red Hat KB article if applicable] +- [Link to OpenShift docs for the specific issue] + +--- + +Would you like me to: +1. Execute one of the recommended fixes +2. Dig deeper into a specific area +3. Debug a related resource (Service, Route, ConfigMap) +4. Exit debugging + +Select an option: +``` + +Proceed automatically without waiting for user input. + +For pod failure categories and exit code reference, see [debugging-patterns.md](../../docs/debugging-patterns.md). + +## Dependencies + +### Required MCP Servers +- `openshift` - Kubernetes/OpenShift resource access for pod status, events, and logs + +### Related Skills +- `/debug-build` - If pod failure is due to bad image from build +- `/debug-network` - If pod is running but service connectivity fails +- `/deploy` - To redeploy after fixing issues + +### Reference Documentation +- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common error patterns and troubleshooting trees +- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (oc), cluster access verification diff --git a/evaluation/with_skills/rh-developer__debug-pod/instruction.md b/evaluation/with_skills/rh-developer__debug-pod/instruction.md new file mode 100644 index 00000000..9a983f81 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pod/instruction.md @@ -0,0 +1,14 @@ +# Pod Debugging Task + +You are a Red Hat developer. A pod in the `web-frontend` namespace keeps crashing and restarting. Your team needs you to investigate, identify the root cause, and recommend a fix. + +## Requirements +- Check the pod status and identify the failure pattern (exit code, restart count, state) +- Examine container logs, including logs from previous crashed containers +- Analyze resource limits and requests to determine if the crash is resource-related +- Review namespace events for warnings or errors related to the pod +- Identify the root cause and recommend a specific fix + +Use MCP tools to examine the cluster. Document your methodology, findings, and recommended remediation in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-developer__debug-pod/solution/solve.sh b/evaluation/with_skills/rh-developer__debug-pod/solution/solve.sh new file mode 100644 index 00000000..dca1ff71 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pod/solution/solve.sh @@ -0,0 +1,39 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Pod Debug Report + +## Investigation Summary +A pod in the web-frontend namespace is crashing repeatedly. + +## Pod Status +- Namespace: web-frontend +- Pod: web-frontend (CrashLoopBackOff) +- Exit code: 137 (OOMKilled — SIGKILL, memory limit exceeded) +- Restart count: 8 + +## Diagnosis Methodology +1. Listed pods in web-frontend namespace — found pod in CrashLoopBackOff +2. Examined container status — exit code 137, reason: OOMKilled +3. Checked previous container logs — server starts but gets Killed +4. Reviewed events — OOMKilled warning with memory limit 64Mi +5. Analyzed resource limits — memory limit 64Mi is too low for Node.js + +## Root Cause +Exit 137 = 128 + 9 (SIGKILL). The container was OOMKilled because the memory limit of 64Mi is insufficient for a Node.js application. The application starts normally but is killed when memory usage exceeds the limit during initialization of middleware. + +## Events Analysis +- Warning: OOMKilled — Container exceeded memory limit of 64Mi +- Warning: BackOff — Back-off restarting failed container + +## Recommended Fix +Increase the memory limit for the web-frontend deployment: +- Current: requests=32Mi, limits=64Mi +- Recommended: requests=128Mi, limits=256Mi (or higher depending on app needs) + +This can be applied by patching the deployment resource limits. + +## Additional Notes +- The application logs show it starts successfully but is killed during middleware initialization +- No memory leak — the base memory requirement simply exceeds the configured limit +- Consider monitoring memory usage after the fix to right-size the limits +REPORT_EOF diff --git a/evaluation/with_skills/rh-developer__debug-pod/task.toml b/evaluation/with_skills/rh-developer__debug-pod/task.toml new file mode 100644 index 00000000..89bac572 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pod/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__debug-pod" +name = "rh-developer Pod Debugging Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "debug-pod", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-developer__debug-pod/tests/llm_judge.py b/evaluation/with_skills/rh-developer__debug-pod/tests/llm_judge.py new file mode 100644 index 00000000..3bad1517 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pod/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "previous_logs_methodology", + "file": "/root/report.md", + "question": "Does the report use --previous flag to retrieve logs from crashed containers when restarts are detected?", + "reference": "A skilled report uses --previous to get logs from the terminated container instance when restart count > 0. An unskilled report only checks current container logs, missing crash context." + }, + { + "id": "readiness_endpoint_link", + "file": "/root/report.md", + "question": "Does the report explain that readiness probe failures remove the pod from Service endpoints, causing traffic loss?", + "reference": "A skilled report explains the readiness→endpoints relationship: failed readiness probes remove the pod from Service endpoints. An unskilled report treats readiness as only affecting pod status." + }, + { + "id": "oom_diagnosis_and_fix", + "file": "/root/report.md", + "question": "Does the report map exit code 137 to OOMKilled and provide concrete oc set resources or oc patch commands to increase memory limits?", + "reference": "A skilled report maps 137→OOM and provides actionable oc commands to fix resource limits. An unskilled report may identify OOM but gives vague recommendations." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-developer__debug-pod/tests/test.sh b/evaluation/with_skills/rh-developer__debug-pod/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pod/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-developer__debug-pod/tests/test_outputs.py b/evaluation/with_skills/rh-developer__debug-pod/tests/test_outputs.py new file mode 100644 index 00000000..fda1b3ed --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-pod/tests/test_outputs.py @@ -0,0 +1,75 @@ +""" +Tests for rh-developer__debug-pod per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_pod_or_container(self): + content = read_report().lower() + assert "pod" in content or "container" in content, "report should mention pod or container" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 200, "report should have substantial content" + + +class TestSkillDependent: + def test_previous_logs_flag(self): + """Skill teaches using --previous to get logs from crashed container + when restarts > 0. Without skill, agents only check current logs.""" + c = read_report() + assert "--previous" in c or "previous" in c.lower(), ( + "should use --previous flag to get logs from crashed container" + ) + + def test_readiness_removes_endpoints(self): + """Skill teaches that readiness probe failures remove pod from Service + endpoints, causing traffic loss. Without skill, agents miss this link.""" + c = read_report().lower() + assert ("readiness" in c and "endpoint" in c) or ("readiness" in c and "service" in c) or ( + "readiness" in c and "traffic" in c + ), "should explain readiness failures remove Service endpoints" + + def test_exit_137_oomkilled_mapping(self): + """Skill teaches exit code 137 = OOMKilled, map to memory limit.""" + c = read_report().lower() + assert ("137" in c or "oom" in c or "oomkill" in c) and any(t in c for t in [ + "memory", "limit", "increase" + ]), "should map exit 137 to OOMKilled and memory limit" + + def test_concrete_remediation_command(self): + """Skill teaches oc set resources deployment/... --limits=memory=.""" + c = read_report().lower() + assert any(t in c for t in ["oc set resources", "oc patch", "memory=", "limits"]) or ( + "```" in read_report() and "oc" in c + ), "should include concrete oc remediation command" + + def test_resource_analysis(self): + """Skill teaches analyzing memory request/limit for OOM remediation.""" + c = read_report().lower() + assert any(t in c for t in ["limit", "request"]) and any(t in c for t in [ + "memory", "resource", "increase" + ]), "should analyze resource limits for OOM" + + def test_events_correlation(self): + """Skill teaches checking events for scheduling, OOM, and image pull failures.""" + c = read_report().lower() + assert "event" in c and any(t in c for t in [ + "oom", "schedule", "pull", "fail", "kill", "backoff" + ]), "should correlate pod events with failure cause" diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/Dockerfile b/evaluation/with_skills/rh-developer__debug-rhel/environment/Dockerfile new file mode 100644 index 00000000..d70159c5 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-rhel/environment/Dockerfile @@ -0,0 +1,74 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "rhel-system": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-rhel-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-developer__debug-rhel/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-rhel/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/mcp-servers/mock-rhel-mcp.py b/evaluation/with_skills/rh-developer__debug-rhel/environment/mcp-servers/mock-rhel-mcp.py new file mode 100644 index 00000000..314f0e3b --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-rhel/environment/mcp-servers/mock-rhel-mcp.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +"""Mock RHEL System MCP Server for RHEL debugging evaluation. + +Simulates a RHEL 9 host with a failing service. Exposes system-level +diagnostic tools (systemctl, journalctl, getenforce, firewall-cmd, ausearch) +as MCP tools so the agent can diagnose the issue. + +Scenario: + Host: app-server-01.example.com (RHEL 9.3) + Failing service: myapp.service + Root causes: + 1. SELinux denial: httpd_t cannot bind to port 9090 + 2. Firewall: port 9090/tcp is not open + 3. Service configuration references correct binary but SELinux blocks it +""" + +import json +from typing import Optional + +from fastmcp import FastMCP + +mcp = FastMCP("rhel-system") + +HOST = "app-server-01.example.com" +RHEL_VER = "9.3" + +SERVICES = { + "myapp.service": { + "loaded": True, + "enabled": True, + "active": "failed", + "sub": "failed", + "description": "My Application Service", + "main_pid": 0, + "exit_code": "exited", + "exit_status": 1, + "exec_start": "/opt/myapp/bin/myapp-server --port 9090 --config /etc/myapp/config.yaml", + "user": "myapp", + "group": "myapp", + "working_directory": "/opt/myapp", + "environment": "APP_ENV=production DB_HOST=localhost DB_PORT=5432", + "restart": "on-failure", + "restart_sec": 5, + "status_output": ( + "● myapp.service - My Application Service\n" + " Loaded: loaded (/etc/systemd/system/myapp.service; enabled; preset: disabled)\n" + " Active: failed (Result: exit-code) since Sun 2026-03-01 18:30:45 UTC; 17h ago\n" + " Process: 45678 ExecStart=/opt/myapp/bin/myapp-server --port 9090 --config /etc/myapp/config.yaml (code=exited, status=1/FAILURE)\n" + " Main PID: 45678 (code=exited, status=1/FAILURE)\n" + " CPU: 125ms\n" + "\n" + "Mar 01 18:30:44 app-server-01 systemd[1]: Starting My Application Service...\n" + "Mar 01 18:30:44 app-server-01 myapp-server[45678]: Starting myapp-server v2.1.0\n" + "Mar 01 18:30:44 app-server-01 myapp-server[45678]: Loading configuration from /etc/myapp/config.yaml\n" + "Mar 01 18:30:45 app-server-01 myapp-server[45678]: Configuration loaded successfully\n" + "Mar 01 18:30:45 app-server-01 myapp-server[45678]: Attempting to bind to 0.0.0.0:9090\n" + "Mar 01 18:30:45 app-server-01 myapp-server[45678]: Error: Permission denied: bind to 0.0.0.0:9090\n" + "Mar 01 18:30:45 app-server-01 myapp-server[45678]: Fatal: Cannot start server, exiting\n" + "Mar 01 18:30:45 app-server-01 systemd[1]: myapp.service: Main process exited, code=exited, status=1/FAILURE\n" + "Mar 01 18:30:45 app-server-01 systemd[1]: myapp.service: Failed with result 'exit-code'.\n" + ), + }, + "sshd.service": { + "loaded": True, + "enabled": True, + "active": "active", + "sub": "running", + "description": "OpenSSH server daemon", + "main_pid": 1234, + "exit_code": "", + "exit_status": 0, + }, + "firewalld.service": { + "loaded": True, + "enabled": True, + "active": "active", + "sub": "running", + "description": "firewalld - dynamic firewall daemon", + "main_pid": 2345, + "exit_code": "", + "exit_status": 0, + }, + "postgresql.service": { + "loaded": True, + "enabled": True, + "active": "active", + "sub": "running", + "description": "PostgreSQL database server", + "main_pid": 3456, + "exit_code": "", + "exit_status": 0, + }, +} + +JOURNAL_LOGS = { + "myapp.service": ( + "-- Journal begins at Sat 2026-02-28 00:00:00 UTC, ends at Sun 2026-03-02 12:00:00 UTC. --\n" + "Mar 01 18:30:44 app-server-01 systemd[1]: Starting My Application Service...\n" + "Mar 01 18:30:44 app-server-01 myapp-server[45678]: Starting myapp-server v2.1.0\n" + "Mar 01 18:30:44 app-server-01 myapp-server[45678]: Loading configuration from /etc/myapp/config.yaml\n" + "Mar 01 18:30:45 app-server-01 myapp-server[45678]: Configuration loaded successfully\n" + "Mar 01 18:30:45 app-server-01 myapp-server[45678]: Connecting to database at localhost:5432... OK\n" + "Mar 01 18:30:45 app-server-01 myapp-server[45678]: Attempting to bind to 0.0.0.0:9090\n" + "Mar 01 18:30:45 app-server-01 myapp-server[45678]: Error: Permission denied: bind to 0.0.0.0:9090\n" + "Mar 01 18:30:45 app-server-01 myapp-server[45678]: Fatal: Cannot start server, exiting\n" + "Mar 01 18:30:45 app-server-01 systemd[1]: myapp.service: Main process exited, code=exited, status=1/FAILURE\n" + "Mar 01 18:30:45 app-server-01 systemd[1]: myapp.service: Failed with result 'exit-code'.\n" + "Mar 01 18:30:50 app-server-01 systemd[1]: myapp.service: Scheduled restart job, restart counter is at 1.\n" + "Mar 01 18:30:50 app-server-01 systemd[1]: Starting My Application Service...\n" + "Mar 01 18:30:50 app-server-01 myapp-server[45690]: Starting myapp-server v2.1.0\n" + "Mar 01 18:30:51 app-server-01 myapp-server[45690]: Loading configuration from /etc/myapp/config.yaml\n" + "Mar 01 18:30:51 app-server-01 myapp-server[45690]: Configuration loaded successfully\n" + "Mar 01 18:30:51 app-server-01 myapp-server[45690]: Connecting to database at localhost:5432... OK\n" + "Mar 01 18:30:51 app-server-01 myapp-server[45690]: Attempting to bind to 0.0.0.0:9090\n" + "Mar 01 18:30:51 app-server-01 myapp-server[45690]: Error: Permission denied: bind to 0.0.0.0:9090\n" + "Mar 01 18:30:51 app-server-01 myapp-server[45690]: Fatal: Cannot start server, exiting\n" + "Mar 01 18:30:51 app-server-01 systemd[1]: myapp.service: Main process exited, code=exited, status=1/FAILURE\n" + "Mar 01 18:30:51 app-server-01 systemd[1]: myapp.service: Failed with result 'exit-code'.\n" + "Mar 01 18:30:56 app-server-01 systemd[1]: myapp.service: Scheduled restart job, restart counter is at 2.\n" + "Mar 01 18:30:56 app-server-01 systemd[1]: Starting My Application Service...\n" + "Mar 01 18:30:56 app-server-01 myapp-server[45705]: Starting myapp-server v2.1.0\n" + "Mar 01 18:30:57 app-server-01 myapp-server[45705]: Error: Permission denied: bind to 0.0.0.0:9090\n" + "Mar 01 18:30:57 app-server-01 myapp-server[45705]: Fatal: Cannot start server, exiting\n" + "Mar 01 18:30:57 app-server-01 systemd[1]: myapp.service: Main process exited, code=exited, status=1/FAILURE\n" + "Mar 01 18:30:57 app-server-01 systemd[1]: myapp.service: Failed with result 'exit-code'.\n" + "Mar 01 18:30:57 app-server-01 systemd[1]: myapp.service: Start request repeated too quickly.\n" + "Mar 01 18:30:57 app-server-01 systemd[1]: myapp.service: Failed with result 'exit-code'.\n" + ), +} + + +@mcp.tool() +def systemctl_status(service: str) -> str: + """Get the status of a systemd service (equivalent to 'systemctl status ').""" + svc = SERVICES.get(service) + if not svc: + return f"Unit {service} could not be found." + + if svc.get("status_output"): + return svc["status_output"] + + state = "active (running)" if svc["active"] == "active" else "failed" + return ( + f"● {service} - {svc['description']}\n" + f" Loaded: loaded (/usr/lib/systemd/system/{service}; " + f"{'enabled' if svc['enabled'] else 'disabled'}; preset: disabled)\n" + f" Active: {state}\n" + f" Main PID: {svc['main_pid']}\n" + ) + + +@mcp.tool() +def systemctl_list_failed() -> str: + """List all failed systemd services (equivalent to 'systemctl --failed').""" + failed = [(name, svc) for name, svc in SERVICES.items() if svc["active"] == "failed"] + if not failed: + return "0 loaded units listed." + + lines = [" UNIT LOAD ACTIVE SUB DESCRIPTION"] + for name, svc in failed: + lines.append( + f" {name:<24s} loaded failed failed {svc['description']}" + ) + lines.append(f"\n{len(failed)} loaded units listed.") + return "\n".join(lines) + + +@mcp.tool() +def journalctl(unit: Optional[str] = None, lines: int = 100, priority: Optional[str] = None) -> str: + """Get journal logs, optionally filtered by unit or priority.""" + if unit and unit in JOURNAL_LOGS: + log = JOURNAL_LOGS[unit] + if priority and priority in ("err", "3"): + return "\n".join( + line for line in log.split("\n") + if "Error" in line or "Fatal" in line or "FAILURE" in line or "failed" in line.lower() + ) + return log + + if unit: + return f"-- No entries for unit {unit} --" + + return ( + "-- Journal begins at Sat 2026-02-28 00:00:00 UTC --\n" + "Mar 02 12:00:00 app-server-01 kernel: Linux version 5.14.0-362.el9.x86_64\n" + "Mar 02 12:00:00 app-server-01 systemd[1]: Started system.\n" + ) + + +@mcp.tool() +def getenforce() -> str: + """Get SELinux enforcement mode (equivalent to 'getenforce').""" + return "Enforcing" + + +@mcp.tool() +def ausearch_avc(recent: bool = True, comm: Optional[str] = None) -> str: + """Search for SELinux AVC denial messages (equivalent to 'ausearch -m AVC').""" + denials = [ + { + "timestamp": "Mar 01 18:30:45", + "type": "AVC", + "result": "denied", + "permission": "name_bind", + "scontext": "system_u:system_r:httpd_t:s0", + "tcontext": "system_u:object_r:unreserved_port_t:s0", + "tclass": "tcp_socket", + "comm": "myapp-server", + "port": 9090, + }, + { + "timestamp": "Mar 01 18:30:50", + "type": "AVC", + "result": "denied", + "permission": "name_bind", + "scontext": "system_u:system_r:httpd_t:s0", + "tcontext": "system_u:object_r:unreserved_port_t:s0", + "tclass": "tcp_socket", + "comm": "myapp-server", + "port": 9090, + }, + { + "timestamp": "Mar 01 18:30:56", + "type": "AVC", + "result": "denied", + "permission": "name_bind", + "scontext": "system_u:system_r:httpd_t:s0", + "tcontext": "system_u:object_r:unreserved_port_t:s0", + "tclass": "tcp_socket", + "comm": "myapp-server", + "port": 9090, + }, + ] + + if comm: + denials = [d for d in denials if d["comm"] == comm] + + if not denials: + return "No AVC denials found." + + lines = [] + for d in denials: + lines.append( + f"----\n" + f"time->{d['timestamp']}\n" + f"type=AVC msg=audit: avc: denied {{ {d['permission']} }} for " + f"comm=\"{d['comm']}\" " + f"src={d['port']} " + f"scontext={d['scontext']} " + f"tcontext={d['tcontext']} " + f"tclass={d['tclass']} permissive=0" + ) + return "\n".join(lines) + + +@mcp.tool() +def firewall_cmd_state() -> str: + """Check if firewalld is running (equivalent to 'firewall-cmd --state').""" + return "running" + + +@mcp.tool() +def firewall_cmd_list_all() -> str: + """List all firewall rules for the default zone (equivalent to 'firewall-cmd --list-all').""" + return ( + "public (active)\n" + " target: default\n" + " icmp-block-inversion: no\n" + " interfaces: eth0\n" + " sources: \n" + " services: cockpit dhcpv6-client ssh\n" + " ports: 5432/tcp\n" + " protocols: \n" + " forward: yes\n" + " masquerade: no\n" + " forward-ports: \n" + " source-ports: \n" + " icmp-blocks: \n" + " rich rules: \n" + ) + + +@mcp.tool() +def firewall_cmd_query_port(port: str) -> str: + """Check if a specific port is open in the firewall (e.g. '9090/tcp').""" + open_ports = {"5432/tcp", "22/tcp"} + if port in open_ports: + return "yes" + return "no" + + +@mcp.tool() +def semanage_port_list(port_type: Optional[str] = None) -> str: + """List SELinux port type assignments (equivalent to 'semanage port -l').""" + entries = [ + ("http_port_t", "tcp", "80, 81, 443, 488, 8008, 8009, 8443, 9000"), + ("ssh_port_t", "tcp", "22"), + ("postgresql_port_t", "tcp", "5432"), + ("unreserved_port_t", "tcp", "1024-32767"), + ("unreserved_port_t", "udp", "1024-32767"), + ] + if port_type: + entries = [(t, p, ports) for t, p, ports in entries if t == port_type] + + lines = ["SELinux Port Type Proto Port Number"] + for t, p, ports in entries: + lines.append(f"{t:<26s} {p:<8s} {ports}") + return "\n".join(lines) + + +@mcp.tool() +def system_info() -> str: + """Get basic system information (hostname, OS, kernel, uptime).""" + return json.dumps({ + "hostname": HOST, + "os": f"Red Hat Enterprise Linux {RHEL_VER}", + "kernel": "5.14.0-362.el9.x86_64", + "arch": "x86_64", + "uptime": "15 days, 3:42", + "load_average": "0.45, 0.38, 0.32", + "memory": { + "total": "16384 MB", + "used": "5120 MB", + "free": "8192 MB", + "available": "11264 MB", + }, + "disk": { + "/": {"total": "50G", "used": "18G", "available": "32G", "use_percent": "36%"}, + "/var": {"total": "100G", "used": "45G", "available": "55G", "use_percent": "45%"}, + }, + }, indent=2) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/skills/debug-rhel/SKILL.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/skills/debug-rhel/SKILL.md new file mode 100644 index 00000000..4c3601ad --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-rhel/environment/skills/debug-rhel/SKILL.md @@ -0,0 +1,455 @@ +--- +name: debug-rhel +description: | + Diagnose RHEL system issues including systemd service failures, SELinux denials, firewall blocking, and system resource problems. Automates multi-step diagnosis: journalctl log analysis, SELinux denial detection (ausearch), firewall rule inspection, and systemd unit status. Use this skill when applications fail on standalone RHEL/Fedora/CentOS hosts deployed via /rhel-deploy. Triggers on /debug-rhel command or phrases like "service won't start on RHEL", "SELinux blocking", "systemd failed", "firewall blocking". +model: inherit +color: cyan +metadata: + user_invocable: "true" +--- + +# /debug-rhel Skill + +Diagnose RHEL system issues by automatically gathering systemd status, journal logs, SELinux denials, and firewall configuration. + +## Overview + +``` +[Connect] → [Identify Service] → [systemd Status] → [Journal Logs] → [SELinux] → [Firewall] → [Summary] +``` + +**This skill diagnoses:** +- systemd service failures +- SELinux access denials (AVC) +- Firewall port blocking +- Permission issues +- Resource constraints + +## Prerequisites + +1. SSH access to target RHEL host +2. sudo privileges on the target host +3. RHEL 8+, CentOS Stream, Rocky Linux, or Fedora + +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +## Note: SSH/Bash Required + +This skill operates on **remote RHEL hosts** via SSH, not local MCP servers. Unlike OpenShift/Podman skills, direct Bash commands with SSH are the correct approach here since MCP servers run locally and cannot access remote systems. + +## When to Use This Skill + +Use `/debug-rhel` when applications fail on standalone RHEL, Fedora, or CentOS hosts. This skill automates multi-step diagnosis of systemd service failures, SELinux denials, firewall blocking, and system resource problems via SSH. + +## Workflow + +### Phase 1: SSH Connection + +```markdown +## RHEL System Debugging + +I'll help you diagnose issues on your RHEL system. + +**SSH Target:** +[If RHEL_HOST in session state from /rhel-deploy:] +- Using previous connection: [user]@[host] + +Is this correct? (yes/no/different host) + +[If no RHEL_HOST:] +Please provide your RHEL host details: + +| Setting | Value | Default | +|---------|-------|---------| +| Host | [required] | - | +| User | [current user] | $USER | +| Port | 22 | 22 | + +**Enter your SSH target:** +``` + +Proceed automatically without waiting for user input. + +**Connection verification:** + +```bash +# Test SSH connection +ssh -o BatchMode=yes -o ConnectTimeout=10 [user]@[host] "echo 'Connection successful'" +``` + +If connection fails: +```markdown +**SSH Connection Failed** + +Unable to connect to [host]. + +**Troubleshooting:** +1. Check host is reachable: `ping [host]` +2. Verify SSH key is configured: `ssh-add -l` +3. Check firewall allows SSH: port 22 +4. Verify username is correct + +Would you like to: +1. Try a different host +2. Get help with SSH setup +3. Exit +``` + +### Phase 2: Identify Target Service + +```markdown +## Phase 2: Identify Service + +Which service would you like me to debug? + +1. **Specify service name** - Enter the systemd unit name +2. **List failed services** - Show failed services on the host +3. **From /rhel-deploy** - Debug the last deployed service + +Select an option or enter a service name: +``` + +Proceed automatically without waiting for user input. + +If user selects "List failed services": + +```bash +# Get failed services +ssh [user]@[host] "systemctl --failed --no-pager" +``` + +```markdown +## Failed Services on [host] + +| Unit | Load | Active | Sub | Description | +|------|------|--------|-----|-------------| +| [myapp.service] | loaded | failed | failed | My Application | +| [other.service] | loaded | failed | failed | Other Service | + +Which service would you like me to debug? +``` + +Proceed automatically without waiting for user input. + +### Phase 3: Get Service Status + +```bash +# Get detailed service status +ssh [user]@[host] "systemctl status [service] --no-pager -l" +``` + +```markdown +## Service Status: [service-name] + +**Status Overview:** +| Field | Value | +|-------|-------| +| Loaded | [loaded/not-found/masked] | +| Active | [active (running)/inactive (dead)/failed] | +| Main PID | [pid or N/A] | +| Status | [status text] | +| Since | [timestamp] | + +**Recent Activity:** +``` +[systemctl status output - last 10 lines] +``` + +**Quick Assessment:** +[Based on status, provide initial assessment - e.g., "Service failed to start - exit code 1 suggests application error"] + +Continue with journal logs? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Phase 4: Analyze Journal Logs + +```bash +# Get service logs +ssh [user]@[host] "journalctl -u [service] -n 100 --no-pager" +``` + +```markdown +## Journal Logs: [service-name] + +**Last 100 log entries:** +``` +[journalctl output] +``` + +**Log Analysis:** + +[Analyze logs and identify errors:] + +**Errors Found:** +- [timestamp]: [error - e.g., "Permission denied: /var/data/config.yaml"] +- [timestamp]: [error - e.g., "Connection refused: localhost:5432"] +- [timestamp]: [error - e.g., "Port 8080 already in use"] + +**Error Categories:** +| Category | Count | Example | +|----------|-------|---------| +| Permission | [X] | [first occurrence] | +| Connection | [Y] | [first occurrence] | +| Resource | [Z] | [first occurrence] | + +Continue to check SELinux? (yes/no/skip) +``` + +Proceed automatically without waiting for user input. + +### Phase 5: Check SELinux Denials + +```bash +# Check SELinux status +ssh [user]@[host] "getenforce" + +# Get recent AVC denials +ssh [user]@[host] "sudo ausearch -m AVC -ts recent 2>/dev/null || echo 'No recent denials or ausearch not available'" +``` + +```markdown +## SELinux Analysis + +**SELinux Status:** [Enforcing/Permissive/Disabled] + +**Recent AVC Denials:** + +[If denials found:] +| Time | Source | Target | Permission | Denied | +|------|--------|--------|------------|--------| +| [time] | [source_context] | [target_context] | [permission] | [target_file] | +| [time] | [source_context] | [target_context] | [permission] | [target_port] | + +**Denial Analysis:** + +**Denial 1: [description]** +- **What happened:** Process `[name]` tried to [action] on `[target]` +- **Why denied:** SELinux type `[source_type]` cannot [action] `[target_type]` +- **Impact:** [how this affects the application] + +**Recommended Fixes:** + +1. **Set SELinux boolean** (if applicable): + ```bash + sudo setsebool -P [boolean_name] on + ``` + +2. **Change file context** (if file access): + ```bash + sudo semanage fcontext -a -t [correct_type] "[path](/.*)?" + sudo restorecon -Rv [path] + ``` + +3. **Allow port** (if port binding): + ```bash + sudo semanage port -a -t [port_type] -p tcp [port] + ``` + +[If no denials:] +No recent SELinux denials found. SELinux is likely not the issue. + +Continue to check firewall? (yes/no/skip) +``` + +Proceed automatically without waiting for user input. + +### Phase 6: Check Firewall + +```bash +# Get firewall status +ssh [user]@[host] "sudo firewall-cmd --state 2>/dev/null || echo 'firewalld not running'" + +# List firewall rules +ssh [user]@[host] "sudo firewall-cmd --list-all 2>/dev/null" +``` + +```markdown +## Firewall Analysis + +**Firewall Status:** [running/not running] + +**Active Zone:** [zone-name] + +**Current Rules:** +| Type | Value | +|------|-------| +| Services | [ssh, http, https, ...] | +| Ports | [8080/tcp, 3000/tcp, ...] | +| Rich Rules | [count] | + +**Application Port:** [detected-port from logs/config] + +**Port Status:** +| Port | Protocol | Status | +|------|----------|--------| +| [8080] | TCP | [OPEN/BLOCKED] | +| [443] | TCP | [OPEN/BLOCKED] | + +[If port blocked:] +**WARNING: Application port [port] is NOT open in firewall!** + +**To open port:** +```bash +sudo firewall-cmd --permanent --add-port=[port]/tcp +sudo firewall-cmd --reload +``` + +**Or add service:** +```bash +sudo firewall-cmd --permanent --add-service=[service] +sudo firewall-cmd --reload +``` + +Continue to diagnosis summary? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Phase 7: Red Hat Insights Check (Optional) + +**This phase runs only if the `lightspeed-mcp` server is available.** Use `ToolSearch` to check for Lightspeed MCP tools. If not available, skip this phase silently and proceed to Phase 8. + +**Step 1:** Use `find_host_by_name` with the hostname from `RHEL_HOST` to look up the system in Red Hat Insights. + +**Step 2:** If system found, use `get_system_cves` with the system ID to check for known CVEs affecting this system. + +**Step 3:** Use `get_active_rules` to get advisor configuration recommendations. Optionally use `get_rule_by_text_search` with error text found in Phase 4 logs to find relevant advisor recommendations. + +```markdown +## Red Hat Insights Check + +**System in Insights:** [Found / Not registered] + +[If found:] +**System Details:** +| Field | Value | +|-------|-------| +| Display Name | [hostname] | +| RHEL Version | [version] | +| Last Check-in | [timestamp] | +| Stale | [yes/no] | + +**Known Vulnerabilities:** +| CVE | CVSS | Severity | Remediation | +|-----|------|----------|-------------| +| [CVE-ID] | [score] | [severity] | [Available/None] | + +**Advisor Recommendations:** +| Rule | Category | Risk | Description | +|------|----------|------|-------------| +| [rule-id] | [Security/Performance/Availability/Stability] | [Critical/Important/Moderate/Low] | [description] | + +[If any CVE or advisor rule matches the symptoms from earlier phases:] +**Potentially Related to Current Issue:** +- [CVE or advisor rule that matches the symptoms] + +Continue to diagnosis summary? (yes/no) +``` + +Proceed automatically without waiting for user input. + +[If system not registered in Insights, just note it:] +```markdown +## Red Hat Insights Check + +System [hostname] is not registered in Red Hat Insights. Skipping vulnerability and advisor checks. + +Continue to diagnosis summary? (yes/no) +``` + +### Phase 8: Present Diagnosis Summary + +```markdown +## Diagnosis Summary: [service-name] on [host] + +### Root Cause + +**Primary Issue:** [Categorized root cause] + +| Category | Status | Details | +|----------|--------|---------| +| Service Unit | [OK/FAIL] | [loaded/enabled status] | +| Application | [OK/FAIL] | [exit code, error] | +| SELinux | [OK/BLOCKED] | [denial count] | +| Firewall | [OK/BLOCKED] | [port status] | +| Permissions | [OK/FAIL] | [file/dir issues] | +| Resources | [OK/FAIL] | [memory/cpu/disk] | +| Insights/CVE | [OK/WARN/N/A] | [CVE count or "Not registered"] | + +### Detailed Findings + +**[Category 1: e.g., SELinux Denial]** +- Problem: [specific problem - e.g., "httpd_t cannot bind to port 8080"] +- Evidence: [AVC denial message] +- Impact: [application cannot start] + +**[Category 2: e.g., Missing Dependency]** +- Problem: [specific problem - e.g., "libpq.so.5 not found"] +- Evidence: [error from logs] +- Impact: [application crashes on startup] + +### Recommended Actions + +1. **[Action 1 - Highest Priority]** - [description] + ```bash + ssh [user]@[host] "[command]" + ``` + +2. **[Action 2]** - [description] + ```bash + ssh [user]@[host] "[command]" + ``` + +3. **[Action 3]** - [description] + ```bash + ssh [user]@[host] "[command]" + ``` + +### Verify Fix + +After applying fixes: +```bash +# Restart service +ssh [user]@[host] "sudo systemctl restart [service]" + +# Check status +ssh [user]@[host] "systemctl status [service]" + +# View logs +ssh [user]@[host] "journalctl -u [service] -f" +``` + +--- + +Would you like me to: +1. Execute one of the recommended fixes +2. Dig deeper into a specific area +3. Restart the service +4. View live logs +5. Exit debugging + +Select an option: +``` + +Proceed automatically without waiting for user input. + +For common RHEL issues (systemd exit codes, SELinux denials, firewall), see [debugging-patterns.md](../../docs/debugging-patterns.md) and [selinux-troubleshooting.md](../../docs/selinux-troubleshooting.md). + +## Dependencies + +### Required MCP Servers +- `lightspeed-mcp` (optional) - Red Hat Insights CVE and advisor checks in Phase 7 + +### Related Skills +- `/rhel-deploy` - redeploy after fixing issues +- `/debug-container` - debug Podman containers on the host + +### Reference Documentation +- [docs/selinux-troubleshooting.md](../../docs/selinux-troubleshooting.md) - SELinux denial analysis +- [docs/rhel-deployment.md](../../docs/rhel-deployment.md) - RHEL deployment patterns +- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common error patterns +- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools and setup diff --git a/evaluation/with_skills/rh-developer__debug-rhel/instruction.md b/evaluation/with_skills/rh-developer__debug-rhel/instruction.md new file mode 100644 index 00000000..ca2ade3a --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-rhel/instruction.md @@ -0,0 +1,12 @@ +# RHEL System Debugging Task + +You are a Red Hat developer. A RHEL-based service is failing to start or accept connections. Investigate the system configuration to identify the issue. + +## Requirements +- Check service status, SELinux, and firewall configuration +- Identify the system-level root cause +- Recommend a fix + +Use available tools to examine the environment. Document your methodology, findings, and recommendations in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-developer__debug-rhel/solution/solve.sh b/evaluation/with_skills/rh-developer__debug-rhel/solution/solve.sh new file mode 100644 index 00000000..350dd5d5 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-rhel/solution/solve.sh @@ -0,0 +1,36 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# RHEL Debug Report + +## Issue: Flask app can't bind to port 8080 + +### Systemd Check +```bash +systemctl status flask-app +``` + +### Journal Logs +```bash +journalctl -u flask-app -n 100 +``` + +### SELinux Check +```bash +getenforce +ausearch -m AVC -ts recent +# Found: denied bind to port 8080 +``` + +### Fix: Add port to SELinux +```bash +sudo semanage port -a -t http_port_t -p tcp 8080 +sudo restorecon -Rv /opt/flask-app +``` + +### Firewall Check +```bash +sudo firewall-cmd --list-all +sudo firewall-cmd --permanent --add-port=8080/tcp +sudo firewall-cmd --reload +``` +REPORT_EOF diff --git a/evaluation/with_skills/rh-developer__debug-rhel/task.toml b/evaluation/with_skills/rh-developer__debug-rhel/task.toml new file mode 100644 index 00000000..32fb504b --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-rhel/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__debug-rhel" +name = "rh-developer RHEL Deployment Debugging Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "debug-rhel", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-developer__debug-rhel/tests/llm_judge.py b/evaluation/with_skills/rh-developer__debug-rhel/tests/llm_judge.py new file mode 100644 index 00000000..e170f4bb --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-rhel/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "ausearch_avc_workflow", + "file": "/root/report.md", + "question": "Does the report use ausearch -m AVC for investigating SELinux denials, rather than generic SELinux commands?", + "reference": "A skilled report uses 'ausearch -m AVC -ts recent' to find recent SELinux AVC denials. An unskilled report checks getenforce or sestatus without examining specific denials." + }, + { + "id": "semanage_port_labeling", + "file": "/root/report.md", + "question": "Does the report use semanage port for labeling nonstandard bind ports in SELinux?", + "reference": "A skilled report uses 'semanage port -a -t http_port_t -p tcp ' for nonstandard ports. An unskilled report suggests disabling SELinux or only uses setsebool." + }, + { + "id": "concrete_rhel_remediation", + "file": "/root/report.md", + "question": "Does the report provide concrete systemctl, firewall-cmd, and semanage/restorecon commands for RHEL troubleshooting?", + "reference": "A skilled report provides specific commands for each layer: systemctl restart for services, firewall-cmd --add-port for networking, semanage+restorecon for SELinux. An unskilled report gives high-level suggestions." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-developer__debug-rhel/tests/test.sh b/evaluation/with_skills/rh-developer__debug-rhel/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-rhel/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-developer__debug-rhel/tests/test_outputs.py b/evaluation/with_skills/rh-developer__debug-rhel/tests/test_outputs.py new file mode 100644 index 00000000..6ba9216b --- /dev/null +++ b/evaluation/with_skills/rh-developer__debug-rhel/tests/test_outputs.py @@ -0,0 +1,97 @@ +""" +Tests for rh-developer__debug-rhel per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_rhel_or_system(self): + content = read_report().lower() + assert "rhel" in content or "system" in content or "service" in content, ( + "report should mention RHEL or system" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_ausearch_avc_command(self): + """Skill teaches ausearch -m AVC -ts recent for recent SELinux denials. + Without skill, agents use generic SELinux checks without ausearch.""" + c = read_report().lower() + assert "ausearch" in c, ( + "should use ausearch for SELinux AVC denial investigation" + ) + + def test_semanage_port_labeling(self): + """Skill teaches semanage port for nonstandard bind port SELinux labeling. + Without skill, agents skip port-level SELinux context management.""" + c = read_report().lower() + assert "semanage port" in c or ("semanage" in c and "port" in c), ( + "should use semanage port for nonstandard port SELinux labeling" + ) + + def test_systemd_journal_workflow(self): + """Skill teaches systemctl status + journalctl -u for service logs.""" + c = read_report().lower() + assert any(t in c for t in ["systemctl", "journalctl"]) and any(t in c for t in [ + "status", "-u", "service", "log" + ]), "should use systemd/journal workflow" + + def test_firewall_cmd(self): + """Skill teaches firewall-cmd for port management.""" + c = read_report().lower() + assert "firewall-cmd" in c or "firewall" in c, ( + "should check firewall configuration" + ) + + def test_concrete_remediation(self): + """Skill teaches concrete remediation commands for RHEL issues.""" + c = read_report().lower() + assert any(t in c for t in ["systemctl restart", "firewall-cmd", "semanage", "restorecon"]) or ( + "```" in read_report() and any(t in c for t in ["sudo", "systemctl"]) + ), "should include concrete RHEL remediation commands" + + def test_permanent_firewall_flag(self): + """Skill teaches using --permanent flag with firewall-cmd to persist rules + across reboots. Without skill, agents use firewall-cmd without --permanent, + creating rules that are lost on reboot.""" + c = read_report() + assert "--permanent" in c, ( + "should use --permanent flag with firewall-cmd for persistent rules" + ) + + def test_http_port_t_selinux_type(self): + """Skill teaches the specific SELinux type http_port_t for web service ports. + Without skill, agents use generic semanage commands without specifying the + correct SELinux type for HTTP ports.""" + c = read_report() + assert "http_port_t" in c, ( + "should reference http_port_t SELinux type for port labeling" + ) + + def test_getenforce_check(self): + """Skill teaches using getenforce to verify SELinux mode (Enforcing/Permissive) + as a first diagnostic step. Without skill, agents jump to specific SELinux + fixes without verifying the enforcement mode.""" + c = read_report().lower() + assert "getenforce" in c, ( + "should use getenforce to check SELinux enforcement mode" + ) diff --git a/evaluation/with_skills/rh-developer__deploy/environment/Dockerfile b/evaluation/with_skills/rh-developer__deploy/environment/Dockerfile new file mode 100644 index 00000000..1cbfefcf --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-developer__deploy/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__deploy/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__deploy/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__deploy/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/with_skills/rh-developer__deploy/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__deploy/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/with_skills/rh-developer__deploy/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__deploy/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__deploy/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__deploy/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/with_skills/rh-developer__deploy/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__deploy/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__deploy/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__deploy/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__deploy/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__deploy/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/with_skills/rh-developer__deploy/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__deploy/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__deploy/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-developer__deploy/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-developer__deploy/environment/skills/deploy/SKILL.md b/evaluation/with_skills/rh-developer__deploy/environment/skills/deploy/SKILL.md new file mode 100644 index 00000000..b24e7d12 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/skills/deploy/SKILL.md @@ -0,0 +1,277 @@ +--- +name: deploy +description: | + Create Kubernetes Deployment, Service, and Route resources on OpenShift to deploy and expose an application. Use this skill after /s2i-build to make the built image accessible. Handles port detection, replica configuration, HTTPS route creation, rollout monitoring, and rollback on failure. Triggers on /deploy command when user wants to deploy a container image to OpenShift. +model: inherit +color: green +metadata: + user_invocable: "true" +--- + +# /deploy Skill + +Create Kubernetes/OpenShift resources (Deployment, Service, Route) to deploy and expose an application from a container image. + +## Prerequisites + +Before running this skill: +1. User is logged into OpenShift cluster +2. Container image exists (from ImageStream or external registry) +3. Target namespace exists + +## When to Use This Skill + +Use `/deploy` after building a container image (via `/s2i-build` or external registry) to create Deployment, Service, and Route resources on OpenShift. This skill handles port detection, replica configuration, rollout monitoring, and rollback on failure. + +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +## Workflow + +### Step 1: Gather Deployment Information + +```markdown +## Deployment Configuration + +**Current OpenShift Context:** +- Cluster: [cluster] +- Namespace: [namespace] + +**Please confirm deployment settings:** + +| Setting | Value | Source | +|---------|-------|--------| +| App Name | `[name]` | [from s2i-build / input] | +| Image | `[image-ref]` | [from ImageStream / input] | +| Container Port | `[port]` | [detected / needs input] | +| Replicas | `1` | [default] | +| Expose Route | `yes` | [default] | + +Confirm these settings or tell me what to change. +``` + +Proceed automatically without waiting for user input. + +### Step 2: Detect Container Port + +Try to detect port from project files: + +1. **Dockerfile:** Look for `EXPOSE ` (Most accurate for container builds) +2. **Web Server Config:** Look for `listen ` in `nginx.conf`, `httpd.conf`, etc. +3. **Framework Defaults:** + - **Node.js:** Look for `PORT` env var usage, common: 3000 (dev), 8080 (prod/S2I) + - **Python:** Flask default 5000, FastAPI/Uvicorn 8000 + - **Java:** Spring Boot 8080, Quarkus 8080 + - **Go:** Common 8080 + - **Ruby Rails:** 3000 + +```markdown +## Port Detection + +I detected port **[port]** based on: +- [reason - e.g., "PORT environment variable in package.json scripts"] + +Is this correct? +- yes - Use port [port] +- no - Specify the correct port +``` + +Proceed automatically without waiting for user input. + +If unable to detect: +```markdown +## Port Required + +I couldn't automatically detect the container port. + +Common ports by framework: +- Node.js/Express: 3000 or 8080 +- Python Flask: 5000 +- Python FastAPI: 8000 +- Java Spring Boot: 8080 +- Go: 8080 + +**What port does your application listen on?** +``` + +### Step 3: Create Deployment + +Show the Deployment manifest: + +```markdown +## Step 1 of 3: Create Deployment + +Read `templates/deployment.yaml.template` and substitute `${APP_NAME}`, `${NAMESPACE}`, `${PORT}`, `${REPLICAS}` with session state values. + +Show the rendered YAML to user and confirm. + +**Proceed with creating this Deployment?** (yes/no) +``` + +Proceed automatically without waiting for user input. + +- If user says "yes" → Use kubernetes MCP `resources_create_or_update` to apply +- If user says "no" → Ask what they would like to change +- If user wants modifications → Update the YAML and show again for confirmation + +### Step 4: Create Service + +```markdown +## Step 2 of 3: Create Service + +Read `templates/service.yaml.template` and substitute `${APP_NAME}`, `${NAMESPACE}`, `${PORT}`. + +Show the rendered YAML to user and confirm. + +**Proceed with creating this Service?** (yes/no) +``` + +Proceed automatically without waiting for user input. + +- If user says "yes" → Use kubernetes MCP `resources_create_or_update` to apply +- If user says "no" → Ask what they would like to change +- If user wants modifications → Update the YAML and show again for confirmation + +### Step 5: Create Route (Optional) + +If user wants external exposure: + +```markdown +## Step 3 of 3: Create Route + +Read `templates/route.yaml.template` and substitute `${APP_NAME}`, `${NAMESPACE}`. + +Show the rendered YAML to user and confirm. + +**Proceed with creating this Route?** (yes/no/skip) +``` + +Proceed automatically without waiting for user input. + +- If user says "yes" → Use kubernetes MCP `resources_create_or_update` to apply +- If user says "skip" → Skip Route creation and proceed to rollout monitoring +- If user says "no" → Ask what they would like to change +- If user wants modifications → Update the YAML and show again for confirmation + +### Step 6: Wait for Rollout + +Monitor deployment status: + +```markdown +## Deployment Rollout + +Waiting for pods to be ready... + +**Deployment:** [app-name] +**Desired:** [replicas] +**Ready:** [current]/[replicas] + +**Pod Status:** +| Pod | Status | Ready | Restarts | +|-----|--------|-------|----------| +| [app-name]-xxx-yyy | Running | 1/1 | 0 | + +[Poll until ready or timeout after 5 minutes] +``` + +### Step 6a: Handle Deployment Failure + +If pods do not become ready within the timeout period, or pods are in error states (CrashLoopBackOff, ImagePullBackOff, Pending): + +```markdown +## Deployment Failed + +**Status:** Rollout did not complete successfully + +**Pod Status:** +| Pod | Status | Ready | Restarts | Reason | +|-----|--------|-------|----------|--------| +| [app-name]-xxx-yyy | [CrashLoopBackOff/ImagePullBackOff/Pending] | 0/1 | [count] | [reason] | + +**Events:** +| Time | Type | Message | +|------|------|---------| +| [time] | Warning | [event message] | + +--- + +**Would you like me to diagnose the issue?** + +1. **Debug Pod** - Investigate pod failures (runs `/debug-pod`) + - Analyzes pod status, events, logs, and resource constraints + - Identifies root cause (OOM, image pull issues, crashes, etc.) + +2. **Debug Network** - Investigate connectivity issues (runs `/debug-network`) + - Checks service endpoints, route status, network policies + - Useful if pods are running but service is unreachable + +3. **View logs manually** - Show pod logs without full diagnosis + +4. **Rollback deployment** - Delete created resources and stop + +5. **Continue waiting** - Wait another 5 minutes for rollout + +Select an option: +``` + +Proceed automatically without waiting for user input. + +- If user selects "Debug Pod" → Invoke `/debug-pod` skill with pod name +- If user selects "Debug Network" → Invoke `/debug-network` skill with service name +- If user selects "View logs" → Show pod logs using `pod_logs` +- If user selects "Rollback" → Delete Deployment, Service, Route +- If user selects "Continue" → Wait another polling cycle + +### Step 7: Deployment Complete + +```markdown +## Deployment Complete! + +**Application:** [app-name] +**Namespace:** [namespace] + +**Access URLs:** +| Type | URL | +|------|-----| +| External | https://[route-host] | +| Internal | http://[app-name].[namespace].svc.cluster.local:[port] | + +**Resources Created:** +| Resource | Name | Status | +|----------|------|--------| +| Deployment | [app-name] | [replicas]/[replicas] Ready | +| Service | [app-name] | Active | +| Route | [app-name] | Admitted | + +**Quick Commands:** +```bash +# View logs +oc logs -f deployment/[app-name] -n [namespace] + +# Scale replicas +oc scale deployment/[app-name] --replicas=3 -n [namespace] + +# Restart pods +oc rollout restart deployment/[app-name] -n [namespace] + +# Delete all +oc delete all -l app=[app-name] -n [namespace] +``` + +Your application is now live! +``` + +## Dependencies + +### Required MCP Servers +- `openshift` - cluster resource creation and management + +### Related Skills +- `/debug-pod` - Pod failures (CrashLoopBackOff, OOMKilled, ImagePullBackOff) +- `/debug-network` - Service connectivity issues (no endpoints, 503 errors) +- `/debug-build` - Build failures before deployment + +### Reference Documentation +- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (oc), cluster access verification +- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common error patterns and troubleshooting diff --git a/evaluation/with_skills/rh-developer__deploy/environment/templates/buildconfig.yaml.template b/evaluation/with_skills/rh-developer__deploy/environment/templates/buildconfig.yaml.template new file mode 100644 index 00000000..b3294eb2 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/templates/buildconfig.yaml.template @@ -0,0 +1,38 @@ +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: build + app.kubernetes.io/part-of: ${APP_NAME} +spec: + source: + type: Git + git: + uri: ${GIT_URL} + ref: ${GIT_BRANCH} + strategy: + type: Source + sourceStrategy: + from: + kind: DockerImage + name: ${BUILDER_IMAGE} + env: [] + output: + to: + kind: ImageStreamTag + name: ${APP_NAME}:latest + triggers: + - type: ConfigChange + - type: ImageChange + runPolicy: Serial + resources: + limits: + memory: "1Gi" + cpu: "1" + requests: + memory: "512Mi" + cpu: "500m" diff --git a/evaluation/with_skills/rh-developer__deploy/environment/templates/deployment.yaml.template b/evaluation/with_skills/rh-developer__deploy/environment/templates/deployment.yaml.template new file mode 100644 index 00000000..eb3b481a --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/templates/deployment.yaml.template @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: application + app.kubernetes.io/part-of: ${APP_NAME} + annotations: + image.openshift.io/triggers: | + [{"from":{"kind":"ImageStreamTag","name":"${APP_NAME}:latest"},"fieldPath":"spec.template.spec.containers[0].image"}] +spec: + replicas: ${REPLICAS} + selector: + matchLabels: + app: ${APP_NAME} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + template: + metadata: + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + spec: + containers: + - name: ${APP_NAME} + image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/${APP_NAME}:latest + ports: + - containerPort: ${CONTAINER_PORT} + protocol: TCP + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + livenessProbe: + httpGet: + path: / + port: ${CONTAINER_PORT} + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: / + port: ${CONTAINER_PORT} + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + env: [] + restartPolicy: Always + terminationGracePeriodSeconds: 30 diff --git a/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/Chart.yaml.template b/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/Chart.yaml.template new file mode 100644 index 00000000..1aa22dd1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/Chart.yaml.template @@ -0,0 +1,13 @@ +apiVersion: v2 +name: ${APP_NAME} +description: ${APP_DESCRIPTION} +type: application +version: 0.1.0 +appVersion: "${APP_VERSION}" +keywords: + - ${LANGUAGE} + - ${FRAMEWORK} + - openshift +maintainers: + - name: ${MAINTAINER_NAME} + email: ${MAINTAINER_EMAIL} diff --git a/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/NOTES.txt.template b/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/NOTES.txt.template new file mode 100644 index 00000000..154e628d --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/NOTES.txt.template @@ -0,0 +1,32 @@ +Congratulations! Your application {{ include "${APP_NAME}.fullname" . }} has been deployed. + +{{- if .Values.route.enabled }} + +Access your application at: +{{- if .Values.route.host }} + https://{{ .Values.route.host }} +{{- else }} + Run: oc get route {{ include "${APP_NAME}.fullname" . }} -o jsonpath='{.spec.host}' +{{- end }} + +{{- else }} + +Your application is available internally at: + {{ include "${APP_NAME}.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }} + +To expose it externally, create a Route or set route.enabled=true. + +{{- end }} + +Useful commands: + # View pods + oc get pods -l app.kubernetes.io/name={{ include "${APP_NAME}.name" . }} + + # View logs + oc logs -l app.kubernetes.io/name={{ include "${APP_NAME}.name" . }} -f + + # Upgrade release + helm upgrade {{ .Release.Name }} ./{{ .Chart.Name }} -f values.yaml + + # Uninstall release + helm uninstall {{ .Release.Name }} diff --git a/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/_helpers.tpl.template b/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/_helpers.tpl.template new file mode 100644 index 00000000..15873b10 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/_helpers.tpl.template @@ -0,0 +1,60 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "${APP_NAME}.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "${APP_NAME}.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "${APP_NAME}.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "${APP_NAME}.labels" -}} +helm.sh/chart: {{ include "${APP_NAME}.chart" . }} +{{ include "${APP_NAME}.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "${APP_NAME}.selectorLabels" -}} +app.kubernetes.io/name: {{ include "${APP_NAME}.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "${APP_NAME}.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "${APP_NAME}.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/deployment.yaml.template b/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/deployment.yaml.template new file mode 100644 index 00000000..a6cbd868 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/deployment.yaml.template @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "${APP_NAME}.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "${APP_NAME}.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "${APP_NAME}.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/route.yaml.template b/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/route.yaml.template new file mode 100644 index 00000000..e2bab29a --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/route.yaml.template @@ -0,0 +1,24 @@ +{{- if .Values.route.enabled }} +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + {{- if .Values.route.host }} + host: {{ .Values.route.host }} + {{- end }} + to: + kind: Service + name: {{ include "${APP_NAME}.fullname" . }} + weight: 100 + port: + targetPort: http + {{- with .Values.route.tls }} + tls: + termination: {{ .termination }} + insecureEdgeTerminationPolicy: {{ .insecureEdgeTerminationPolicy }} + {{- end }} + wildcardPolicy: None +{{- end }} diff --git a/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/service.yaml.template b/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/service.yaml.template new file mode 100644 index 00000000..837bc888 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/templates/service.yaml.template @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "${APP_NAME}.selectorLabels" . | nindent 4 }} diff --git a/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/values.yaml.template b/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/values.yaml.template new file mode 100644 index 00000000..1cca6017 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/templates/helm/values.yaml.template @@ -0,0 +1,67 @@ +# Default values for ${APP_NAME} +replicaCount: 1 + +image: + repository: ${IMAGE_REPOSITORY} + pullPolicy: IfNotPresent + tag: "${IMAGE_TAG}" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: "" + +podAnnotations: {} +podSecurityContext: {} +securityContext: {} + +service: + type: ClusterIP + port: ${CONTAINER_PORT} + +route: + enabled: true + host: "" + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + +resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + +livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + +readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + +nodeSelector: {} +tolerations: [] +affinity: {} + +env: [] +# - name: MY_VAR +# value: "my-value" diff --git a/evaluation/with_skills/rh-developer__deploy/environment/templates/imagestream.yaml.template b/evaluation/with_skills/rh-developer__deploy/environment/templates/imagestream.yaml.template new file mode 100644 index 00000000..46572193 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/templates/imagestream.yaml.template @@ -0,0 +1,13 @@ +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: image + app.kubernetes.io/part-of: ${APP_NAME} +spec: + lookupPolicy: + local: false diff --git a/evaluation/with_skills/rh-developer__deploy/environment/templates/route.yaml.template b/evaluation/with_skills/rh-developer__deploy/environment/templates/route.yaml.template new file mode 100644 index 00000000..7c53d2e7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/templates/route.yaml.template @@ -0,0 +1,21 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: route + app.kubernetes.io/part-of: ${APP_NAME} +spec: + to: + kind: Service + name: ${APP_NAME} + weight: 100 + port: + targetPort: http + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + wildcardPolicy: None diff --git a/evaluation/with_skills/rh-developer__deploy/environment/templates/service.yaml.template b/evaluation/with_skills/rh-developer__deploy/environment/templates/service.yaml.template new file mode 100644 index 00000000..7e1cf371 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/templates/service.yaml.template @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: service + app.kubernetes.io/part-of: ${APP_NAME} +spec: + selector: + app: ${APP_NAME} + ports: + - name: http + port: ${CONTAINER_PORT} + targetPort: ${CONTAINER_PORT} + protocol: TCP + type: ClusterIP + sessionAffinity: None diff --git a/evaluation/with_skills/rh-developer__deploy/environment/templates/systemd/systemd-container-rootful.service b/evaluation/with_skills/rh-developer__deploy/environment/templates/systemd/systemd-container-rootful.service new file mode 100644 index 00000000..c1e8fe8f --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/templates/systemd/systemd-container-rootful.service @@ -0,0 +1,27 @@ +# Rootful Podman container managed by systemd (system service) +# Location: /etc/systemd/system/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${PORT} - Port number (used for both host and container binding) +# ${IMAGE} - Container image reference + +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run --name ${APP_NAME} \ + -p ${PORT}:${PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target diff --git a/evaluation/with_skills/rh-developer__deploy/environment/templates/systemd/systemd-container-rootless.service b/evaluation/with_skills/rh-developer__deploy/environment/templates/systemd/systemd-container-rootless.service new file mode 100644 index 00000000..ca9dc371 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/templates/systemd/systemd-container-rootless.service @@ -0,0 +1,27 @@ +# Rootless Podman container managed by systemd (user service) +# Location: ~/.config/systemd/user/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${PORT} - Port number (used for both host and container binding) +# ${IMAGE} - Container image reference + +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run --name ${APP_NAME} \ + -p ${PORT}:${PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target diff --git a/evaluation/with_skills/rh-developer__deploy/environment/templates/systemd/systemd-native.service b/evaluation/with_skills/rh-developer__deploy/environment/templates/systemd/systemd-native.service new file mode 100644 index 00000000..c55cfc07 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/environment/templates/systemd/systemd-native.service @@ -0,0 +1,39 @@ +# Native application managed by systemd (system service) +# Location: /etc/systemd/system/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${SERVICE_USER} - User to run the service as +# ${APP_PATH} - Application install path (e.g., /opt/app-name) +# ${PORT} - Application listen port +# ${START_COMMAND} - Application start command +# +# Start command examples by language: +# Node.js: /usr/bin/node ${APP_PATH}/server.js +# Python: /usr/bin/python3 ${APP_PATH}/app.py +# Java: /usr/bin/java -jar ${APP_PATH}/app.jar +# Go: ${APP_PATH}/binary-name + +[Unit] +Description=${APP_NAME} Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=${APP_PATH} +Environment=PORT=${PORT} +ExecStart=${START_COMMAND} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=${APP_PATH} + +[Install] +WantedBy=multi-user.target diff --git a/evaluation/with_skills/rh-developer__deploy/instruction.md b/evaluation/with_skills/rh-developer__deploy/instruction.md new file mode 100644 index 00000000..f84c6177 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/instruction.md @@ -0,0 +1,14 @@ +# Application Deployment Task + +You are a Red Hat developer. Your team needs to deploy a web application to OpenShift that will be accessible to external users via HTTPS. + +## Requirements +- Examine the target namespace and available resources on the cluster +- Define the deployment: container image, replica count, resource requests and limits, and health checks +- Configure a Service to expose the application pods internally +- Configure a Route for external HTTPS access with appropriate TLS settings +- Verify the deployment plan addresses image pull access and correct container port mapping + +Document your deployment plan and the complete resource definitions in `/root/report.md`. + +Use MCP tools to examine the cluster. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-developer__deploy/solution/solve.sh b/evaluation/with_skills/rh-developer__deploy/solution/solve.sh new file mode 100644 index 00000000..b8f9ec1b --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/solution/solve.sh @@ -0,0 +1,61 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Deployment Plan: customer-portal + +## Deployment +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: customer-portal +spec: + replicas: 1 + selector: + matchLabels: + app: customer-portal + template: + metadata: + labels: + app: customer-portal + spec: + containers: + - name: customer-portal + image: image-registry.openshift-image-registry.svc:5000/myproject/customer-portal:latest + ports: + - containerPort: 3000 +``` + +## Service +```yaml +apiVersion: v1 +kind: Service +metadata: + name: customer-portal +spec: + selector: + app: customer-portal + ports: + - port: 3000 + targetPort: 3000 +``` + +## Route +```yaml +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: customer-portal +spec: + to: + kind: Service + name: customer-portal + port: + targetPort: 3000 + tls: + termination: edge +``` + +### Internal DNS: `http://customer-portal.myproject.svc.cluster.local:3000` + +### On failure: Debug Pod (/debug-pod) or Debug Network (/debug-network) +REPORT_EOF diff --git a/evaluation/with_skills/rh-developer__deploy/task.toml b/evaluation/with_skills/rh-developer__deploy/task.toml new file mode 100644 index 00000000..86e6c127 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__deploy" +name = "rh-developer Deployment Planning Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "deploy", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-developer__deploy/tests/llm_judge.py b/evaluation/with_skills/rh-developer__deploy/tests/llm_judge.py new file mode 100644 index 00000000..5ce75615 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "security_hardening", + "file": "/root/report.md", + "question": "Does the report include deployment security hardening such as runAsNonRoot, allowPrivilegeEscalation: false, seccompProfile, or insecureEdgeTerminationPolicy: Redirect on the Route?", + "reference": "A skilled report includes security context on the Deployment (runAsNonRoot: true, allowPrivilegeEscalation: false) and configures Route with insecureEdgeTerminationPolicy: Redirect. An unskilled report creates basic Deployment+Service+Route without security hardening." + }, + { + "id": "deployment_service_route", + "file": "/root/report.md", + "question": "Does the report create all three resources (Deployment, Service, Route) with correct selector/port alignment?", + "reference": "A skilled report defines Deployment + Service + Route with matching selectors, targetPort, and containerPort. An unskilled report may miss selector alignment or skip the Route." + }, + { + "id": "tls_and_port_detection", + "file": "/root/report.md", + "question": "Does the report address TLS termination for the Route and port detection based on framework defaults?", + "reference": "A skilled report configures TLS (edge/passthrough) on the Route and detects the application port from framework conventions. An unskilled report hardcodes port 8080 and skips TLS." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-developer__deploy/tests/test.sh b/evaluation/with_skills/rh-developer__deploy/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-developer__deploy/tests/test_outputs.py b/evaluation/with_skills/rh-developer__deploy/tests/test_outputs.py new file mode 100644 index 00000000..01ea8257 --- /dev/null +++ b/evaluation/with_skills/rh-developer__deploy/tests/test_outputs.py @@ -0,0 +1,87 @@ +""" +Tests for rh-developer__deploy per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_deploy(self): + content = read_report().lower() + assert "deploy" in content, "report should mention deployment" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_insecure_redirect_policy(self): + """Skill teaches insecureEdgeTerminationPolicy: Redirect on Route to force + HTTP→HTTPS. Without skill, agents create Routes without redirect policy, + leaving HTTP access open.""" + c = read_report() + assert "insecureEdgeTerminationPolicy" in c or ( + "Redirect" in c and ("http" in c.lower() and "https" in c.lower()) + ), "should configure insecureEdgeTerminationPolicy: Redirect on Route" + + def test_framework_port_detection(self): + """Skill teaches port inference by framework defaults (Node 3000/8080, + Python 5000/8000, Java 8080). Without skill, agents hardcode 8080.""" + c = read_report().lower() + assert any(t in c for t in ["port", "8080", "3000", "5000"]) and any(t in c for t in [ + "detect", "expose", "listen", "framework", "default", "infer" + ]), "should address port detection from framework defaults" + + def test_deployment_service_route_triad(self): + """Skill teaches creating Deployment, Service, Route in sequence.""" + c = read_report().lower() + assert any(t in c for t in ["deployment"]) and "service" in c and any(t in c for t in [ + "route", "external", "https" + ]), "should define Deployment + Service + Route" + + def test_selector_alignment(self): + """Skill teaches Service selector must match Deployment pod labels.""" + c = read_report().lower() + assert any(t in c for t in ["selector", "label", "targetport", "target port"]) or ( + "service" in c and "port" in c and "match" in c + ), "should address selector/port alignment" + + def test_tls_route_config(self): + """Skill teaches Route with TLS termination (edge/passthrough).""" + c = read_report().lower() + assert any(t in c for t in ["tls", "https", "edge", "termination"]), ( + "should address Route TLS for external access" + ) + + def test_hpa_autoscaling(self): + """Skill teaches including HorizontalPodAutoscaler configuration for + production deployments. Without skill, agents set static replica count + without autoscaling.""" + c = read_report() + assert "HorizontalPodAutoscaler" in c or "autoscaling/v2" in c or ( + "hpa" in c.lower() and "autoscal" in c.lower() + ), "should include HorizontalPodAutoscaler for production scaling" + + def test_hsts_security_headers(self): + """Skill teaches HSTS headers or Strict-Transport-Security configuration + on OpenShift Routes. Without skill, agents skip transport security headers.""" + c = read_report() + assert any(t in c for t in [ + "HSTS", "Strict-Transport-Security", "hsts", + "haproxy.router.openshift.io", + ]), "should configure HSTS or transport security headers on Route" diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/Dockerfile b/evaluation/with_skills/rh-developer__detect-project/environment/Dockerfile new file mode 100644 index 00000000..608ae0df --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/Dockerfile @@ -0,0 +1,71 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs +COPY sample-project /root/project + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__detect-project/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__detect-project/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__detect-project/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__detect-project/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__detect-project/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__detect-project/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__detect-project/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__detect-project/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__detect-project/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-developer__detect-project/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/sample-project/.s2i/environment b/evaluation/with_skills/rh-developer__detect-project/environment/sample-project/.s2i/environment new file mode 100644 index 00000000..a16a265c --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/sample-project/.s2i/environment @@ -0,0 +1 @@ +APP_FILE=app.py diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/sample-project/Dockerfile b/evaluation/with_skills/rh-developer__detect-project/environment/sample-project/Dockerfile new file mode 100644 index 00000000..a7fb87b7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/sample-project/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.11-slim + +WORKDIR /app +COPY requirements.txt . +RUN pip install -r requirements.txt +COPY . . + +EXPOSE 8080 +CMD ["gunicorn", "-b", "0.0.0.0:8080", "app:app"] diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/sample-project/app.py b/evaluation/with_skills/rh-developer__detect-project/environment/sample-project/app.py new file mode 100644 index 00000000..4761fe8a --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/sample-project/app.py @@ -0,0 +1,12 @@ +from flask import Flask + +app = Flask(__name__) + + +@app.route("/") +def hello(): + return "Hello, World!" + + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=8080) diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/sample-project/requirements.txt b/evaluation/with_skills/rh-developer__detect-project/environment/sample-project/requirements.txt new file mode 100644 index 00000000..cb04ebda --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/sample-project/requirements.txt @@ -0,0 +1,3 @@ +flask +gunicorn +psycopg2-binary diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/sample-project/tests/test_app.py b/evaluation/with_skills/rh-developer__detect-project/environment/sample-project/tests/test_app.py new file mode 100644 index 00000000..5e8fbc93 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/sample-project/tests/test_app.py @@ -0,0 +1,9 @@ +import pytest +from app import app + + +def test_hello(): + with app.test_client() as client: + r = client.get("/") + assert r.status_code == 200 + assert b"Hello" in r.data diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/skills/detect-project/SKILL.md b/evaluation/with_skills/rh-developer__detect-project/environment/skills/detect-project/SKILL.md new file mode 100644 index 00000000..2f6d126e --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/skills/detect-project/SKILL.md @@ -0,0 +1,277 @@ +--- +name: detect-project +description: | + Analyze a project folder or GitHub repository to detect programming language, framework, and version requirements. Use this skill when containerizing an application, selecting an S2I builder image, deploying to OpenShift or RHEL, or determining a project's tech stack. Supports Node.js, Python, Java, Go, Ruby, .NET, PHP, and Perl. Triggers on /detect-project command or when user needs build strategy recommendations. Run before /s2i-build or /rhel-deploy. +model: inherit +color: cyan +metadata: + user_invocable: "true" +--- + +# /detect-project Skill + +## Critical Restrictions +- **DO NOT CLONE** remote repositories unless the user explicitly selects the "Clone & Inspect" option. +- **ALWAYS** use `github-mcp-server` tools (`list_directory`, `get_file_contents`) for initial analysis of remote URLs. +- **NEVER** assume you have permission to write to the local filesystem for analysis purposes. + +Analyze the project to detect language/framework and recommend a build strategy. This skill handles both local project directories and remote Git repositories. + +## When to Use This Skill + +- User wants to containerize or deploy an application and needs language/framework detection +- User asks what tech stack a project uses or needs a build strategy recommendation +- Run before `/s2i-build`, `/recommend-image`, or `/rhel-deploy` to identify project type + +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +## Workflow + +### Step 1: Context Analysis + +**Scenario A: Local Files Available** +If you are in a project directory with source code: +1. Proceed to **Step 2: Scan Project Files**. + +**Scenario B: Remote Git URL Provided** +If the user provided a Git URL (e.g., `https://github.com/...`): + +Use the **github-mcp-server** to analyze the repository directly without cloning: + +1. Use `mcp_github_get_file_contents(owner, repo, path="/")` to list repository contents +2. Read key files using `fetch_mcp_resource` with URI format: `repo://{owner}/{repo}/contents/{file-path}` + - Example: `repo://myorg/myrepo/contents/package.json` +3. Proceed with analysis as if local files + +```markdown +## Analyzing Remote Repository + +I'm inspecting the repository: `[git-url]` + +Using GitHub API to analyze the project structure... + +[Use github MCP to get_file_contents for indicator files] + +**Files Found:** +- [list files from repo root] + +[Continue with Step 2: Scan Project Files using the remote file contents] +``` + +If GitHub MCP is unavailable or repo is private without access: + +```markdown +## Remote Repository Access + +I see you want to deploy from: `[git-url]` + +I couldn't access the repository directly. Options: + +1. **Remote S2I Build** (Recommended for standard apps) + - OpenShift will clone and build the code directly. + - I need you to confirm the language/framework. + +2. **Remote Podman Build** (Recommended if Containerfile/Dockerfile exists) + - OpenShift will use the Containerfile/Dockerfile in the repo. + - Best if you already have a custom build process. + +3. **Clone & Inspect** + - I will clone the repo locally to analyze it first. + - This helps if you're unsure about the project details. + +**Which approach do you prefer?** +``` + +Proceed automatically without waiting for user input. + +**Scenario C: No Context** +If no files and no URL: +1. Ask the user for the Git URL or to navigate to a project folder. + +### Step 2: Scan Project Files (Local Only) + +Look for these indicator files in the project root: + +| File | Language | Framework Hint | +|---|----|----| +| `Chart.yaml` | Helm Chart | Existing Helm deployment available | +| `package.json` | Node.js | Check for next, angular, vue, react | +| `pom.xml` | Java | Check for spring-boot, quarkus deps | +| `build.gradle` / `build.gradle.kts` | Java | Check for spring, quarkus plugins | +| `requirements.txt` | Python | - | +| `Pipfile` | Python | Pipenv | +| `pyproject.toml` | Python | Poetry or modern Python | +| `go.mod` | Go | - | +| `Gemfile` | Ruby | Check for rails | +| `composer.json` | PHP | Check for laravel, symfony | +| `*.csproj` / `*.sln` | .NET | - | +| `Cargo.toml` | Rust | No official S2I | +| `Dockerfile` / `Containerfile` | Pre-containerized | May not need S2I | + +### Helm Chart Detection + +Also check for Helm charts in these locations (in order): + +| Priority | Path | Description | +|----------|------|-------------| +| 1 | `./Chart.yaml` | Root directory | +| 2 | `./chart/Chart.yaml` | Chart subdirectory | +| 3 | `./charts/*/Chart.yaml` | Charts directory | +| 4 | `./helm/Chart.yaml` | Helm subdirectory | +| 5 | `./deploy/helm/Chart.yaml` | Deploy directory | + +If Chart.yaml is found, parse it to extract: +- `name`: Chart name +- `version`: Chart version (SemVer) +- `appVersion`: Application version +- `description`: Chart description + +Also check for: +- `values.yaml`: Default configuration +- `templates/`: Template files + +### Step 3: Detect Version Requirements + +For each detected language, extract version info: + +**Node.js:** +- Check `engines.node` in package.json +- Example: `"engines": { "node": ">=18" }` + +**Python:** +- Check `python_requires` in pyproject.toml +- Check `runtime.txt` for version +- Check `.python-version` file + +**Java:** +- Check `` or `` in pom.xml +- Check `sourceCompatibility` in build.gradle + +**Go:** +- Check `go` directive in go.mod +- Example: `go 1.21` + +### Step 4: Detect Framework + +Look for framework-specific indicators: + +**Node.js frameworks:** +- `next.config.js` or `next.config.mjs` → Next.js +- `angular.json` → Angular +- `vue.config.js` or `vite.config.ts` with vue → Vue.js +- `remix.config.js` → Remix + +**Java frameworks:** +- `quarkus` in dependencies → Quarkus +- `spring-boot` in dependencies → Spring Boot +- `micronaut` in dependencies → Micronaut + +**Python frameworks:** +- `django` in requirements → Django +- `flask` in requirements → Flask +- `fastapi` in requirements → FastAPI + +### Step 4.5: Detect Python Entry Point (Python projects only) + +For Python projects, detect the application entry point to ensure proper S2I configuration: + +**Check for entry point files (in order of S2I preference):** +1. `app.py` - Default S2I Python entry point (no config needed) +2. `application.py` - Alternative default +3. `wsgi.py` - WSGI module +4. `main.py` - Common alternative (requires APP_MODULE config) +5. Any file with `if __name__ == "__main__"` and Flask/FastAPI app + +**Check requirements.txt/Pipfile/pyproject.toml for WSGI server:** +- `gunicorn` - Required for APP_MODULE to work with S2I Python +- `uwsgi` - Alternative WSGI server + +### Step 5: Present Findings + +Format your response: + +```markdown +## Project Analysis Results + +**Detected Language:** [Language] +**Framework:** [Framework or "None detected"] +**Version:** [Version or "Not specified"] + +**Detection Confidence:** [High/Medium/Low] +- High: Clear indicator file with version info +- Medium: Indicator file found but no version specified +- Low: Multiple conflicting indicators or unusual setup + +**Indicator Files Found:** +- [list of files] + +--- + +**Recommended S2I Builder Image:** +`registry.access.redhat.com/ubi9/[image-name]` + +**Why this image:** +- [Brief explanation] + +**Alternative Options:** +1. `[alternative-1]` - [when to choose] +2. `[alternative-2]` - [when to choose] + +--- + +**Suggested App Name:** `[derived-name]` +(based on [folder name / package.json name / pom artifactId]) + +--- + +**Image Selection Options:** +- **quick** - Use the recommended image above (good for most cases) +- **smart** - Run `/recommend-image` for use-case aware selection (production vs dev, security, performance) + +Please confirm: +1. Is the detected language/framework correct? +2. Image selection: quick or smart? +3. Is the app name acceptable? + +Type 'yes' to confirm all with quick image selection, 'smart' for tailored recommendation, or tell me what to change. +``` + +Proceed automatically without waiting for user input. + +- If user says "yes" → Save configuration with quick image selection +- If user says "smart" → Invoke `/recommend-image` skill +- If user provides corrections → Update values and show again for confirmation + +**Note:** If the user selects "smart", invoke the `/recommend-image` skill with the detected `LANGUAGE`, `FRAMEWORK`, and `VERSION` values. + +## Output Variables + +After successful detection, these values should be available for other skills: + +| Variable | Description | Example | +|----|----|---| +| `APP_NAME` | Application name | `my-nodejs-app` | +| `LANGUAGE` | Detected language | `nodejs` | +| `FRAMEWORK` | Detected framework | `express` | +| `VERSION` | Language version | `20` | +| `BUILDER_IMAGE` | Full S2I image reference | `registry.access.redhat.com/ubi9/nodejs-20` | +| `BUILD_STRATEGY` | Build strategy | `Source` (S2I) or `Podman` | +| `CONTAINER_PORT` | Application listen port | `8080` | +| `HELM_CHART_PATH` | Path to Helm chart | `./chart` | + +## Dependencies + +### Required MCP Servers +- `github` - Remote repository analysis via GitHub API (for URL-based detection) + +### Related Skills +- `/s2i-build` - Build with the detected S2I builder image +- `/recommend-image` - Advanced image selection based on detection results +- `/rhel-deploy` - Deploy to RHEL using detected project info + +### Reference Documentation +- [docs/builder-images.md](../../docs/builder-images.md) - Language detection matrix, version-to-image mapping, S2I builder selection +- [docs/python-s2i-entrypoints.md](../../docs/python-s2i-entrypoints.md) - Python entry point detection, APP_MODULE configuration +- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (git) diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/templates/buildconfig.yaml.template b/evaluation/with_skills/rh-developer__detect-project/environment/templates/buildconfig.yaml.template new file mode 100644 index 00000000..b3294eb2 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/templates/buildconfig.yaml.template @@ -0,0 +1,38 @@ +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: build + app.kubernetes.io/part-of: ${APP_NAME} +spec: + source: + type: Git + git: + uri: ${GIT_URL} + ref: ${GIT_BRANCH} + strategy: + type: Source + sourceStrategy: + from: + kind: DockerImage + name: ${BUILDER_IMAGE} + env: [] + output: + to: + kind: ImageStreamTag + name: ${APP_NAME}:latest + triggers: + - type: ConfigChange + - type: ImageChange + runPolicy: Serial + resources: + limits: + memory: "1Gi" + cpu: "1" + requests: + memory: "512Mi" + cpu: "500m" diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/templates/deployment.yaml.template b/evaluation/with_skills/rh-developer__detect-project/environment/templates/deployment.yaml.template new file mode 100644 index 00000000..eb3b481a --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/templates/deployment.yaml.template @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: application + app.kubernetes.io/part-of: ${APP_NAME} + annotations: + image.openshift.io/triggers: | + [{"from":{"kind":"ImageStreamTag","name":"${APP_NAME}:latest"},"fieldPath":"spec.template.spec.containers[0].image"}] +spec: + replicas: ${REPLICAS} + selector: + matchLabels: + app: ${APP_NAME} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + template: + metadata: + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + spec: + containers: + - name: ${APP_NAME} + image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/${APP_NAME}:latest + ports: + - containerPort: ${CONTAINER_PORT} + protocol: TCP + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + livenessProbe: + httpGet: + path: / + port: ${CONTAINER_PORT} + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: / + port: ${CONTAINER_PORT} + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + env: [] + restartPolicy: Always + terminationGracePeriodSeconds: 30 diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/Chart.yaml.template b/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/Chart.yaml.template new file mode 100644 index 00000000..1aa22dd1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/Chart.yaml.template @@ -0,0 +1,13 @@ +apiVersion: v2 +name: ${APP_NAME} +description: ${APP_DESCRIPTION} +type: application +version: 0.1.0 +appVersion: "${APP_VERSION}" +keywords: + - ${LANGUAGE} + - ${FRAMEWORK} + - openshift +maintainers: + - name: ${MAINTAINER_NAME} + email: ${MAINTAINER_EMAIL} diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/NOTES.txt.template b/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/NOTES.txt.template new file mode 100644 index 00000000..154e628d --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/NOTES.txt.template @@ -0,0 +1,32 @@ +Congratulations! Your application {{ include "${APP_NAME}.fullname" . }} has been deployed. + +{{- if .Values.route.enabled }} + +Access your application at: +{{- if .Values.route.host }} + https://{{ .Values.route.host }} +{{- else }} + Run: oc get route {{ include "${APP_NAME}.fullname" . }} -o jsonpath='{.spec.host}' +{{- end }} + +{{- else }} + +Your application is available internally at: + {{ include "${APP_NAME}.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }} + +To expose it externally, create a Route or set route.enabled=true. + +{{- end }} + +Useful commands: + # View pods + oc get pods -l app.kubernetes.io/name={{ include "${APP_NAME}.name" . }} + + # View logs + oc logs -l app.kubernetes.io/name={{ include "${APP_NAME}.name" . }} -f + + # Upgrade release + helm upgrade {{ .Release.Name }} ./{{ .Chart.Name }} -f values.yaml + + # Uninstall release + helm uninstall {{ .Release.Name }} diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/_helpers.tpl.template b/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/_helpers.tpl.template new file mode 100644 index 00000000..15873b10 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/_helpers.tpl.template @@ -0,0 +1,60 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "${APP_NAME}.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "${APP_NAME}.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "${APP_NAME}.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "${APP_NAME}.labels" -}} +helm.sh/chart: {{ include "${APP_NAME}.chart" . }} +{{ include "${APP_NAME}.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "${APP_NAME}.selectorLabels" -}} +app.kubernetes.io/name: {{ include "${APP_NAME}.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "${APP_NAME}.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "${APP_NAME}.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/deployment.yaml.template b/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/deployment.yaml.template new file mode 100644 index 00000000..a6cbd868 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/deployment.yaml.template @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "${APP_NAME}.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "${APP_NAME}.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "${APP_NAME}.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/route.yaml.template b/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/route.yaml.template new file mode 100644 index 00000000..e2bab29a --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/route.yaml.template @@ -0,0 +1,24 @@ +{{- if .Values.route.enabled }} +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + {{- if .Values.route.host }} + host: {{ .Values.route.host }} + {{- end }} + to: + kind: Service + name: {{ include "${APP_NAME}.fullname" . }} + weight: 100 + port: + targetPort: http + {{- with .Values.route.tls }} + tls: + termination: {{ .termination }} + insecureEdgeTerminationPolicy: {{ .insecureEdgeTerminationPolicy }} + {{- end }} + wildcardPolicy: None +{{- end }} diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/service.yaml.template b/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/service.yaml.template new file mode 100644 index 00000000..837bc888 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/templates/service.yaml.template @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "${APP_NAME}.selectorLabels" . | nindent 4 }} diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/values.yaml.template b/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/values.yaml.template new file mode 100644 index 00000000..1cca6017 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/templates/helm/values.yaml.template @@ -0,0 +1,67 @@ +# Default values for ${APP_NAME} +replicaCount: 1 + +image: + repository: ${IMAGE_REPOSITORY} + pullPolicy: IfNotPresent + tag: "${IMAGE_TAG}" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: "" + +podAnnotations: {} +podSecurityContext: {} +securityContext: {} + +service: + type: ClusterIP + port: ${CONTAINER_PORT} + +route: + enabled: true + host: "" + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + +resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + +livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + +readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + +nodeSelector: {} +tolerations: [] +affinity: {} + +env: [] +# - name: MY_VAR +# value: "my-value" diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/templates/imagestream.yaml.template b/evaluation/with_skills/rh-developer__detect-project/environment/templates/imagestream.yaml.template new file mode 100644 index 00000000..46572193 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/templates/imagestream.yaml.template @@ -0,0 +1,13 @@ +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: image + app.kubernetes.io/part-of: ${APP_NAME} +spec: + lookupPolicy: + local: false diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/templates/route.yaml.template b/evaluation/with_skills/rh-developer__detect-project/environment/templates/route.yaml.template new file mode 100644 index 00000000..7c53d2e7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/templates/route.yaml.template @@ -0,0 +1,21 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: route + app.kubernetes.io/part-of: ${APP_NAME} +spec: + to: + kind: Service + name: ${APP_NAME} + weight: 100 + port: + targetPort: http + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + wildcardPolicy: None diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/templates/service.yaml.template b/evaluation/with_skills/rh-developer__detect-project/environment/templates/service.yaml.template new file mode 100644 index 00000000..7e1cf371 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/templates/service.yaml.template @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: service + app.kubernetes.io/part-of: ${APP_NAME} +spec: + selector: + app: ${APP_NAME} + ports: + - name: http + port: ${CONTAINER_PORT} + targetPort: ${CONTAINER_PORT} + protocol: TCP + type: ClusterIP + sessionAffinity: None diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/templates/systemd/systemd-container-rootful.service b/evaluation/with_skills/rh-developer__detect-project/environment/templates/systemd/systemd-container-rootful.service new file mode 100644 index 00000000..c1e8fe8f --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/templates/systemd/systemd-container-rootful.service @@ -0,0 +1,27 @@ +# Rootful Podman container managed by systemd (system service) +# Location: /etc/systemd/system/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${PORT} - Port number (used for both host and container binding) +# ${IMAGE} - Container image reference + +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run --name ${APP_NAME} \ + -p ${PORT}:${PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/templates/systemd/systemd-container-rootless.service b/evaluation/with_skills/rh-developer__detect-project/environment/templates/systemd/systemd-container-rootless.service new file mode 100644 index 00000000..ca9dc371 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/templates/systemd/systemd-container-rootless.service @@ -0,0 +1,27 @@ +# Rootless Podman container managed by systemd (user service) +# Location: ~/.config/systemd/user/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${PORT} - Port number (used for both host and container binding) +# ${IMAGE} - Container image reference + +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run --name ${APP_NAME} \ + -p ${PORT}:${PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/templates/systemd/systemd-native.service b/evaluation/with_skills/rh-developer__detect-project/environment/templates/systemd/systemd-native.service new file mode 100644 index 00000000..c55cfc07 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/environment/templates/systemd/systemd-native.service @@ -0,0 +1,39 @@ +# Native application managed by systemd (system service) +# Location: /etc/systemd/system/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${SERVICE_USER} - User to run the service as +# ${APP_PATH} - Application install path (e.g., /opt/app-name) +# ${PORT} - Application listen port +# ${START_COMMAND} - Application start command +# +# Start command examples by language: +# Node.js: /usr/bin/node ${APP_PATH}/server.js +# Python: /usr/bin/python3 ${APP_PATH}/app.py +# Java: /usr/bin/java -jar ${APP_PATH}/app.jar +# Go: ${APP_PATH}/binary-name + +[Unit] +Description=${APP_NAME} Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=${APP_PATH} +Environment=PORT=${PORT} +ExecStart=${START_COMMAND} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=${APP_PATH} + +[Install] +WantedBy=multi-user.target diff --git a/evaluation/with_skills/rh-developer__detect-project/instruction.md b/evaluation/with_skills/rh-developer__detect-project/instruction.md new file mode 100644 index 00000000..04695ff5 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/instruction.md @@ -0,0 +1,13 @@ +# Project Detection Task + +You are a Red Hat developer. A colleague has handed you a source repository and asked you to figure out what it is and how to deploy it to OpenShift. + +## Requirements +- Examine the project files to identify the programming language, version, and package manager +- Detect the application framework (e.g., Flask, Express, Spring) and build system +- Based on what you find, recommend a deployment strategy: which builder image or base image to use, what build process to follow, and how the application should be started +- Explain your reasoning for the recommended approach + +Document your analysis and deployment recommendation in `/root/report.md`. + +Use available tools to examine the environment. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-developer__detect-project/solution/solve.sh b/evaluation/with_skills/rh-developer__detect-project/solution/solve.sh new file mode 100644 index 00000000..700e7ad4 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/solution/solve.sh @@ -0,0 +1,37 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Project Detection Report + +## Repository: /root/project + +### Detection Methodology +Scanned for indicator files: requirements.txt, package.json, pom.xml, go.mod, Gemfile. +Found: `requirements.txt` → Python project. + +### Detected Type +- **Language**: Python +- **Indicator**: `requirements.txt` found +- **Framework**: Flask (detected from `from flask import Flask` in app.py) +- **Entry Point**: `app.py` with `app = Flask(__name__)` + +### Helm Chart Search +Searched locations: ./Chart.yaml, ./chart/Chart.yaml, ./charts/*/Chart.yaml, ./helm/Chart.yaml, ./deploy/helm/Chart.yaml +Result: No Helm chart found — S2I or Dockerfile strategy recommended. + +### S2I Python Configuration +- **APP_MODULE**: `app:app` (module `app` from `app.py`, WSGI callable `app`) +- **gunicorn** is present in `requirements.txt` — required for the S2I Python builder to serve via APP_MODULE +- S2I Python builder uses gunicorn as the WSGI server when APP_MODULE is set + +### Recommended Builder Image +`registry.access.redhat.com/ubi9/python-39` (UBI base image) + +### Health Checks +- Add `/health` and `/ready` endpoints for OpenShift liveness/readiness probes + +### Recommended Deployment Strategy +1. **Primary**: S2I with `ubi9/python-39` builder image + - Set `APP_MODULE=app:app` in BuildConfig sourceStrategy.env + - Ensure gunicorn is in requirements.txt +2. **Alternative**: Containerize with Dockerfile using UBI base image +REPORT_EOF diff --git a/evaluation/with_skills/rh-developer__detect-project/task.toml b/evaluation/with_skills/rh-developer__detect-project/task.toml new file mode 100644 index 00000000..78be6504 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__detect-project" +name = "rh-developer Project Detection Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "detect-project", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-developer__detect-project/tests/llm_judge.py b/evaluation/with_skills/rh-developer__detect-project/tests/llm_judge.py new file mode 100644 index 00000000..67b69834 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/tests/llm_judge.py @@ -0,0 +1,102 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "s2i_entry_point_sequence", + "file": "/root/report.md", + "question": "Does the report describe the S2I Python builder's entry point detection order — specifically mentioning that the builder checks for files like app.sh before falling back to app.py, and how app.py being the default entry point affects startup?", + "reference": "A skilled report describes the S2I Python startup sequence (check app.sh first, then application.py, then app.py) and explains that since app.py is found, gunicorn will serve it automatically. An unskilled report mentions app.py as the entry point without describing the detection sequence the builder follows." + }, + { + "id": "app_module_gunicorn_link", + "file": "/root/report.md", + "question": "Does the report explain the connection between gunicorn in requirements.txt and APP_MODULE configuration for the S2I Python builder — specifically that gunicorn is required for APP_MODULE to work?", + "reference": "A skilled report connects gunicorn to APP_MODULE, explaining that the S2I Python builder needs gunicorn in requirements.txt to serve the app specified by APP_MODULE (e.g., app:app). An unskilled report mentions gunicorn as a generic web server without connecting it to S2I builder mechanics." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-developer__detect-project/tests/test.sh b/evaluation/with_skills/rh-developer__detect-project/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-developer__detect-project/tests/test_outputs.py b/evaluation/with_skills/rh-developer__detect-project/tests/test_outputs.py new file mode 100644 index 00000000..3da3a2dc --- /dev/null +++ b/evaluation/with_skills/rh-developer__detect-project/tests/test_outputs.py @@ -0,0 +1,79 @@ +""" +Tests for rh-developer__detect-project per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_project_or_language(self): + content = read_report().lower() + assert any(t in content for t in ["project", "language", "framework", "detect"]), ( + "report should mention project detection" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 100, "report should have substantial content" + + +class TestSkillDependent: + def test_s2i_deployment_recommendation(self): + """Skill teaches S2I as preferred deployment for OpenShift.""" + c = read_report().lower() + assert "s2i" in c or "source-to-image" in c or "source to image" in c, ( + "should recommend S2I as deployment strategy for OpenShift" + ) + + def test_app_module_format(self): + """Skill teaches APP_MODULE format 'module:callable' (e.g., app:app) for + S2I Python. Without skill, agents don't know this configuration.""" + c = read_report().lower() + assert "app_module" in c and any(t in c for t in [ + "app:app", "module:", ":app", "module:callable", "wsgi", + ]), "should specify APP_MODULE format (e.g., app:app) for S2I Python" + + def test_gunicorn_s2i_link(self): + """Skill teaches gunicorn is required IN requirements.txt for the S2I + Python builder to use APP_MODULE. Without skill, agents mention gunicorn + generically without connecting it to S2I builder requirements.""" + c = read_report().lower() + assert "gunicorn" in c and ("s2i" in c or "app_module" in c or "builder" in c), ( + "should connect gunicorn to S2I/APP_MODULE (not just as a generic server)" + ) + + def test_ubi_base_image_recommendation(self): + """Skill teaches UBI as the base image for OpenShift.""" + c = read_report().lower() + assert "ubi" in c or "universal base image" in c, ( + "should recommend UBI base image for OpenShift deployment" + ) + + def test_s2i_entry_point_detection(self): + """Skill teaches the S2I Python entry point detection order + (app.sh → application.py → app.py). Without skill, agents don't + describe the builder's startup sequence.""" + c = read_report().lower() + has_sequence = "app.sh" in c + has_default_entry = ("default" in c or "entry point" in c) and "app.py" in c + has_startup = any(t in c for t in [ + "startup logic", "startup sequence", "s2i startup", + "entry point detection", "entry point order", + ]) + assert has_sequence or has_default_entry or has_startup, ( + "should describe S2I Python entry point detection (app.sh/app.py sequence)" + ) diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/Dockerfile b/evaluation/with_skills/rh-developer__helm-deploy/environment/Dockerfile new file mode 100644 index 00000000..f0cfbbda --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/Dockerfile @@ -0,0 +1,74 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "helm": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-helm-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/mcp-servers/mock-helm-mcp.py b/evaluation/with_skills/rh-developer__helm-deploy/environment/mcp-servers/mock-helm-mcp.py new file mode 100644 index 00000000..8909ad01 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/mcp-servers/mock-helm-mcp.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +""" +Mock Helm MCP Server for rh-developer helm-deploy benchmark task. + +Simulates Helm CLI operations for OpenShift deployment planning. +""" + +from typing import Optional + +from fastmcp import FastMCP + +mcp = FastMCP("helm") + +# Mock data for existing releases +MOCK_RELEASES = [ + { + "name": "api-service", + "namespace": "api-platform", + "revision": 3, + "updated": "2026-02-15T10:30:00Z", + "status": "deployed", + "chart": "api-service-1.2.0", + "app_version": "1.0.0", + }, + { + "name": "web-frontend", + "namespace": "web-frontend", + "revision": 1, + "updated": "2026-02-14T14:20:00Z", + "status": "deployed", + "chart": "web-frontend-0.1.0", + "app_version": "1.0.0", + }, +] + +MOCK_CHART_METADATA = { + "name": "my-app", + "version": "0.1.0", + "appVersion": "1.0.0", + "description": "OpenShift deployment chart for my-app", + "keywords": ["openshift", "deployment"], + "maintainers": [{"name": "Red Hat", "email": "openshift@redhat.com"}], +} + +MOCK_DEFAULT_VALUES = """replicaCount: 1 + +image: + repository: quay.io/example/my-app + tag: latest + pullPolicy: IfNotPresent + +service: + type: ClusterIP + port: 8080 + +route: + enabled: true + host: "" + +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 256Mi +""" + +MOCK_RENDERED_YAML = """--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-app + labels: + app: my-app +spec: + replicas: 1 + selector: + matchLabels: + app: my-app + template: + metadata: + labels: + app: my-app + spec: + containers: + - name: my-app + image: quay.io/example/my-app:latest + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: my-app +spec: + ports: + - port: 8080 + targetPort: 8080 + selector: + app: my-app +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: my-app +spec: + to: + kind: Service + name: my-app + port: + targetPort: 8080 +""" + + +@mcp.tool +def helm_list(namespace: str) -> dict: + """List installed Helm releases in a namespace. + + Args: + namespace: The Kubernetes/OpenShift namespace to list releases from. + """ + releases = [r for r in MOCK_RELEASES if r["namespace"] == namespace] + return { + "releases": releases, + "count": len(releases), + "namespace": namespace, + } + + +@mcp.tool +def helm_show_chart(chart: str) -> dict: + """Show chart metadata (name, version, description). + + Args: + chart: Path to chart directory or chart name (e.g. ./chart or my-chart). + """ + return { + "chart": chart, + "metadata": MOCK_CHART_METADATA, + } + + +@mcp.tool +def helm_show_values(chart: str) -> dict: + """Show default values for a chart. + + Args: + chart: Path to chart directory or chart name. + """ + return { + "chart": chart, + "values": MOCK_DEFAULT_VALUES, + } + + +@mcp.tool +def helm_template( + release_name: str, + chart: str, + namespace: str, + values: Optional[str] = None, +) -> dict: + """Render chart templates to YAML with given values. + + Args: + release_name: Name for the release. + chart: Path to chart directory. + namespace: Target namespace. + values: Optional YAML string of values to override defaults. + """ + return { + "release_name": release_name, + "chart": chart, + "namespace": namespace, + "rendered": MOCK_RENDERED_YAML, + } + + +@mcp.tool +def helm_install_dry_run( + release_name: str, + chart: str, + namespace: str, + values: Optional[str] = None, +) -> dict: + """Simulate helm install (dry-run) to validate before deploying. + + Args: + release_name: Name for the release. + chart: Path to chart directory. + namespace: Target namespace. + values: Optional YAML string of values to override defaults. + """ + return { + "release_name": release_name, + "chart": chart, + "namespace": namespace, + "dry_run": True, + "status": "would_create", + "resources": ["Deployment/my-app", "Service/my-app", "Route/my-app"], + } + + +@mcp.tool +def helm_status(release_name: str, namespace: str) -> dict: + """Get status of an installed Helm release. + + Args: + release_name: Name of the release. + namespace: The namespace where the release is installed. + """ + release = next( + (r for r in MOCK_RELEASES if r["name"] == release_name and r["namespace"] == namespace), + None, + ) + if release: + return { + "release": release_name, + "namespace": namespace, + "status": release, + } + return { + "release": release_name, + "namespace": namespace, + "error": f"Release '{release_name}' not found in namespace '{namespace}'", + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-developer__helm-deploy/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/skills/helm-deploy/SKILL.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/skills/helm-deploy/SKILL.md new file mode 100644 index 00000000..a44ce170 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/skills/helm-deploy/SKILL.md @@ -0,0 +1,356 @@ +--- +name: helm-deploy +description: | + Deploy applications to OpenShift using Helm charts. Use this skill when user wants to deploy with Helm, when a Helm chart is detected in the project, or when /helm-deploy command is invoked. Supports both existing charts and chart creation. Handles chart detection, values customization, install/upgrade operations, and rollback. Requires kubernetes MCP Helm tools. +model: inherit +color: green +metadata: + user_invocable: "true" +--- + +# /helm-deploy Skill + +Deploy applications to OpenShift using Helm charts. Supports existing charts or creates new ones. + +## Prerequisites + +1. User logged into OpenShift cluster +2. Helm chart exists OR user wants to create one +3. Container image available (from registry or will be built) + +## When to Use This Skill + +- User wants to deploy an application using Helm charts on OpenShift +- A Helm chart is detected in the project (Chart.yaml found) +- User invokes `/helm-deploy` or asks about Helm-based deployment + +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +## Workflow + +### Step 1: Check OpenShift Connection + +Use kubernetes MCP to verify cluster connection: + +```markdown +## Checking OpenShift Connection... + +**Cluster:** [cluster-url] +**User:** [username] +**Namespace:** [namespace] + +Is this the correct cluster and namespace? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Step 2: Detect Helm Chart + +Search for Helm charts using the same priority order as `/detect-project`: +- `./Chart.yaml`, `./chart/Chart.yaml`, `./charts/*/Chart.yaml`, `./helm/Chart.yaml`, `./deploy/helm/Chart.yaml` + +> **Note:** If `/detect-project` was already run, use the `HELM_CHART_PATH` and `HELM_CHART_DETECTED` values from session state. + +**If chart found:** + +```markdown +## Helm Chart Detected + +**Location:** [chart-path] + +| Field | Value | +|-------|-------| +| Name | [chart-name] | +| Version | [chart-version] | +| App Version | [app-version] | +| Description | [description] | + +**Templates found:** +- [list of template files] + +**Values file:** [values.yaml path] + +Would you like to: +1. Deploy using this chart (recommended) +2. Customize values before deploying +3. Use a different chart location +``` + +Proceed automatically without waiting for user input. + +**If no chart found:** + +```markdown +## No Helm Chart Found + +I searched these locations but found no Helm chart: +- ./Chart.yaml +- ./chart/Chart.yaml +- ./charts/*/Chart.yaml +- ./helm/Chart.yaml +- ./deploy/helm/Chart.yaml + +**Options:** +1. **Create a new Helm chart** - I'll generate one based on your project +2. **Specify chart path** - Point me to your chart location +3. **Use a different deployment method** - Try /deploy or /containerize-deploy + +Which would you prefer? +``` + +Proceed automatically without waiting for user input. + +### Step 3: Create Helm Chart (if needed) + +If user chooses to create a chart: + +```markdown +## Creating Helm Chart + +I'll create a Helm chart based on your project. + +**Detected Project Info:** +| Setting | Value | +|---------|-------| +| App Name | [app-name] | +| Language | [language] | +| Framework | [framework] | +| Port | [port] | + +**Chart will include:** +- Chart.yaml with project metadata +- values.yaml with configurable options +- Deployment template +- Service template +- Route template (OpenShift) +- Helper templates + +**Target directory:** ./chart/ + +Proceed with creating the Helm chart? (yes/no) +``` + +Proceed automatically without waiting for user input. + +Use templates from templates/helm/ to generate: +1. Chart.yaml +2. values.yaml +3. templates/deployment.yaml +4. templates/service.yaml +5. templates/route.yaml +6. templates/_helpers.tpl +7. templates/NOTES.txt + +Replace `${APP_NAME}` placeholders with actual app name in all template files. + +### Step 4: Check for Existing Release + +Before installing, check if a release with the same name exists: + +```markdown +## Checking for Existing Release... + +[Use helm_list to check] +``` + +**If release exists:** + +```markdown +## Existing Release Found + +A release named '[name]' already exists. + +| Field | Value | +|-------|-------| +| Status | [status] | +| Revision | [revision] | +| Chart | [chart-name] v[version] | +| Updated | [timestamp] | + +**Options:** +1. Upgrade the release with new configuration +2. Rollback to a previous revision +3. Uninstall and reinstall +4. Cancel + +Which would you like to do? +``` + +Proceed automatically without waiting for user input. + +### Step 5: Review Values + +```markdown +## Chart Values Configuration + +**Current values.yaml:** + +```yaml +replicaCount: [value] +image: + repository: [value] + tag: [value] +service: + port: [value] +route: + enabled: [value] +resources: + limits: + memory: [value] +``` + +**Common customizations:** + +| Value | Current | Description | +|-------|---------|-------------| +| `replicaCount` | 1 | Number of pods | +| `image.repository` | [repo] | Container image | +| `image.tag` | [tag] | Image version | +| `service.port` | [port] | Service port | +| `resources.limits.memory` | 512Mi | Memory limit | + +**Options:** +1. Deploy with current values +2. Modify values interactively +3. Use a custom values file + +Which would you prefer? +``` + +Proceed automatically without waiting for user input. + +### Step 6: Pre-Deploy Summary + +```markdown +## Helm Deployment Summary + +**Release Configuration:** + +| Setting | Value | +|---------|-------| +| Release Name | [release-name] | +| Namespace | [namespace] | +| Chart | [chart-path] | +| Chart Version | [version] | + +**Resources to be created:** + +| Resource | Name | +|----------|------| +| Deployment | [name] | +| Service | [name] | +| Route | [name] (if enabled) | + +**Values to apply:** +```yaml +[show customized values or "Using defaults"] +``` + +**Helm command equivalent:** +```bash +helm install [release-name] [chart-path] -n [namespace] [--set options] +``` + +**Proceed with Helm deployment?** (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Step 7: Execute Deployment + +Use kubernetes MCP `helm_install` or `helm_upgrade`: + +```markdown +## Deploying with Helm... + +**Release:** [release-name] +**Chart:** [chart-name] v[version] + +[x] Chart validated +[x] Templates rendered +[ ] Installing release... + +--- + +**Installation Progress:** + +Waiting for resources to be ready... + +| Resource | Status | +|----------|--------| +| Deployment/[name] | [status] | +| Service/[name] | [status] | +| Route/[name] | [status] | + +--- +``` + +Monitor pod status using `pods_list_in_namespace` until pods are ready or timeout. + +### Step 8: Deployment Complete + +```markdown +## Helm Deployment Complete! + +**Release:** [release-name] +**Status:** deployed +**Revision:** 1 +**Namespace:** [namespace] + +--- + +**Resources Created:** + +| Resource | Name | Status | +|----------|------|--------| +| Deployment | [name] | [replicas] Ready | +| Service | [name] | Active | +| Route | [name] | Admitted | + +**Access URL:** https://[route-host] + +--- + +**Quick Commands:** + +```bash +# Check release status +helm status [release-name] -n [namespace] + +# View release history +helm history [release-name] -n [namespace] + +# Upgrade with new values +helm upgrade [release-name] [chart-path] -n [namespace] -f new-values.yaml + +# Rollback to previous version +helm rollback [release-name] 1 -n [namespace] + +# Uninstall release +helm uninstall [release-name] -n [namespace] + +# View logs +oc logs -l app.kubernetes.io/instance=[release-name] -n [namespace] -f +``` + +--- + +Your application is live! +``` + +## Dependencies + +### Required MCP Servers +- `openshift` - Helm install, upgrade, list, and uninstall operations + +### Related Skills +- `/deploy` - Alternative deployment without Helm charts +- `/debug-pod` - Troubleshoot pods after Helm deployment +- `/debug-network` - Diagnose networking issues with deployed services + +### Reference Documentation +- [docs/builder-images.md](../../docs/builder-images.md) - Container image references for chart values +- [docs/image-selection-criteria.md](../../docs/image-selection-criteria.md) - Image variant selection for production deployments +- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (oc, helm) diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/buildconfig.yaml.template b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/buildconfig.yaml.template new file mode 100644 index 00000000..b3294eb2 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/buildconfig.yaml.template @@ -0,0 +1,38 @@ +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: build + app.kubernetes.io/part-of: ${APP_NAME} +spec: + source: + type: Git + git: + uri: ${GIT_URL} + ref: ${GIT_BRANCH} + strategy: + type: Source + sourceStrategy: + from: + kind: DockerImage + name: ${BUILDER_IMAGE} + env: [] + output: + to: + kind: ImageStreamTag + name: ${APP_NAME}:latest + triggers: + - type: ConfigChange + - type: ImageChange + runPolicy: Serial + resources: + limits: + memory: "1Gi" + cpu: "1" + requests: + memory: "512Mi" + cpu: "500m" diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/deployment.yaml.template b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/deployment.yaml.template new file mode 100644 index 00000000..eb3b481a --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/deployment.yaml.template @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: application + app.kubernetes.io/part-of: ${APP_NAME} + annotations: + image.openshift.io/triggers: | + [{"from":{"kind":"ImageStreamTag","name":"${APP_NAME}:latest"},"fieldPath":"spec.template.spec.containers[0].image"}] +spec: + replicas: ${REPLICAS} + selector: + matchLabels: + app: ${APP_NAME} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + template: + metadata: + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + spec: + containers: + - name: ${APP_NAME} + image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/${APP_NAME}:latest + ports: + - containerPort: ${CONTAINER_PORT} + protocol: TCP + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + livenessProbe: + httpGet: + path: / + port: ${CONTAINER_PORT} + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: / + port: ${CONTAINER_PORT} + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + env: [] + restartPolicy: Always + terminationGracePeriodSeconds: 30 diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/Chart.yaml.template b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/Chart.yaml.template new file mode 100644 index 00000000..1aa22dd1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/Chart.yaml.template @@ -0,0 +1,13 @@ +apiVersion: v2 +name: ${APP_NAME} +description: ${APP_DESCRIPTION} +type: application +version: 0.1.0 +appVersion: "${APP_VERSION}" +keywords: + - ${LANGUAGE} + - ${FRAMEWORK} + - openshift +maintainers: + - name: ${MAINTAINER_NAME} + email: ${MAINTAINER_EMAIL} diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/NOTES.txt.template b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/NOTES.txt.template new file mode 100644 index 00000000..154e628d --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/NOTES.txt.template @@ -0,0 +1,32 @@ +Congratulations! Your application {{ include "${APP_NAME}.fullname" . }} has been deployed. + +{{- if .Values.route.enabled }} + +Access your application at: +{{- if .Values.route.host }} + https://{{ .Values.route.host }} +{{- else }} + Run: oc get route {{ include "${APP_NAME}.fullname" . }} -o jsonpath='{.spec.host}' +{{- end }} + +{{- else }} + +Your application is available internally at: + {{ include "${APP_NAME}.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }} + +To expose it externally, create a Route or set route.enabled=true. + +{{- end }} + +Useful commands: + # View pods + oc get pods -l app.kubernetes.io/name={{ include "${APP_NAME}.name" . }} + + # View logs + oc logs -l app.kubernetes.io/name={{ include "${APP_NAME}.name" . }} -f + + # Upgrade release + helm upgrade {{ .Release.Name }} ./{{ .Chart.Name }} -f values.yaml + + # Uninstall release + helm uninstall {{ .Release.Name }} diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/_helpers.tpl.template b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/_helpers.tpl.template new file mode 100644 index 00000000..15873b10 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/_helpers.tpl.template @@ -0,0 +1,60 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "${APP_NAME}.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "${APP_NAME}.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "${APP_NAME}.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "${APP_NAME}.labels" -}} +helm.sh/chart: {{ include "${APP_NAME}.chart" . }} +{{ include "${APP_NAME}.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "${APP_NAME}.selectorLabels" -}} +app.kubernetes.io/name: {{ include "${APP_NAME}.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "${APP_NAME}.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "${APP_NAME}.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/deployment.yaml.template b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/deployment.yaml.template new file mode 100644 index 00000000..a6cbd868 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/deployment.yaml.template @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "${APP_NAME}.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "${APP_NAME}.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "${APP_NAME}.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/route.yaml.template b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/route.yaml.template new file mode 100644 index 00000000..e2bab29a --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/route.yaml.template @@ -0,0 +1,24 @@ +{{- if .Values.route.enabled }} +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + {{- if .Values.route.host }} + host: {{ .Values.route.host }} + {{- end }} + to: + kind: Service + name: {{ include "${APP_NAME}.fullname" . }} + weight: 100 + port: + targetPort: http + {{- with .Values.route.tls }} + tls: + termination: {{ .termination }} + insecureEdgeTerminationPolicy: {{ .insecureEdgeTerminationPolicy }} + {{- end }} + wildcardPolicy: None +{{- end }} diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/service.yaml.template b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/service.yaml.template new file mode 100644 index 00000000..837bc888 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/templates/service.yaml.template @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "${APP_NAME}.selectorLabels" . | nindent 4 }} diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/values.yaml.template b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/values.yaml.template new file mode 100644 index 00000000..1cca6017 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/helm/values.yaml.template @@ -0,0 +1,67 @@ +# Default values for ${APP_NAME} +replicaCount: 1 + +image: + repository: ${IMAGE_REPOSITORY} + pullPolicy: IfNotPresent + tag: "${IMAGE_TAG}" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: "" + +podAnnotations: {} +podSecurityContext: {} +securityContext: {} + +service: + type: ClusterIP + port: ${CONTAINER_PORT} + +route: + enabled: true + host: "" + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + +resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + +livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + +readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + +nodeSelector: {} +tolerations: [] +affinity: {} + +env: [] +# - name: MY_VAR +# value: "my-value" diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/imagestream.yaml.template b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/imagestream.yaml.template new file mode 100644 index 00000000..46572193 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/imagestream.yaml.template @@ -0,0 +1,13 @@ +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: image + app.kubernetes.io/part-of: ${APP_NAME} +spec: + lookupPolicy: + local: false diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/route.yaml.template b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/route.yaml.template new file mode 100644 index 00000000..7c53d2e7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/route.yaml.template @@ -0,0 +1,21 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: route + app.kubernetes.io/part-of: ${APP_NAME} +spec: + to: + kind: Service + name: ${APP_NAME} + weight: 100 + port: + targetPort: http + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + wildcardPolicy: None diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/service.yaml.template b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/service.yaml.template new file mode 100644 index 00000000..7e1cf371 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/service.yaml.template @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: service + app.kubernetes.io/part-of: ${APP_NAME} +spec: + selector: + app: ${APP_NAME} + ports: + - name: http + port: ${CONTAINER_PORT} + targetPort: ${CONTAINER_PORT} + protocol: TCP + type: ClusterIP + sessionAffinity: None diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-container-rootful.service b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-container-rootful.service new file mode 100644 index 00000000..c1e8fe8f --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-container-rootful.service @@ -0,0 +1,27 @@ +# Rootful Podman container managed by systemd (system service) +# Location: /etc/systemd/system/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${PORT} - Port number (used for both host and container binding) +# ${IMAGE} - Container image reference + +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run --name ${APP_NAME} \ + -p ${PORT}:${PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-container-rootless.service b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-container-rootless.service new file mode 100644 index 00000000..ca9dc371 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-container-rootless.service @@ -0,0 +1,27 @@ +# Rootless Podman container managed by systemd (user service) +# Location: ~/.config/systemd/user/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${PORT} - Port number (used for both host and container binding) +# ${IMAGE} - Container image reference + +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run --name ${APP_NAME} \ + -p ${PORT}:${PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-native.service b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-native.service new file mode 100644 index 00000000..c55cfc07 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-native.service @@ -0,0 +1,39 @@ +# Native application managed by systemd (system service) +# Location: /etc/systemd/system/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${SERVICE_USER} - User to run the service as +# ${APP_PATH} - Application install path (e.g., /opt/app-name) +# ${PORT} - Application listen port +# ${START_COMMAND} - Application start command +# +# Start command examples by language: +# Node.js: /usr/bin/node ${APP_PATH}/server.js +# Python: /usr/bin/python3 ${APP_PATH}/app.py +# Java: /usr/bin/java -jar ${APP_PATH}/app.jar +# Go: ${APP_PATH}/binary-name + +[Unit] +Description=${APP_NAME} Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=${APP_PATH} +Environment=PORT=${PORT} +ExecStart=${START_COMMAND} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=${APP_PATH} + +[Install] +WantedBy=multi-user.target diff --git a/evaluation/with_skills/rh-developer__helm-deploy/instruction.md b/evaluation/with_skills/rh-developer__helm-deploy/instruction.md new file mode 100644 index 00000000..5ea35a0f --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/instruction.md @@ -0,0 +1,12 @@ +# Helm Deployment Task + +You are a Red Hat developer. Plan the deployment of an application using Helm charts on OpenShift. + +## Requirements +- Evaluate or create a Helm chart structure +- Configure values for the target environment +- Address OpenShift-specific considerations + +Use MCP tools to examine the cluster. Document your methodology, chart configuration, and deployment plan in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-developer__helm-deploy/solution/solve.sh b/evaluation/with_skills/rh-developer__helm-deploy/solution/solve.sh new file mode 100644 index 00000000..caf0f768 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/solution/solve.sh @@ -0,0 +1,31 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Helm Deployment Plan + +## Chart Location +Searched: ./Chart.yaml, ./chart/Chart.yaml, ./charts/*/Chart.yaml, ./helm/Chart.yaml +Found: `./chart/Chart.yaml` + +## Values Override +```yaml +replicaCount: 2 +image: + repository: image-registry.openshift-image-registry.svc:5000/myproject/myapp + tag: latest +service: + port: 8080 +resources: + limits: + memory: 512Mi +``` + +## Deploy Command +```bash +helm install myapp ./chart/ -f values-override.yaml -n myproject +``` + +## Quick Commands +helm status myapp -n myproject +helm history myapp -n myproject +helm rollback myapp 1 -n myproject +REPORT_EOF diff --git a/evaluation/with_skills/rh-developer__helm-deploy/task.toml b/evaluation/with_skills/rh-developer__helm-deploy/task.toml new file mode 100644 index 00000000..89f35c82 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__helm-deploy" +name = "rh-developer Helm Deployment Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "helm-deploy", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-developer__helm-deploy/tests/llm_judge.py b/evaluation/with_skills/rh-developer__helm-deploy/tests/llm_judge.py new file mode 100644 index 00000000..5632c542 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/tests/llm_judge.py @@ -0,0 +1,102 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "openshift_helm_considerations", + "file": "/root/report.md", + "question": "Does the report address OpenShift-specific Helm concerns like Route vs Ingress and SecurityContextConstraints?", + "reference": "A skilled report addresses that OpenShift uses Routes and has SCC requirements that may affect Helm charts designed for vanilla Kubernetes. An unskilled report treats the chart as platform-agnostic." + }, + { + "id": "buildconfig_in_chart", + "file": "/root/report.md", + "question": "Does the report describe including an OpenShift BuildConfig template as part of the Helm chart structure, so that the chart manages the build pipeline alongside the deployment?", + "reference": "A skilled report includes a BuildConfig YAML template inside the Helm chart (e.g., templates/buildconfig.yaml) for S2I builds. An unskilled report assumes pre-built images and does not integrate build pipelines into the chart." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-developer__helm-deploy/tests/test.sh b/evaluation/with_skills/rh-developer__helm-deploy/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-developer__helm-deploy/tests/test_outputs.py b/evaluation/with_skills/rh-developer__helm-deploy/tests/test_outputs.py new file mode 100644 index 00000000..2f4af59c --- /dev/null +++ b/evaluation/with_skills/rh-developer__helm-deploy/tests/test_outputs.py @@ -0,0 +1,61 @@ +""" +Tests for rh-developer__helm-deploy per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: OpenShift-Helm integration (not generic Helm knowledge). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_helm(self): + content = read_report().lower() + assert "helm" in content, "report should mention Helm" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 100, "report should have substantial content" + + +class TestSkillDependent: + def test_values_customization(self): + """Customizing values before deployment.""" + c = read_report().lower() + assert any(t in c for t in ["values", "override", "set", "customize"]) and any(t in c for t in [ + "install", "upgrade", "deploy" + ]), "should address values customization" + + def test_openshift_considerations(self): + """OpenShift-specific Helm considerations (Route, SCC).""" + c = read_report().lower() + assert any(t in c for t in ["openshift", "route", "scc", "security"]), ( + "should address OpenShift-specific Helm concerns" + ) + + def test_buildconfig_integration(self): + """OpenShift BuildConfig integration in Helm charts for S2I builds. + Without skill, agents use static image references.""" + c = read_report() + assert "BuildConfig" in c or "buildconfig" in c.lower() or "build.openshift.io" in c, ( + "should address OpenShift BuildConfig integration in Helm deployment" + ) + + def test_s2i_in_helm_chart(self): + """OpenShift S2I build integration as part of the Helm chart, + so the chart manages both the build and deploy lifecycle.""" + c = read_report().lower() + assert ("s2i" in c or "source-to-image" in c or "source to image" in c) and ( + "helm" in c or "chart" in c or "template" in c + ), "should integrate S2I builds within the Helm chart structure" diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/Dockerfile b/evaluation/with_skills/rh-developer__recommend-image/environment/Dockerfile new file mode 100644 index 00000000..1cbfefcf --- /dev/null +++ b/evaluation/with_skills/rh-developer__recommend-image/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-developer__recommend-image/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/with_skills/rh-developer__recommend-image/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/skills/recommend-image/SKILL.md b/evaluation/with_skills/rh-developer__recommend-image/environment/skills/recommend-image/SKILL.md new file mode 100644 index 00000000..d5c81d44 --- /dev/null +++ b/evaluation/with_skills/rh-developer__recommend-image/environment/skills/recommend-image/SKILL.md @@ -0,0 +1,282 @@ +--- +name: recommend-image +description: | + Intelligently recommend the optimal S2I builder image or container base image for a project based on detected language/framework, use-case requirements, security posture, and deployment target. Supports GitHub URLs for remote project analysis (delegates to /detect-project). Use this skill when the user needs a container image recommendation, wants to compare image options, or asks about production vs development images. Triggers on /recommend-image command, or when advanced image selection beyond basic version matching is needed. Supports Node.js, Python, Java, Go, Ruby, .NET, PHP, and Perl on Red Hat UBI. +model: inherit +color: cyan +metadata: + user_invocable: "true" +--- + +# /recommend-image Skill + +Provide intelligent, use-case-aware container image recommendations that go beyond simple language-to-image mapping. + +## When to Use This Skill + +- User asks for the "best" image for their use case +- User needs to choose between production vs development images +- User wants to compare image options (minimal vs full-featured) +- `/detect-project` completed and user wants a tailored recommendation +- User asks about image size, security, or performance trade-offs + +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +## Workflow + +### Step 1: Gather Context + +**If invoked after `/detect-project`:** +Use the already-detected values: +- `LANGUAGE` - Programming language +- `FRAMEWORK` - Framework (if detected) +- `VERSION` - Language version + +**If invoked with a GitHub URL:** + +Example: `/recommend-image for https://github.com/RHEcosystemAppEng/sast-ai-frontend` + +When a GitHub URL is provided: + +```markdown +## Analyzing Remote Repository + +I'll analyze the repository to detect the project type first. + +Invoking `/detect-project` for: `[github-url]` +``` + +**Delegate to `/detect-project`:** +- Pass the GitHub URL to `/detect-project` +- `/detect-project` will use GitHub MCP to analyze the repository +- Receive back: `LANGUAGE`, `FRAMEWORK`, `VERSION`, `APP_NAME` +- Continue to Step 2 (Use-Case Assessment) + +**If invoked standalone (no URL, no prior detection):** +Ask the user: + +```markdown +## Image Recommendation + +To recommend the best image, I need some information: + +**Option 1:** Provide a GitHub URL +- Example: `/recommend-image for https://github.com/user/repo` + +**Option 2:** Tell me about your project +1. **What language/framework is your project?** + (e.g., Python 3.11, Node.js 20, Java 17 with Spring Boot) + +2. **What version do you need?** + (or say "latest LTS" if unsure) +``` + +### Step 2: Use-Case Assessment + +Present use-case questions: + +```markdown +## Use-Case Assessment + +To recommend the optimal image, please tell me about your requirements: + +**1. Deployment Environment:** +- **Production** - Stability, security, long-term support critical +- **Development** - Tooling, debugging features preferred +- **Edge/IoT** - Minimal footprint essential + +**2. Security Priority:** +- **Standard** - Red Hat UBI with regular updates +- **Hardened** - Minimal attack surface, fewer packages +- **Compliance** - FIPS or specific compliance requirements + +**3. Performance Priority:** +- **Fast startup** - Serverless, scale-to-zero workloads +- **Low memory** - High-density deployments +- **Balanced** - General purpose applications + +**4. Build Requirements:** +- **Need build tools** - Native extensions, compilation during build +- **Runtime only** - Pre-compiled, no build tools needed + +Please describe your use case or select from the options above. +``` + +Proceed automatically without waiting for user input. + +### Step 3: Evaluate Image Options + +For each language, evaluate available variants against user requirements. + +**Image Variants:** Full (build tools), Minimal (smaller, secure), Runtime (smallest, pre-compiled only) + +**Key Scoring Factors:** Image size, security posture, build tools availability, startup time, LTS status + +> **See [docs/image-selection-criteria.md](../../docs/image-selection-criteria.md)** for comprehensive scoring matrices with weighted criteria by environment (production/development/edge/serverless). + +### Step 3.5: Dynamic Image Validation + +Before presenting recommendations, validate with dynamic sources to provide accurate, real-time data. + +#### Check if Skopeo is Available + +First, verify skopeo is installed: + +```bash +which skopeo +``` + +**If skopeo is NOT installed**, present: + +```markdown +## Skopeo Required for Image Validation + +To provide accurate image recommendations, I need `skopeo` to inspect container images. + +**Skopeo is not installed.** This tool allows me to: +- Verify the image exists before recommending it +- Get exact image size (not estimates) +- Check architecture support (amd64, arm64) +- Show when the image was last built + +**Install skopeo:** See [docs/prerequisites.md](../../docs/prerequisites.md) for installation commands by OS. + +After installing, run `/recommend-image` again for enhanced recommendations. + +**Continue without skopeo?** +- **yes** - Use static reference data only (less accurate) +- **install** - I'll install skopeo first +``` + +Proceed automatically without waiting for user input. + +If user continues without skopeo, proceed with static data and note: "Image metadata from static reference (not verified)". + +#### Skopeo Verification + +For each candidate image, verify availability and get metadata: + +```bash +# Verify image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/[candidate-image] +``` + +**Note:** The `docker://` transport is OCI-standard and works with Podman registries - it's not Docker-specific. + +### Step 4: Present Recommendation + +Format your recommendation: + +```markdown +## Image Recommendation + +Based on your requirements: + +| Factor | Your Input | +|--------|------------| +| Language | [language] [version] | +| Framework | [framework or "None"] | +| Environment | [Production/Development/Edge] | +| Security | [Standard/Hardened/Compliance] | +| Priority | [startup/memory/balanced] | +| Build Tools | [needed/not needed] | + +--- + +### Recommended Image + +`registry.access.redhat.com/ubi9/[image-name]` + +**Why this image:** +- [Reason 1 - matches primary requirement] +- [Reason 2 - matches secondary requirement] +- [Reason 3 - version/LTS consideration] + +**Image Details:** +| Property | Value | Source | +|----------|-------|--------| +| Base | UBI 9 | Static | +| Variant | [Full/Minimal/Runtime] | Static | +| Size | [exact-size]MB | Skopeo | +| Built | [build-date] | Skopeo | +| Architecture | amd64, arm64 | Skopeo | +| LTS | [Yes/No - EOL date] | Static | + +**Security Status:** [status-icon] [status-message] +- Last checked: [timestamp] +- Source: Red Hat Security Data API + +*(If skopeo unavailable: "Image metadata from static reference - install skopeo for verified data")* + +**Trade-offs:** +- [What you give up with this choice] +- [When you might choose differently] + +--- + +### Alternative Options + +| Image | Best For | Trade-off | +|-------|----------|-----------| +| `[alternative-1]` | [use case] | [trade-off] | +| `[alternative-2]` | [use case] | [trade-off] | + +--- + +**Confirm this recommendation?** +- Type **yes** to use `[recommended-image]` +- Type **alternative N** to use an alternative +- Tell me if you have different requirements +``` + +Proceed automatically without waiting for user input. + +### Step 5: Handle Confirmation + +**If user confirms:** + +```markdown +## Image Selected + +| Setting | Value | +|---------|-------| +| Builder Image | `[full-image-reference]` | +| Variant | [variant] | +| Rationale | [brief reason] | + +Configuration saved. You can now: +- Run `/s2i-build` to build with this image +- Run `/containerize-deploy` for the full workflow +``` + +**If user selects alternative:** +Update the selection and confirm. + +**If user has different requirements:** +Return to Step 2 with new inputs. + +## Image Reference + +**Quick variant selection:** +- **Production** → Minimal or Runtime variant +- **Development** → Full variant +- **Serverless** → Smallest available (minimal or native binary) + +> **See [docs/image-selection-criteria.md](../../docs/image-selection-criteria.md)** for comprehensive image size references, LTS timelines, decision trees, and framework-specific recommendations (Quarkus, Spring Boot, Next.js, Django/Flask). + +## Dependencies + +### Required MCP Servers +- None required (uses Bash for skopeo image inspection) + +### Related Skills +- `/detect-project` - Provides language/framework detection input for recommendations +- `/s2i-build` - Build with the recommended image + +### Reference Documentation +- [docs/image-selection-criteria.md](../../docs/image-selection-criteria.md) - Comprehensive scoring matrices, image size reference, LTS timelines, decision trees +- [docs/builder-images.md](../../docs/builder-images.md) - UBI image registry, framework-specific recommendations, variant availability +- [docs/dynamic-validation.md](../../docs/dynamic-validation.md) - Skopeo commands, Red Hat Security Data API, image verification patterns +- [docs/prerequisites.md](../../docs/prerequisites.md) - Skopeo installation instructions diff --git a/evaluation/with_skills/rh-developer__recommend-image/instruction.md b/evaluation/with_skills/rh-developer__recommend-image/instruction.md new file mode 100644 index 00000000..7d5e0138 --- /dev/null +++ b/evaluation/with_skills/rh-developer__recommend-image/instruction.md @@ -0,0 +1,13 @@ +# Image Recommendation Task + +You are a Red Hat developer. Your team is choosing a container base image for a production Python application. The image must be secure, supported, and appropriately sized. + +## Requirements +- Evaluate the available base images that support the application's language and runtime +- Compare at least two candidate images on: security posture (CVE exposure, update cadence), image size, vendor support lifecycle, and compatibility with the application's dependencies +- Recommend a specific image with clear justification for why it is the best fit +- Note any trade-offs or caveats with the recommendation (e.g., larger size for better compatibility) + +Document your analysis and recommendation in `/root/report.md`. + +Use available tools to examine the environment. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-developer__recommend-image/solution/solve.sh b/evaluation/with_skills/rh-developer__recommend-image/solution/solve.sh new file mode 100644 index 00000000..ccbb9f6c --- /dev/null +++ b/evaluation/with_skills/rh-developer__recommend-image/solution/solve.sh @@ -0,0 +1,18 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Image Recommendations + +## Use Case Assessment +Production: prefer Minimal/Runtime. Development: prefer Full variant. + +## 1. Python 3.11 Flask API +**Image**: `registry.access.redhat.com/ubi9/python-311` +**Variant**: Full (build tools needed for pip install) +**Verify**: `skopeo inspect docker://registry.access.redhat.com/ubi9/python-311` + +## 2. Java 17 Quarkus (pre-built JAR) +**Image**: `registry.access.redhat.com/ubi9/openjdk-17-runtime` +**Variant**: Runtime (no build tools, smaller attack surface, faster startup) +**Rationale**: Pre-built JAR doesn't need compilation tools. Runtime variant is ~60% smaller. Security: reduced attack surface. +**Verify**: `skopeo inspect docker://registry.access.redhat.com/ubi9/openjdk-17-runtime` +REPORT_EOF diff --git a/evaluation/with_skills/rh-developer__recommend-image/task.toml b/evaluation/with_skills/rh-developer__recommend-image/task.toml new file mode 100644 index 00000000..2888fbf5 --- /dev/null +++ b/evaluation/with_skills/rh-developer__recommend-image/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__recommend-image" +name = "rh-developer Image Recommendation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "recommend-image", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-developer__recommend-image/tests/llm_judge.py b/evaluation/with_skills/rh-developer__recommend-image/tests/llm_judge.py new file mode 100644 index 00000000..1d03045e --- /dev/null +++ b/evaluation/with_skills/rh-developer__recommend-image/tests/llm_judge.py @@ -0,0 +1,102 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "remote_image_inspection", + "file": "/root/report.md", + "question": "Does the report discuss a remote image inspection approach (such as skopeo inspect docker://) for querying image metadata without pulling the full image?", + "reference": "A skilled report discusses using skopeo or a similar remote inspection approach to verify image metadata (size, architecture, build date) without pulling. If skopeo is unavailable, the report should still mention it as the recommended tool or note that static reference data was used instead. An unskilled report only considers pulling images locally with podman/docker." + }, + { + "id": "variant_tradeoffs", + "file": "/root/report.md", + "question": "Does the report compare at least two image variant categories (e.g., Full/build-tools vs Minimal/secure vs Runtime/smallest) with explicit trade-offs for each?", + "reference": "A skilled report distinguishes image variant categories and explains trade-offs (size vs tools vs security). An unskilled report recommends one image without comparing alternatives." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-developer__recommend-image/tests/test.sh b/evaluation/with_skills/rh-developer__recommend-image/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__recommend-image/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-developer__recommend-image/tests/test_outputs.py b/evaluation/with_skills/rh-developer__recommend-image/tests/test_outputs.py new file mode 100644 index 00000000..00dfabc3 --- /dev/null +++ b/evaluation/with_skills/rh-developer__recommend-image/tests/test_outputs.py @@ -0,0 +1,66 @@ +""" +Tests for rh-developer__recommend-image per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_image(self): + content = read_report().lower() + assert "image" in content, "report should mention container images" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 100, "report should have substantial content" + + +class TestSkillDependent: + def test_remote_image_inspection_approach(self): + """Skill teaches skopeo inspect docker:// for remote image inspection. + Without skill, agents only consider local podman/docker pull.""" + c = read_report().lower() + assert any(t in c for t in [ + "skopeo", "remote inspect", "registry inspect", + "docker://", "image metadata", "without pulling" + ]), "should discuss remote image inspection approach (e.g., skopeo, registry API)" + + def test_image_variant_categories(self): + """Skill teaches three variant categories: Full (build tools), Minimal + (smaller/secure), Runtime (smallest, no build tools). Without skill, + agents don't distinguish these categories.""" + c = read_report().lower() + variants = ["full", "minimal", "runtime"] + mentioned = sum(1 for v in variants if v in c) + assert mentioned >= 2, ( + "should compare image variant categories (Full, Minimal, Runtime)" + ) + + def test_security_data_awareness(self): + """Skill teaches Red Hat Security Data API for CVE/security status per image. + Without skill, agents skip security posture evaluation.""" + c = read_report().lower() + assert any(t in c for t in ["security data", "cve", "vulnerability", "security api"]) and any(t in c for t in [ + "image", "scan", "check", "posture", "red hat" + ]), "should address security/CVE posture for image selection" + + def test_ubi_registry_awareness(self): + """Skill teaches UBI images from registry.access.redhat.com.""" + c = read_report().lower() + assert any(t in c for t in ["ubi", "red hat", "registry"]) and any(t in c for t in [ + "python", "node", "java", "image" + ]), "should recommend UBI images from Red Hat registry" diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/Dockerfile b/evaluation/with_skills/rh-developer__rhel-deploy/environment/Dockerfile new file mode 100644 index 00000000..e5e4879b --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/Dockerfile @@ -0,0 +1,74 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "rhel-host": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-rhel-host-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-developer__rhel-deploy/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/mcp-servers/mock-rhel-host-mcp.py b/evaluation/with_skills/rh-developer__rhel-deploy/environment/mcp-servers/mock-rhel-host-mcp.py new file mode 100644 index 00000000..f10dd2f8 --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/mcp-servers/mock-rhel-host-mcp.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +""" +Mock RHEL Host MCP Server for rh-developer rhel-deploy benchmark task. + +Simulates a RHEL 9.3 host with Podman 4.9.4 for container deployment planning. +Scenario: Deploy a Flask app container as a systemd service on port 8080. +""" + +from typing import Optional + +from fastmcp import FastMCP + +mcp = FastMCP("rhel-host") + +# Mock state +MOCK_SYSTEM_INFO = { + "os": "Red Hat Enterprise Linux 9.3 (Plow)", + "kernel": "5.14.0-362.18.1.el9_3.x86_64", + "architecture": "x86_64", + "podman_version": "podman version 4.9.4", + "selinux": "Enforcing", + "firewall": "running", +} + +MOCK_OPEN_PORTS = {8080} # Port 8080 opened for Flask app +MOCK_SERVICES = { + "flask-app": { + "name": "flask-app", + "active": "active", + "state": "running", + "enabled": True, + "description": "Flask application container", + }, + "container-flask-app": { + "name": "container-flask-app", + "active": "active", + "state": "running", + "enabled": True, + "description": "Podman container flask-app.service", + }, +} + +MOCK_PODMAN_PS = """CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +a1b2c3d4e5f6 quay.io/ubi9/python-311:latest flask run 2 hours ago Up 2 hours ago 0.0.0.0:8080->8080/tcp flask-app +""" + +MOCK_PODMAN_INSPECT = """[ + { + "Id": "a1b2c3d4e5f6", + "Name": "flask-app", + "State": { + "Status": "running", + "Running": true + }, + "Config": { + "Image": "quay.io/ubi9/python-311:latest", + "Cmd": ["flask", "run", "--host=0.0.0.0", "--port=8080"] + }, + "HostConfig": { + "PortBindings": { + "8080/tcp": [{"HostPort": "8080"}] + } + } + } +] +""" + + +def _match_command(cmd: str) -> Optional[str]: + """Return a command category for pattern matching.""" + cmd_lower = cmd.strip().lower() + if "podman pull" in cmd_lower: + return "podman_pull" + if "podman run" in cmd_lower: + return "podman_run" + if "podman ps" in cmd_lower or cmd_lower == "podman ps": + return "podman_ps" + if "podman inspect" in cmd_lower: + return "podman_inspect" + if "systemctl enable" in cmd_lower: + return "systemctl_enable" + if "systemctl start" in cmd_lower: + return "systemctl_start" + if "systemctl status" in cmd_lower: + return "systemctl_status" + if "firewall-cmd" in cmd_lower: + return "firewall_cmd" + if "semanage fcontext" in cmd_lower: + return "semanage_fcontext" + if "restorecon" in cmd_lower: + return "restorecon" + return None + + +@mcp.tool +def run_command(command: str) -> dict: + """Simulate running a shell command on a RHEL host. + + Supports common deployment patterns: podman, systemctl, firewall-cmd, semanage. + Returns realistic output for supported commands; error for unknown commands. + + Args: + command: The shell command to execute (e.g. 'podman ps', 'systemctl status flask-app'). + """ + kind = _match_command(command) + if kind == "podman_pull": + return { + "command": command, + "exit_code": 0, + "stdout": "Trying to pull quay.io/ubi9/python-311:latest...\nGetting image source signatures\nCopying blob sha256:...\nCopying config sha256:...\nWriting manifest to image destination\nStoring signatures\n", + "stderr": "", + } + if kind == "podman_run": + return { + "command": command, + "exit_code": 0, + "stdout": "a1b2c3d4e5f6", + "stderr": "", + } + if kind == "podman_ps": + return { + "command": command, + "exit_code": 0, + "stdout": MOCK_PODMAN_PS, + "stderr": "", + } + if kind == "podman_inspect": + return { + "command": command, + "exit_code": 0, + "stdout": MOCK_PODMAN_INSPECT, + "stderr": "", + } + if kind == "systemctl_enable": + return { + "command": command, + "exit_code": 0, + "stdout": "", + "stderr": "", + } + if kind == "systemctl_start": + return { + "command": command, + "exit_code": 0, + "stdout": "", + "stderr": "", + } + if kind == "systemctl_status": + return { + "command": command, + "exit_code": 0, + "stdout": """● flask-app.service - Flask application container + Loaded: loaded (/etc/systemd/system/flask-app.service; enabled) + Active: active (running) since Tue 2026-03-17 10:00:00 UTC; 2h ago + Main PID: 1234 (conmon) + Tasks: 8 + Memory: 128.0M + CGroup: /system.slice/flask-app.service +""", + "stderr": "", + } + if kind == "firewall_cmd": + return { + "command": command, + "exit_code": 0, + "stdout": "success\n", + "stderr": "", + } + if kind == "semanage_fcontext": + return { + "command": command, + "exit_code": 0, + "stdout": "", + "stderr": "", + } + if kind == "restorecon": + return { + "command": command, + "exit_code": 0, + "stdout": "", + "stderr": "", + } + return { + "command": command, + "exit_code": 1, + "stdout": "", + "stderr": f"Error: Unknown or unsupported command. Supported: podman pull/run/ps/inspect, systemctl enable/start/status, firewall-cmd, semanage fcontext, restorecon.", + } + + +@mcp.tool +def get_system_info() -> dict: + """Return RHEL version, architecture, and Podman version for the target host.""" + return MOCK_SYSTEM_INFO.copy() + + +@mcp.tool +def check_service(name: str) -> dict: + """Return systemd service status for a given service name. + + Args: + name: Service name (e.g. 'flask-app', 'container-flask-app'). + """ + svc = MOCK_SERVICES.get(name) + if svc: + return {"service": name, "status": svc, "found": True} + return { + "service": name, + "found": False, + "error": f"Service '{name}' not found. Known services: {list(MOCK_SERVICES.keys())}", + } + + +@mcp.tool +def check_port(port: int) -> dict: + """Return whether a port is open in the firewall. + + Args: + port: Port number to check (e.g. 8080). + """ + open_port = port in MOCK_OPEN_PORTS + return { + "port": port, + "open": open_port, + "message": f"Port {port} is {'open' if open_port else 'closed'} in firewall.", + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/skills/rhel-deploy/SKILL.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/skills/rhel-deploy/SKILL.md new file mode 100644 index 00000000..66efbdee --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/skills/rhel-deploy/SKILL.md @@ -0,0 +1,482 @@ +--- +name: rhel-deploy +description: | + CRITICAL: When user types /rhel-deploy, use THIS skill immediately. This skill deploys applications to standalone RHEL/Fedora/CentOS systems (NOT OpenShift) using Podman containers with systemd, or native dnf builds. Handles SSH connectivity, SELinux, firewall-cmd, and systemd unit creation. Triggers: /rhel-deploy command, 'deploy to RHEL', 'deploy to Fedora', 'deploy to my server via SSH'. +model: inherit +color: yellow +metadata: + user_invocable: "true" +--- + +# /rhel-deploy Skill + +**IMPORTANT:** This skill is for deploying to standalone RHEL/Fedora/CentOS systems via SSH. If user invoked `/rhel-deploy`, skip any OpenShift-related steps and proceed directly with SSH-based deployment. + +Deploy applications to standalone RHEL systems using Podman containers or native builds with systemd service management. + +## Overview + +``` +[Intro] → [SSH Connect] → [Analyze] → [Strategy] ──┬─→ [Container Path] ──→ [Complete] + │ (Podman + systemd) + │ + └─→ [Native Path] ─────→ [Complete] + (dnf + systemd) +``` + +**Deployment Strategies (user chooses one):** +- **Container** - Build/pull container image, run with Podman, manage with systemd +- **Native** - Install dependencies with dnf, run application directly, manage with systemd + +## Prerequisites + +1. SSH access to target RHEL host with sudo privileges +2. RHEL 8+, CentOS Stream, Rocky Linux, or Fedora +3. For container deployments: Podman installed on target +4. For native deployments: Required development tools available via dnf + +## When to Use This Skill + +Use `/rhel-deploy` when deploying applications to standalone RHEL, Fedora, or CentOS systems via SSH. This skill handles Podman container or native dnf deployments with systemd service management, SELinux, and firewall configuration. + +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +## Workflow + +### Phase 0: Introduction + +Present the workflow overview: Connect → Analyze → Strategy → Build/Deploy → Verify. Describe Container (Podman + systemd) vs Native (dnf + systemd) strategies. Ask: **Ready to begin?** (yes/no) + +Proceed automatically without waiting for user input. + +### Phase 1: SSH Connection + +```markdown +## Phase 1: Connecting to RHEL Host + +**SSH Target Configuration:** + +Please provide your RHEL host details: + +| Setting | Value | Default | +|---------|-------|---------| +| Host | [required] | - | +| User | [current user] | $USER | +| Port | 22 | 22 | + +Example: `user@192.168.1.100` or `deploy@myserver.example.com` + +**Enter your SSH target:** +``` + +**Connection verification:** + +```bash +ssh -o BatchMode=yes -o ConnectTimeout=10 [user]@[host] "echo 'Connection successful'" +``` + +If connection fails, troubleshoot: host reachability, SSH key configuration, firewall port 22. + +Store `RHEL_HOST`, `RHEL_USER`, `RHEL_PORT` in session state. + +### Phase 2: Target Host Analysis + +```markdown +## Phase 2: Analyzing Target Host + +Checking capabilities of [host]... + +| Setting | Value | +|---------|-------| +| OS | [cat /etc/redhat-release] | +| Kernel | [uname -r] | +| Architecture | [uname -m] | +| Podman | [Installed v4.x / Not installed] | +| SELinux | [Enforcing / Permissive / Disabled] | +| Firewall | [Active / Inactive] | + +Is this the correct target host? (yes/no) +``` + +Proceed automatically without waiting for user input. + +**Commands to gather information:** + +```bash +ssh [target] "cat /etc/redhat-release" +ssh [target] "podman --version 2>/dev/null || echo 'Not installed'" +ssh [target] "getenforce" +ssh [target] "firewall-cmd --state 2>/dev/null || echo 'Not running'" +``` + +Store `RHEL_VERSION`, `PODMAN_AVAILABLE`, `SELINUX_STATUS`, `FIREWALL_STATUS` in session state. + +### Phase 2b: Red Hat Insights Pre-Deploy Check (Optional) + +**This phase runs only if the `lightspeed-mcp` server is available.** Use `ToolSearch` to check for Lightspeed MCP tools. If not available, skip silently and proceed to Phase 3. + +**Step 1:** Use `find_host_by_name` with the target hostname to look up the system in Red Hat Insights. + +**Step 2:** If found, use `get_system_cves` to check for critical/important CVEs on the target. + +**Step 3:** Use `get_rhel_lifecycle` to verify the target RHEL version is still supported. + +Append to Phase 2 output: + +```markdown +**Red Hat Insights (Optional):** +| Check | Status | Details | +|-------|--------|---------| +| Registered in Insights | [Yes/No] | [system-id or "Not found"] | +| RHEL Lifecycle | [Active/Maintenance/EOL] | [end date] | +| Critical/Important CVEs | [count] | [top 3 CVE IDs] | + +[If critical CVEs found:] +**WARNING:** Target system has [N] critical/important CVEs. Consider remediating before deploying. + +[If RHEL version is EOL:] +**WARNING:** RHEL [version] has reached End of Life ([date]). Consider upgrading before deploying. +``` + +These are informational warnings only — they do not block deployment. + +### Phase 3: Strategy Selection + +```markdown +## Deployment Strategy + +Based on your project ([language]/[framework]) and target capabilities: + +| Strategy | Description | Requirements | +|----------|-------------|--------------| +| **Container** | Build image, run with Podman + systemd | Podman installed | +| **Native** | Install with dnf, run directly + systemd | Runtime packages available | + +**Recommendation:** [Container/Native] because [reason] + +**Which deployment strategy would you like to use?** +1. Container - Deploy using Podman +2. Native - Deploy directly on host +``` + +Proceed automatically without waiting for user input. + +**If Podman not installed and user selects Container:** +```markdown +Podman is not installed on the target. Would you like me to install it? + +```bash +sudo dnf install -y podman +``` + +Proceed with Podman installation? (yes/no) +``` + +Proceed automatically without waiting for user input. + +Store `DEPLOYMENT_STRATEGY` in session state. + +--- + +## CONTAINER PATH (If DEPLOYMENT_STRATEGY is "Container") + +### Phase 4a-1: Image Selection + +```markdown +## Container Image + +**Options:** + +1. **Build on target** - Transfer source, build with Podman on RHEL host +2. **Build locally and transfer** - Build here, push to registry or transfer +3. **Use existing image** - Pull from registry (e.g., quay.io, docker.io) + +Which approach would you prefer? +``` + +Proceed automatically without waiting for user input. + +**For options 1 and 2 (building an image):** + +If no Containerfile/Dockerfile exists in the project, delegate to `/recommend-image`: + +```markdown +## Selecting Base Image + +To build your container, I need to select an appropriate base image. + +Invoking `/recommend-image` to get the optimal UBI image for your [language]/[framework] project... +``` + +Use the `BUILDER_IMAGE` output from `/recommend-image` as the base image in the Containerfile. + +**For build on target:** +```bash +# Transfer source and build +rsync -avz --exclude node_modules --exclude .git ./ [target]:/tmp/[app-name]-build/ +# If no Containerfile exists, generate one using BUILDER_IMAGE from /recommend-image +ssh [target] "cd /tmp/[app-name]-build && podman build -t [app-name]:latest ." +``` + +**For existing image:** +```bash +ssh [target] "podman pull [image-reference]" +``` + +### Phase 4a-2: Container Configuration + +```markdown +## Container Configuration + +**Container Settings:** +| Setting | Value | +|---------|-------| +| Name | [app-name] | +| Image | [image-ref] | +| Port Mapping | [host-port]:[container-port] | +| Volume Mounts | [list any persistent data paths] | +| Environment | [list env vars] | +| Run Mode | [rootless / rootful] | + +**SELinux Volume Labels:** Use `:z` for shared volumes, `:Z` for private volumes. See [docs/rhel-deployment.md](../../docs/rhel-deployment.md) for SELinux configuration details. + +Proceed with this configuration? (yes/modify/cancel) +``` + +Proceed automatically without waiting for user input. + +### Phase 4a-3: Systemd Unit Creation + +```markdown +## Systemd Service Configuration + +Creating systemd unit for Podman container. + +**Template to use:** +- Rootful: `templates/systemd/systemd-container-rootful.service` +- Rootless: `templates/systemd/systemd-container-rootless.service` + +**Variables to substitute:** +| Variable | Value | +|----------|-------| +| `${APP_NAME}` | [app-name] | +| `${PORT}` | [container-port] | +| `${IMAGE}` | [container-image] | + +**Target locations:** +- Rootful: `/etc/systemd/system/[app-name].service` +- Rootless: `~/.config/systemd/user/[app-name].service` + +Proceed with creating this service? (yes/no) +``` + +Proceed automatically without waiting for user input. + +**Steps to execute:** + +1. Read the appropriate template from `templates/systemd/` +2. Substitute `${APP_NAME}`, `${PORT}`, `${IMAGE}` with session state values +3. Transfer the generated unit file to the target host +4. Enable and start the service + +```bash +# Rootful: transfer to /etc/systemd/system/, daemon-reload, enable --now +# Rootless: transfer to ~/.config/systemd/user/, daemon-reload, enable --now, enable-linger +ssh [target] "sudo systemctl daemon-reload && sudo systemctl enable --now [app-name]" +``` + +### Phase 4a-4: Firewall Configuration + +```markdown +## Firewall Configuration + +Opening port [port] for application access. + +**Commands to execute:** +```bash +# Open port permanently +sudo firewall-cmd --permanent --add-port=[port]/tcp + +# Reload firewall +sudo firewall-cmd --reload + +# Verify +sudo firewall-cmd --list-ports +``` + +Proceed with firewall configuration? (yes/skip) +``` + +Proceed automatically without waiting for user input. + +--- + +## NATIVE PATH (If DEPLOYMENT_STRATEGY is "Native") + +### Phase 4b-1: Dependency Installation + +```markdown +## Installing Dependencies + +**Runtime packages for [language]:** + +See [docs/rhel-deployment.md](../../docs/rhel-deployment.md) for the complete runtime package mapping by language and RHEL version (Node.js, Python, Java, Go, Ruby, PHP). + +**Commands to execute:** +```bash +ssh [target] "sudo dnf install -y [packages]" +``` + +Proceed with installation? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Phase 4b-2: Application Deployment + +```markdown +## Deploying Application + +**Deployment location:** `/opt/[app-name]` + +**Steps:** +1. Create application directory +2. Transfer source code via rsync +3. Install application dependencies +4. Set ownership and permissions +5. Configure SELinux context + +```bash +ssh [target] "sudo mkdir -p /opt/[app-name]" +rsync -avz --exclude node_modules --exclude .git --exclude __pycache__ ./ [target]:/tmp/[app-name]-deploy/ +ssh [target] "sudo cp -r /tmp/[app-name]-deploy/* /opt/[app-name]/" +ssh [target] "cd /opt/[app-name] && npm install --production" # language-specific +ssh [target] "sudo chown -R [service-user]:[service-user] /opt/[app-name]" +ssh [target] "sudo semanage fcontext -a -t bin_t '/opt/[app-name](/.*)?'" +ssh [target] "sudo restorecon -Rv /opt/[app-name]" +``` + +Proceed with deployment? (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Phase 4b-3: Native Systemd Unit + +```markdown +## Systemd Service Configuration + +**Template to use:** `templates/systemd/systemd-native.service` + +**Variables to substitute:** +| Variable | Value | Notes | +|----------|-------|-------| +| `${APP_NAME}` | [app-name] | Application name | +| `${SERVICE_USER}` | [service-user] | User to run the service as | +| `${APP_PATH}` | /opt/[app-name] | Application install path | +| `${PORT}` | [container-port] | Application listen port | +| `${START_COMMAND}` | [see below] | Language-specific start command | + +**Start commands by language:** See [docs/rhel-deployment.md](../../docs/rhel-deployment.md) for language-specific systemd unit templates (Node.js, Python, Java, Go). + +**Target location:** `/etc/systemd/system/[app-name].service` + +**Note:** The template includes security hardening (NoNewPrivileges, ProtectSystem, ProtectHome, PrivateTmp). + +Proceed with creating this service? (yes/no) +``` + +Proceed automatically without waiting for user input. + +**Steps to execute:** + +1. Read the template from `templates/systemd/systemd-native.service` +2. Substitute all variables with session state values +3. Transfer the generated unit file to the target host +4. Enable and start the service + +### Phase 4b-4: Firewall Configuration + +Same as container path - open required port with firewall-cmd. + +--- + +## COMPLETION (Both paths converge here) + +### Phase 5: Completion + +```markdown +## Deployment Complete! + +Your application is now running on [host]. + +**Application Summary:** +| Setting | Value | +|---------|-------| +| Name | [app-name] | +| Host | [host] | +| Strategy | [Container/Native] | +| Service | [app-name].service | + +**Access URLs:** +| Type | URL | +|------|-----| +| HTTP | http://[host]:[port] | +| SSH | ssh [user]@[host] | + +**Service Status:** [systemctl status output] + +**Quick Commands:** + +Show quick commands for: view logs (journalctl), restart/stop/status (systemctl), container logs (if container), and removal steps. +``` + +### Phase 5a: Handle Deployment Failure + +If the service fails to start or is not accessible: + +```markdown +## Deployment Failed + +The service did not start successfully. + +**Service Status:** [systemctl status output showing failure] + +**Recent Errors:** +| Time | Error | +|------|-------| +| [time] | [error from journalctl] | + +**Would you like me to diagnose the issue?** +1. **Debug RHEL** (`/debug-rhel`) - Full system diagnosis (systemd, journal, SELinux, firewall) +2. **Debug Container** (`/debug-container`) - Container state, logs, exit codes +3. **View full logs** - Complete journalctl output +4. **Check SELinux** - Quick SELinux denial check +5. **Check firewall** - Quick firewall port check +6. **Stop and clean up** + +Select an option: +``` + +Proceed automatically without waiting for user input. + +- If user selects "Debug RHEL" → Invoke `/debug-rhel` skill +- If user selects "Debug Container" → Invoke `/debug-container` skill +- After debugging → Offer to retry deployment + +## Dependencies + +### Required MCP Servers +- `lightspeed-mcp` (optional) - Red Hat Insights pre-deploy checks + +### Related Skills +- `/debug-rhel` - systemd failures, SELinux denials, firewall blocking +- `/debug-container` - Container startup issues on RHEL host + +### Reference Documentation +- [docs/rhel-deployment.md](../../docs/rhel-deployment.md) - Systemd templates, SELinux, firewall, runtime packages +- [docs/selinux-troubleshooting.md](../../docs/selinux-troubleshooting.md) - SELinux denial analysis and fixes +- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common error patterns and troubleshooting +- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (ssh, podman) diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/buildconfig.yaml.template b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/buildconfig.yaml.template new file mode 100644 index 00000000..b3294eb2 --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/buildconfig.yaml.template @@ -0,0 +1,38 @@ +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: build + app.kubernetes.io/part-of: ${APP_NAME} +spec: + source: + type: Git + git: + uri: ${GIT_URL} + ref: ${GIT_BRANCH} + strategy: + type: Source + sourceStrategy: + from: + kind: DockerImage + name: ${BUILDER_IMAGE} + env: [] + output: + to: + kind: ImageStreamTag + name: ${APP_NAME}:latest + triggers: + - type: ConfigChange + - type: ImageChange + runPolicy: Serial + resources: + limits: + memory: "1Gi" + cpu: "1" + requests: + memory: "512Mi" + cpu: "500m" diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/deployment.yaml.template b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/deployment.yaml.template new file mode 100644 index 00000000..eb3b481a --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/deployment.yaml.template @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: application + app.kubernetes.io/part-of: ${APP_NAME} + annotations: + image.openshift.io/triggers: | + [{"from":{"kind":"ImageStreamTag","name":"${APP_NAME}:latest"},"fieldPath":"spec.template.spec.containers[0].image"}] +spec: + replicas: ${REPLICAS} + selector: + matchLabels: + app: ${APP_NAME} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + template: + metadata: + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + spec: + containers: + - name: ${APP_NAME} + image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/${APP_NAME}:latest + ports: + - containerPort: ${CONTAINER_PORT} + protocol: TCP + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + livenessProbe: + httpGet: + path: / + port: ${CONTAINER_PORT} + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: / + port: ${CONTAINER_PORT} + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + env: [] + restartPolicy: Always + terminationGracePeriodSeconds: 30 diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/Chart.yaml.template b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/Chart.yaml.template new file mode 100644 index 00000000..1aa22dd1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/Chart.yaml.template @@ -0,0 +1,13 @@ +apiVersion: v2 +name: ${APP_NAME} +description: ${APP_DESCRIPTION} +type: application +version: 0.1.0 +appVersion: "${APP_VERSION}" +keywords: + - ${LANGUAGE} + - ${FRAMEWORK} + - openshift +maintainers: + - name: ${MAINTAINER_NAME} + email: ${MAINTAINER_EMAIL} diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/NOTES.txt.template b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/NOTES.txt.template new file mode 100644 index 00000000..154e628d --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/NOTES.txt.template @@ -0,0 +1,32 @@ +Congratulations! Your application {{ include "${APP_NAME}.fullname" . }} has been deployed. + +{{- if .Values.route.enabled }} + +Access your application at: +{{- if .Values.route.host }} + https://{{ .Values.route.host }} +{{- else }} + Run: oc get route {{ include "${APP_NAME}.fullname" . }} -o jsonpath='{.spec.host}' +{{- end }} + +{{- else }} + +Your application is available internally at: + {{ include "${APP_NAME}.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }} + +To expose it externally, create a Route or set route.enabled=true. + +{{- end }} + +Useful commands: + # View pods + oc get pods -l app.kubernetes.io/name={{ include "${APP_NAME}.name" . }} + + # View logs + oc logs -l app.kubernetes.io/name={{ include "${APP_NAME}.name" . }} -f + + # Upgrade release + helm upgrade {{ .Release.Name }} ./{{ .Chart.Name }} -f values.yaml + + # Uninstall release + helm uninstall {{ .Release.Name }} diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/_helpers.tpl.template b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/_helpers.tpl.template new file mode 100644 index 00000000..15873b10 --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/_helpers.tpl.template @@ -0,0 +1,60 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "${APP_NAME}.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "${APP_NAME}.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "${APP_NAME}.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "${APP_NAME}.labels" -}} +helm.sh/chart: {{ include "${APP_NAME}.chart" . }} +{{ include "${APP_NAME}.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "${APP_NAME}.selectorLabels" -}} +app.kubernetes.io/name: {{ include "${APP_NAME}.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "${APP_NAME}.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "${APP_NAME}.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/deployment.yaml.template b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/deployment.yaml.template new file mode 100644 index 00000000..a6cbd868 --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/deployment.yaml.template @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "${APP_NAME}.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "${APP_NAME}.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "${APP_NAME}.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/route.yaml.template b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/route.yaml.template new file mode 100644 index 00000000..e2bab29a --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/route.yaml.template @@ -0,0 +1,24 @@ +{{- if .Values.route.enabled }} +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + {{- if .Values.route.host }} + host: {{ .Values.route.host }} + {{- end }} + to: + kind: Service + name: {{ include "${APP_NAME}.fullname" . }} + weight: 100 + port: + targetPort: http + {{- with .Values.route.tls }} + tls: + termination: {{ .termination }} + insecureEdgeTerminationPolicy: {{ .insecureEdgeTerminationPolicy }} + {{- end }} + wildcardPolicy: None +{{- end }} diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/service.yaml.template b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/service.yaml.template new file mode 100644 index 00000000..837bc888 --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/service.yaml.template @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "${APP_NAME}.selectorLabels" . | nindent 4 }} diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/values.yaml.template b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/values.yaml.template new file mode 100644 index 00000000..1cca6017 --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/helm/values.yaml.template @@ -0,0 +1,67 @@ +# Default values for ${APP_NAME} +replicaCount: 1 + +image: + repository: ${IMAGE_REPOSITORY} + pullPolicy: IfNotPresent + tag: "${IMAGE_TAG}" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: "" + +podAnnotations: {} +podSecurityContext: {} +securityContext: {} + +service: + type: ClusterIP + port: ${CONTAINER_PORT} + +route: + enabled: true + host: "" + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + +resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + +livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + +readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + +nodeSelector: {} +tolerations: [] +affinity: {} + +env: [] +# - name: MY_VAR +# value: "my-value" diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/imagestream.yaml.template b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/imagestream.yaml.template new file mode 100644 index 00000000..46572193 --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/imagestream.yaml.template @@ -0,0 +1,13 @@ +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: image + app.kubernetes.io/part-of: ${APP_NAME} +spec: + lookupPolicy: + local: false diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/route.yaml.template b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/route.yaml.template new file mode 100644 index 00000000..7c53d2e7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/route.yaml.template @@ -0,0 +1,21 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: route + app.kubernetes.io/part-of: ${APP_NAME} +spec: + to: + kind: Service + name: ${APP_NAME} + weight: 100 + port: + targetPort: http + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + wildcardPolicy: None diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/service.yaml.template b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/service.yaml.template new file mode 100644 index 00000000..7e1cf371 --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/service.yaml.template @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: service + app.kubernetes.io/part-of: ${APP_NAME} +spec: + selector: + app: ${APP_NAME} + ports: + - name: http + port: ${CONTAINER_PORT} + targetPort: ${CONTAINER_PORT} + protocol: TCP + type: ClusterIP + sessionAffinity: None diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-container-rootful.service b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-container-rootful.service new file mode 100644 index 00000000..c1e8fe8f --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-container-rootful.service @@ -0,0 +1,27 @@ +# Rootful Podman container managed by systemd (system service) +# Location: /etc/systemd/system/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${PORT} - Port number (used for both host and container binding) +# ${IMAGE} - Container image reference + +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run --name ${APP_NAME} \ + -p ${PORT}:${PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-container-rootless.service b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-container-rootless.service new file mode 100644 index 00000000..ca9dc371 --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-container-rootless.service @@ -0,0 +1,27 @@ +# Rootless Podman container managed by systemd (user service) +# Location: ~/.config/systemd/user/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${PORT} - Port number (used for both host and container binding) +# ${IMAGE} - Container image reference + +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run --name ${APP_NAME} \ + -p ${PORT}:${PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-native.service b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-native.service new file mode 100644 index 00000000..c55cfc07 --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-native.service @@ -0,0 +1,39 @@ +# Native application managed by systemd (system service) +# Location: /etc/systemd/system/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${SERVICE_USER} - User to run the service as +# ${APP_PATH} - Application install path (e.g., /opt/app-name) +# ${PORT} - Application listen port +# ${START_COMMAND} - Application start command +# +# Start command examples by language: +# Node.js: /usr/bin/node ${APP_PATH}/server.js +# Python: /usr/bin/python3 ${APP_PATH}/app.py +# Java: /usr/bin/java -jar ${APP_PATH}/app.jar +# Go: ${APP_PATH}/binary-name + +[Unit] +Description=${APP_NAME} Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=${APP_PATH} +Environment=PORT=${PORT} +ExecStart=${START_COMMAND} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=${APP_PATH} + +[Install] +WantedBy=multi-user.target diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/instruction.md b/evaluation/with_skills/rh-developer__rhel-deploy/instruction.md new file mode 100644 index 00000000..b7c3a70e --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/instruction.md @@ -0,0 +1,12 @@ +# RHEL Deployment Task + +You are a Red Hat developer. Plan the deployment of a containerized application on RHEL using Podman and systemd. + +## Requirements +- Configure the container to run as a systemd service +- Address security hardening (SELinux, privilege restrictions) +- Include volume mounts and networking configuration + +Use available tools to examine the environment. Document your methodology, configuration, and deployment plan in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/solution/solve.sh b/evaluation/with_skills/rh-developer__rhel-deploy/solution/solve.sh new file mode 100644 index 00000000..cf537860 --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/solution/solve.sh @@ -0,0 +1,43 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# RHEL Deployment Plan + +## Rootless Podman Setup +```bash +sudo useradd -m appuser +sudo loginctl enable-linger appuser +``` + +## Container Run +```bash +podman run -d --name flask-app -p 8080:5000 -v /opt/app-data:/data:z flask-app:latest +``` + +## Systemd Service +Path: `~/.config/systemd/user/flask-app.service` +```ini +[Unit] +Description=Flask App Container +[Service] +ExecStart=/usr/bin/podman run --rm --name flask-app -p 8080:5000 -v /opt/app-data:/data:Z flask-app:latest +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +[Install] +WantedBy=default.target +``` + +## Firewall +```bash +sudo firewall-cmd --permanent --add-port=8080/tcp +sudo firewall-cmd --reload +``` + +## SELinux +```bash +sudo semanage port -a -t http_port_t -p tcp 8080 +sudo semanage fcontext -a -t container_file_t '/opt/app-data(/.*)?' +sudo restorecon -Rv /opt/app-data +``` +REPORT_EOF diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/task.toml b/evaluation/with_skills/rh-developer__rhel-deploy/task.toml new file mode 100644 index 00000000..0ac61da9 --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__rhel-deploy" +name = "rh-developer RHEL Deployment Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "rhel-deploy", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/tests/llm_judge.py b/evaluation/with_skills/rh-developer__rhel-deploy/tests/llm_judge.py new file mode 100644 index 00000000..5d7ba0df --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "selinux_volume_labels", + "file": "/root/report.md", + "question": "Does the report explain SELinux volume labels :z (shared, multi-container) and :Z (private) for Podman bind mounts?", + "reference": "A skilled report uses :z or :Z suffixes on volume mounts and explains the difference. An unskilled report skips SELinux mount context." + }, + { + "id": "rootless_systemd", + "file": "/root/report.md", + "question": "Does the report address rootless systemd service configuration (~/.config/systemd/user/) and loginctl enable-linger?", + "reference": "A skilled report shows the rootless systemd path and explains enable-linger for services to survive logout. An unskilled report only shows rootful /etc/systemd/system/ paths." + }, + { + "id": "semanage_fcontext_restorecon", + "file": "/root/report.md", + "question": "Does the report use semanage fcontext + restorecon for setting SELinux file contexts on application directories?", + "reference": "A skilled report uses 'semanage fcontext -a -t bin_t' plus 'restorecon -Rv' for app files. An unskilled report skips file-level SELinux context management." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/tests/test.sh b/evaluation/with_skills/rh-developer__rhel-deploy/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/tests/test_outputs.py b/evaluation/with_skills/rh-developer__rhel-deploy/tests/test_outputs.py new file mode 100644 index 00000000..b4a1c092 --- /dev/null +++ b/evaluation/with_skills/rh-developer__rhel-deploy/tests/test_outputs.py @@ -0,0 +1,98 @@ +""" +Tests for rh-developer__rhel-deploy per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_rhel_or_podman(self): + content = read_report().lower() + assert "rhel" in content or "podman" in content, "report should mention RHEL or Podman" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_selinux_volume_labels(self): + """Skill teaches SELinux volume labels: :z = shared (relabeled for multi-container), + :Z = private. Without skill, agents skip SELinux mount context.""" + c = read_report() + assert ":z" in c or ":Z" in c or "selinux" in c.lower(), ( + "should address SELinux volume labels (:z shared, :Z private)" + ) + + def test_rootless_systemd_path(self): + """Skill teaches rootless systemd service location ~/.config/systemd/user/ + vs /etc/systemd/system/ for rootful. Without skill, agents only know rootful.""" + c = read_report() + assert ".config/systemd/user" in c or "rootless" in c.lower(), ( + "should address rootless systemd path (~/.config/systemd/user/)" + ) + + def test_enable_linger(self): + """Skill teaches loginctl enable-linger required for rootless user services + to survive logout. Without skill, agents miss this requirement.""" + c = read_report().lower() + assert "enable-linger" in c or "loginctl" in c or "linger" in c, ( + "should mention loginctl enable-linger for rootless services" + ) + + def test_semanage_fcontext(self): + """Skill teaches semanage fcontext + restorecon for setting SELinux context + on application files. Without skill, agents skip file context management.""" + c = read_report().lower() + assert ("semanage fcontext" in c or "semanage" in c) and ( + "restorecon" in c or "fcontext" in c + ), "should use semanage fcontext + restorecon for file SELinux context" + + def test_firewall_port(self): + """Skill teaches firewall-cmd for opening application ports.""" + c = read_report().lower() + assert "firewall-cmd" in c or ("firewall" in c and "port" in c), ( + "should address firewall port configuration" + ) + + def test_systemd_hardening_directives(self): + """Docs teach systemd hardening directives: NoNewPrivileges=true, + ProtectSystem=strict, ReadWritePaths. Without docs, agents create basic + unit files without security hardening.""" + c = read_report() + assert any(t in c for t in [ + "NoNewPrivileges", "ProtectSystem", "ReadWritePaths", + "PrivateTmp", "ProtectHome", + ]) or "hardening" in c.lower(), ( + "should include systemd hardening directives (NoNewPrivileges, ProtectSystem)" + ) + + def test_container_security_practices(self): + """Skill teaches defence-in-depth for containers: dropping capabilities, + resource limits, read-only root, security options. Without skill, + agents deploy containers with default security settings.""" + c = read_report().lower() + practices = sum(1 for t in [ + "cap-drop", "cap_drop", "capability", + "--read-only", "read-only root", + "resource limit", "memory", "cpus", + "no-new-privileges", "security-opt", + ] if t in c) + assert practices >= 2, ( + "should address at least 2 container security practices " + "(capability dropping, resource limits, read-only root, security options)" + ) diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/Dockerfile b/evaluation/with_skills/rh-developer__s2i-build/environment/Dockerfile new file mode 100644 index 00000000..1cbfefcf --- /dev/null +++ b/evaluation/with_skills/rh-developer__s2i-build/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-developer__s2i-build/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/with_skills/rh-developer__s2i-build/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/skills/s2i-build/SKILL.md b/evaluation/with_skills/rh-developer__s2i-build/environment/skills/s2i-build/SKILL.md new file mode 100644 index 00000000..d5b179fa --- /dev/null +++ b/evaluation/with_skills/rh-developer__s2i-build/environment/skills/s2i-build/SKILL.md @@ -0,0 +1,391 @@ +--- +name: s2i-build +description: | + Create BuildConfig and ImageStream resources on OpenShift and trigger a Source-to-Image (S2I) build. Use this skill after /detect-project to build container images from source code on the cluster. Handles namespace verification, resource creation with user confirmation, build monitoring with log streaming, and failure recovery. Triggers on /s2i-build command. Run before /deploy. +model: inherit +color: green +metadata: + user_invocable: "true" +--- + +# /s2i-build Skill + +Create the necessary OpenShift resources (BuildConfig, ImageStream) and trigger a Source-to-Image build on the cluster. + +## Prerequisites + +Before running this skill, ensure: +1. User is logged into OpenShift cluster +2. Target namespace/project exists or can be created +3. Git repository URL is available (or will use binary build) + +## When to Use This Skill + +Use this skill after `/detect-project` to build container images from source code on OpenShift using Source-to-Image. It creates BuildConfig and ImageStream resources, triggers the build, and monitors progress with log streaming. + +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +## Workflow + +### Step 1: Check OpenShift Connection + +Use kubernetes MCP to verify connection: + +```markdown +## Checking OpenShift Connection... + +**Cluster:** [cluster-url from kubeconfig] +**User:** [current user] +**Current Namespace:** [current namespace] + +Is this the correct cluster and namespace for the build? +- yes - Continue +- no - Let me switch context +``` + +Proceed automatically without waiting for user input. + +### Step 2: Gather Build Information + +Collect required information (from /detect-project or ask user): + +```markdown +## S2I Build Configuration + +I need the following information: + +| Setting | Current Value | Source | +|---------|---------------|--------| +| App Name | `[name]` | [from detect-project / folder name] | +| Git URL | `[url]` | [from .git/config / needs input] | +| Git Branch | `main` | [default] | +| S2I Builder | `[image]` | [from detect-project / needs input] | +| Namespace | `[ns]` | [from current context] | + +[For Python projects only - include these rows if PYTHON_ENTRY_FILE is set] +| Entry Point | `[PYTHON_ENTRY_FILE]` | [from detect-project] | +| APP_MODULE | `[PYTHON_APP_MODULE]` | [Python only - required if entry point != app.py] | +| gunicorn | [Found / Missing] | [from detect-project] | + +Please confirm these values or tell me what to change. +``` + +**Python Entry Point Warning:** + +If `PYTHON_ENTRY_FILE` is NOT `app.py` AND `PYTHON_HAS_GUNICORN` is `false`: + +```markdown +## Python Configuration Issue + +Your application uses `[PYTHON_ENTRY_FILE]` as entry point, but `gunicorn` is not in your requirements. + +**This build will FAIL** because: +- The S2I Python builder requires `gunicorn` to use `APP_MODULE` +- Without gunicorn, it looks for `app.py` (which doesn't exist) + +**Please choose:** +1. **Add gunicorn** - Add `gunicorn` to requirements.txt and retry +2. **Rename entry point** - Rename `[main.py]` to `app.py` +3. **Continue anyway** - Proceed (build will likely fail) +``` + +Proceed automatically without waiting for user input. + +**To detect Git URL:** +- Read `.git/config` and extract `[remote "origin"]` url + +### Step 3: Verify Namespace + +Use kubernetes MCP `resources_list` to check if namespace exists: + +```markdown +## Namespace Check + +Checking if namespace `[namespace]` exists... + +[If exists] +Namespace `[namespace]` exists and you have access. + +[If not exists] +Namespace `[namespace]` does not exist. + +Would you like me to create it? (yes/no) +``` + +Proceed automatically without waiting for user input. + +If creating namespace, use `resources_create_or_update`: +```yaml +apiVersion: v1 +kind: Namespace +metadata: + name: [namespace] +``` + +### Step 4: Create ImageStream + +Show the ImageStream that will be created: + +```markdown +## Step 1 of 3: Create ImageStream + +An ImageStream stores references to your built container images. + +```yaml +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: + name: [app-name] + namespace: [namespace] + labels: + app: [app-name] + app.kubernetes.io/name: [app-name] +spec: + lookupPolicy: + local: false +``` + +**Proceed with creating this ImageStream?** (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Step 5: Create BuildConfig + +Show the BuildConfig: + +**For non-Python projects OR Python with app.py entry point:** + +```markdown +## Step 2 of 3: Create BuildConfig + +A BuildConfig defines how to build your application using S2I. + +```yaml +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: [app-name] + namespace: [namespace] + labels: + app: [app-name] + app.kubernetes.io/name: [app-name] +spec: + source: + type: Git + git: + uri: [git-url] + ref: [git-branch] + strategy: + type: Source + sourceStrategy: + from: + kind: DockerImage + name: [builder-image] + output: + to: + kind: ImageStreamTag + name: [app-name]:latest + triggers: + - type: ConfigChange + - type: ImageChange + runPolicy: Serial +``` + +**This BuildConfig will:** +- Pull source from: `[git-url]` (branch: `[git-branch]`) +- Build using S2I with: `[builder-image]` +- Push result to: `[app-name]:latest` ImageStream + +**Proceed with creating this BuildConfig?** (yes/no) +``` + +**For Python projects with non-default entry point (e.g., main.py):** + +```markdown +## Step 2 of 3: Create BuildConfig + +A BuildConfig defines how to build your application using S2I. + +```yaml +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: [app-name] + namespace: [namespace] + labels: + app: [app-name] + app.kubernetes.io/name: [app-name] +spec: + source: + type: Git + git: + uri: [git-url] + ref: [git-branch] + strategy: + type: Source + sourceStrategy: + from: + kind: DockerImage + name: [builder-image] + # Python S2I: Required when entry point is not app.py + env: + - name: APP_MODULE + value: "[PYTHON_APP_MODULE]" # e.g., "main:app" + output: + to: + kind: ImageStreamTag + name: [app-name]:latest + triggers: + - type: ConfigChange + - type: ImageChange + runPolicy: Serial +``` + +**This BuildConfig will:** +- Pull source from: `[git-url]` (branch: `[git-branch]`) +- Build using S2I with: `[builder-image]` +- Push result to: `[app-name]:latest` ImageStream + +**Python Entry Point Configuration:** +- Entry point file: `[PYTHON_ENTRY_FILE]` +- APP_MODULE: `[PYTHON_APP_MODULE]` +- This tells the S2I Python builder how to start your application with gunicorn. + +**Proceed with creating this BuildConfig?** (yes/no) +``` + +Proceed automatically without waiting for user input. + +### Step 6: Start Build + +```markdown +## Step 3 of 3: Start Build + +Resources created successfully! + +| Resource | Name | Status | +|----------|------|--------| +| ImageStream | [app-name] | Created | +| BuildConfig | [app-name] | Created | + +**Would you like me to start a build now?** (yes/no) + +(You can also trigger builds later with: oc start-build [app-name]) +``` + +Proceed automatically without waiting for user input. + +If yes, create a Build resource: +```yaml +apiVersion: build.openshift.io/v1 +kind: Build +metadata: + generateName: [app-name]- + namespace: [namespace] + labels: + app: [app-name] + buildconfig: [app-name] + annotations: + openshift.io/build-config.name: [app-name] +spec: + serviceAccount: builder + source: + type: Git + git: + uri: [git-url] + ref: [git-branch] + strategy: + type: Source + sourceStrategy: + from: + kind: DockerImage + name: [builder-image] + output: + to: + kind: ImageStreamTag + name: [app-name]:latest + triggeredBy: + - message: Manually triggered +``` + +### Step 7: Monitor Build + +Stream build logs using kubernetes MCP `pod_logs`: + +```markdown +## Build Progress + +**Build:** [app-name]-1 +**Status:** Running +**Phase:** [current phase] + +--- +[Streaming build logs here] +--- + +[When complete] + +## Build Complete! + +**Build:** [app-name]-1 +**Status:** Complete +**Duration:** [X]m [Y]s +**Image:** image-registry.openshift-image-registry.svc:5000/[namespace]/[app-name]:latest + +**CRITICAL: Ensure the build status is 'Complete' before proceeding to deployment.** + +The image is ready for deployment. +Run `/deploy` to create Deployment, Service, and Route. +``` + +### Step 8: Handle Build Failure + +If build fails: + +```markdown +## Build Failed + +**Build:** [app-name]-1 +**Status:** Failed +**Phase:** [phase where it failed] + +**Error:** +``` +[Last 20 lines of build log] +``` + +**Common causes for [phase] failure:** +- [relevant troubleshooting tips] + +**Options:** +1. **Debug Build** (`/debug-build`) - Full build diagnosis + - Analyzes BuildConfig, build logs, source access, registry auth + - Identifies root cause and suggests remediation +2. View full build logs +3. Delete failed build and retry +4. Update BuildConfig and retry +5. Cancel and troubleshoot + +What would you like to do? +``` + +- If user selects "Debug Build" → Invoke `/debug-build` skill with build name +- After debugging → Offer to retry build + +## Dependencies + +### Required MCP Servers +- `openshift` - Kubernetes/OpenShift resource access for BuildConfigs, ImageStreams, and build monitoring + +### Related Skills +- `/debug-build` - Build failures (source access, dependencies, registry issues) +- `/deploy` - After successful build, to deploy the image + +### Reference Documentation +- [docs/builder-images.md](../../docs/builder-images.md) - S2I builder image selection, version mapping +- [docs/python-s2i-entrypoints.md](../../docs/python-s2i-entrypoints.md) - Python APP_MODULE configuration, entry point troubleshooting +- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common build error patterns and troubleshooting +- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (oc) diff --git a/evaluation/with_skills/rh-developer__s2i-build/instruction.md b/evaluation/with_skills/rh-developer__s2i-build/instruction.md new file mode 100644 index 00000000..107967b9 --- /dev/null +++ b/evaluation/with_skills/rh-developer__s2i-build/instruction.md @@ -0,0 +1,12 @@ +# S2I Build Configuration Task + +You are a Red Hat developer. Configure a Source-to-Image (S2I) build for a Python web application. + +## Requirements +- Select the appropriate builder image +- Configure the build process and entry point +- Address application startup configuration + +Use MCP tools to examine the cluster. Document your methodology, configuration, and build plan in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-developer__s2i-build/solution/solve.sh b/evaluation/with_skills/rh-developer__s2i-build/solution/solve.sh new file mode 100644 index 00000000..a25acec6 --- /dev/null +++ b/evaluation/with_skills/rh-developer__s2i-build/solution/solve.sh @@ -0,0 +1,60 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# S2I Build Configuration + +## Problem +Python Flask app uses `main.py` as entry point, not the default `app.py`. + +## Solution +1. Create ImageStream for output image +2. Create BuildConfig with `APP_MODULE=main:app` in `sourceStrategy.env` +3. Ensure `gunicorn` is in `requirements.txt` + +### ImageStream +```yaml +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: + name: flask-app + labels: + app: flask-app +spec: + lookupPolicy: + local: false +``` + +### BuildConfig +```yaml +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: flask-app +spec: + source: + type: Git + git: + uri: https://github.com/example/flask-app + strategy: + type: Source + sourceStrategy: + from: + kind: ImageStreamTag + name: python:3.11-ubi9 + namespace: openshift + env: + - name: APP_MODULE + value: "main:app" + output: + to: + kind: ImageStreamTag + name: flask-app:latest +``` + +### S2I Build Phases +- **Assemble**: Install dependencies from requirements.txt (including gunicorn), compile assets. Customizable via `.s2i/bin/assemble`. +- **Run**: Start the application using gunicorn with APP_MODULE. Customizable via `.s2i/bin/run`. + +### Why APP_MODULE is needed +S2I Python startup sequence: app.sh → gunicorn+APP_MODULE → app.py → ERROR +Since entry is main.py (not app.py), gunicorn must be installed and APP_MODULE must point to main:app. +REPORT_EOF diff --git a/evaluation/with_skills/rh-developer__s2i-build/task.toml b/evaluation/with_skills/rh-developer__s2i-build/task.toml new file mode 100644 index 00000000..8dedc143 --- /dev/null +++ b/evaluation/with_skills/rh-developer__s2i-build/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__s2i-build" +name = "rh-developer S2I Build Configuration Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "s2i-build", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-developer__s2i-build/tests/llm_judge.py b/evaluation/with_skills/rh-developer__s2i-build/tests/llm_judge.py new file mode 100644 index 00000000..5fbc562a --- /dev/null +++ b/evaluation/with_skills/rh-developer__s2i-build/tests/llm_judge.py @@ -0,0 +1,114 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "app_module_in_buildconfig", + "file": "/root/report.md", + "question": "Does the report specify that APP_MODULE should be set in the BuildConfig's sourceStrategy.env section (not as a generic environment variable), using the module:callable format (e.g., app:app or main:app)?", + "reference": "A skilled report places APP_MODULE in sourceStrategy.env of the BuildConfig YAML, using the module:callable format. An unskilled report mentions APP_MODULE generically without specifying its placement in sourceStrategy.env." + }, + { + "id": "s2i_build_phases", + "file": "/root/report.md", + "question": "Does the report explain S2I build phases (assemble for dependency installation and compilation, run for application startup) and how they can be customized via .s2i/bin/ scripts?", + "reference": "A skilled report explains the assemble and run phases and mentions .s2i/bin/assemble or .s2i/bin/run for customization. An unskilled report treats S2I as a monolithic process." + }, + { + "id": "gunicorn_dependency", + "file": "/root/report.md", + "question": "Does the report explicitly state that gunicorn must be in requirements.txt specifically BECAUSE the S2I Python builder uses gunicorn to serve the application specified by APP_MODULE?", + "reference": "A skilled report identifies gunicorn as a required dependency for Python S2I with APP_MODULE. An unskilled report doesn't link gunicorn to the entry point mechanism." + }, + { + "id": "imagestream_as_separate_resource", + "file": "/root/report.md", + "question": "Does the report include a standalone ImageStream YAML manifest (with apiVersion: image.openshift.io/v1 and kind: ImageStream) as a separate resource definition, rather than only referencing ImageStreamTag within the BuildConfig output section?", + "reference": "A skilled report defines the ImageStream as its own YAML resource with apiVersion: image.openshift.io/v1, kind: ImageStream, and lookupPolicy configuration, created as a prerequisite before the BuildConfig. An unskilled report only references ImageStreamTag as an output target in the BuildConfig but does not show the ImageStream resource definition." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-developer__s2i-build/tests/test.sh b/evaluation/with_skills/rh-developer__s2i-build/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__s2i-build/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-developer__s2i-build/tests/test_outputs.py b/evaluation/with_skills/rh-developer__s2i-build/tests/test_outputs.py new file mode 100644 index 00000000..ec2af10d --- /dev/null +++ b/evaluation/with_skills/rh-developer__s2i-build/tests/test_outputs.py @@ -0,0 +1,84 @@ +""" +Tests for rh-developer__s2i-build per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_s2i(self): + content = read_report().lower() + assert "s2i" in content or "source-to-image" in content, ( + "report should mention S2I" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_app_module_format(self): + """Skill teaches APP_MODULE env var format module:app (e.g. main:app) for + non-default Python entry points. Without skill, agents don't know this format.""" + c = read_report() + assert "APP_MODULE" in c or "app_module" in c.lower(), ( + "should reference APP_MODULE env var for Python S2I entry point" + ) + + def test_module_colon_app_syntax(self): + """Skill teaches the module:app syntax (e.g., main:app, wsgi:application). + Without skill, agents don't know the colon-separated format.""" + c = read_report() + assert any(t in c for t in ["main:app", "wsgi:app", "module:app", ":app", ":application"]) or ( + "APP_MODULE" in c and ":" in c + ), "should show module:app format for APP_MODULE" + + def test_s2i_build_phases(self): + """Skill teaches S2I build phases: assemble (install deps, compile) and + run (start app). Without skill, agents treat S2I as a black box.""" + c = read_report().lower() + assert ("assemble" in c and ("run" in c or "start" in c)) or ( + "build phase" in c or "build step" in c or "build process" in c + ), "should explain S2I build phases (assemble and run)" + + def test_buildconfig_imagestream(self): + """Skill teaches creating ImageStream + BuildConfig with source/builder/output.""" + c = read_report().lower() + assert any(t in c for t in ["buildconfig", "imagestream", "build config"]) and any(t in c for t in [ + "source", "builder", "output" + ]), "should define BuildConfig/ImageStream" + + def test_gunicorn_requirement(self): + """Skill teaches gunicorn must be in requirements.txt for APP_MODULE.""" + c = read_report().lower() + assert "gunicorn" in c and any(t in c for t in [ + "requirements", "pip", "install", "wsgi", "app_module" + ]), "should address gunicorn requirement for S2I Python" + + def test_standalone_imagestream_yaml(self): + """Skill teaches creating ImageStream as a separate resource with + image.openshift.io/v1 API group and lookupPolicy. Without skill, + agents reference ImageStreamTag in BuildConfig but don't define + the ImageStream resource itself.""" + c = read_report() + has_is_api = "image.openshift.io" in c + has_lookup = "lookupPolicy" in c + assert has_is_api or has_lookup, ( + "should define ImageStream resource with image.openshift.io API" + ) + diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/Dockerfile b/evaluation/with_skills/rh-developer__validate-environment/environment/Dockerfile new file mode 100644 index 00000000..1cbfefcf --- /dev/null +++ b/evaluation/with_skills/rh-developer__validate-environment/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/with_skills/rh-developer__validate-environment/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/with_skills/rh-developer__validate-environment/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/skills/validate-environment/SKILL.md b/evaluation/with_skills/rh-developer__validate-environment/environment/skills/validate-environment/SKILL.md new file mode 100644 index 00000000..33020fe0 --- /dev/null +++ b/evaluation/with_skills/rh-developer__validate-environment/environment/skills/validate-environment/SKILL.md @@ -0,0 +1,232 @@ +--- +name: validate-environment +description: | + Check and report the status of required tools and environment for rh-developer skills. Validates tool installation (oc, helm, podman, git, skopeo, etc.), cluster connectivity, and permissions. Use this skill before running other deployment skills to ensure prerequisites are met. Triggers on /validate-environment command or when user asks to check their environment setup. +model: inherit +color: cyan +metadata: + user_invocable: "true" +--- + +# Validate Environment Skill + +Check that required tools and environment are properly configured. + +## When to Use This Skill + +- User wants to verify their environment before running deployment skills +- User encounters tool-related errors and needs a diagnostic check +- First-time setup or after environment changes to confirm readiness + +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. Proceed automatically without waiting for user input. +2. Present results clearly and ask if user wants to proceed with fixes +3. Never auto-fix issues without user approval + +## Workflow + +### Step 1: Determine Validation Scope + +Ask user if not clear: + +```markdown +## Environment Validation + +What would you like to validate? + +1. **All** - Check all tools and connections +2. **OpenShift** - Check oc, helm, cluster connectivity +3. **RHEL/Containers** - Check podman, ssh, container tools +4. **Minimal** - Just check core tools (git, curl) + +Select an option (1-4): +``` + +### Step 2: Check Core Tools + +Run these checks using Bash: + +```bash +# Check each tool and capture version +check_tool() { + if command -v "$1" &> /dev/null; then + echo "INSTALLED: $1 ($($1 --version 2>&1 | head -1))" + else + echo "MISSING: $1" + fi +} +``` + +**Tools to check:** git, curl, jq, oc, helm, podman, docker, skopeo, ssh + +> **See [docs/prerequisites.md](../../docs/prerequisites.md)** for the complete tool requirements by skill, check commands, and installation instructions. + +### Step 3: Check OpenShift Connectivity (if TARGET includes openshift) + +```bash +# Check if logged in +oc whoami + +# Check current project +oc project + +# Check permissions +oc auth can-i create deployments +oc auth can-i create buildconfigs +oc auth can-i create imagestreams +``` + +### Step 4: Check Container Runtime (if TARGET includes containers) + +```bash +# Check Podman +podman info --format '{{.Host.OS}} {{.Host.Arch}}' + +# Or Docker +docker info --format '{{.OSType}} {{.Architecture}}' + +# Check if can pull images +podman pull --quiet registry.access.redhat.com/ubi9/ubi-minimal:latest || echo "WARN: Cannot pull images" +``` + +### Step 5: Generate Report + +Present results in this format: + +```markdown +## Environment Validation Report + +### Core Tools + +| Tool | Status | Version | +|------|--------|---------| +| git | OK | 2.43.0 | +| curl | OK | 8.5.0 | +| jq | OK | 1.7.1 | +| oc | OK | 4.14.0 | +| helm | OK | 3.14.0 | +| podman | OK | 4.9.0 | +| skopeo | MISSING | - | +| ssh | OK | OpenSSH_9.6 | + +### OpenShift Cluster + +| Check | Status | Details | +|-------|--------|---------| +| Logged in | OK | user@cluster.example.com | +| Project | OK | my-project | +| Create Deployments | OK | Allowed | +| Create BuildConfigs | OK | Allowed | +| Create ImageStreams | OK | Allowed | + +### Container Runtime + +| Check | Status | Details | +|-------|--------|---------| +| Runtime | OK | Podman 4.9.0 | +| Pull images | OK | Can access registries | + +--- + +### Summary + +**Ready for:** /detect-project, /s2i-build, /deploy, /helm-deploy, /containerize-deploy + +**Missing tools for:** +- /recommend-image (dynamic mode) - Install: `sudo dnf install skopeo` + +### Quick Fix Commands + +```bash +# Install missing tools +sudo dnf install skopeo +``` +``` + +### Step 6: Offer Next Steps + +```markdown +## Next Steps + +Your environment is ready for deployment. + +Would you like to: +1. Run `/detect-project` to analyze your application +2. Run `/containerize-deploy` for end-to-end deployment +3. See detailed prerequisites documentation + +Select an option or describe what you'd like to do: +``` + +--- + +## Validation Status Indicators + +| Status | Meaning | +|--------|---------| +| OK | Tool installed and working | +| MISSING | Tool not found in PATH | +| ERROR | Tool found but not working | +| WARN | Optional tool missing | +| SKIP | Check skipped (not in scope) | + +## Error Handling + +### Tool Not Found + +```markdown +**Missing: [tool-name]** + +This tool is required for [skill-names]. + +See [docs/prerequisites.md](../../docs/prerequisites.md) for installation commands by OS. +``` + +### Cluster Connection Failed + +```markdown +**OpenShift cluster not accessible** + +You are not logged in to an OpenShift cluster. + +To connect: +1. Get login command from OpenShift console +2. Run: `oc login ` + +Or set KUBECONFIG: +```bash +export KUBECONFIG=/path/to/kubeconfig +``` +``` + +### Permission Denied + +```markdown +**Insufficient permissions in namespace [namespace]** + +You need 'edit' or 'admin' role to deploy applications. + +Options: +1. Contact cluster admin for permissions +2. Switch to a different namespace: `oc project ` +3. Create a new project: `oc new-project ` +``` + +--- + +## Dependencies + +### Required MCP Servers +- None required (uses Bash to check tool availability and cluster connectivity) + +### Related Skills +- `/containerize-deploy` - End-to-end deployment workflow (validate environment first) +- `/s2i-build` - S2I build requiring oc and cluster access +- `/deploy` - Deployment requiring oc and cluster access + +### Reference Documentation +- [docs/prerequisites.md](../../docs/prerequisites.md) - Comprehensive tool requirements by skill, installation commands, cluster access verification diff --git a/evaluation/with_skills/rh-developer__validate-environment/instruction.md b/evaluation/with_skills/rh-developer__validate-environment/instruction.md new file mode 100644 index 00000000..b9024f98 --- /dev/null +++ b/evaluation/with_skills/rh-developer__validate-environment/instruction.md @@ -0,0 +1,13 @@ +# Environment Validation Task + +You are a Red Hat developer. Before deploying a new application, you need to confirm the OpenShift environment is ready and properly configured. + +## Requirements +- Verify cluster connectivity: confirm you can reach the API server and authenticate successfully +- Check namespace readiness: does the target namespace exist, and do you have permissions to create deployments, services, and routes in it? +- Verify resource availability: are there sufficient CPU and memory quotas remaining for a new deployment? +- Produce a readiness checklist with pass/fail status for each check and an overall go/no-go recommendation + +Document your validation results and readiness assessment in `/root/report.md`. + +Use MCP tools to examine the cluster. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-developer__validate-environment/solution/solve.sh b/evaluation/with_skills/rh-developer__validate-environment/solution/solve.sh new file mode 100644 index 00000000..3cb34892 --- /dev/null +++ b/evaluation/with_skills/rh-developer__validate-environment/solution/solve.sh @@ -0,0 +1,36 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Environment Validation Report + +## Validation Scope: All +(Options: All, OpenShift, RHEL/Containers, Minimal) + +### Tool Availability +| Tool | Status | Version | +|------|--------|---------| +| git | OK | 2.43.0 | +| curl | OK | 8.5.0 | +| jq | OK | 1.7.1 | +| oc | OK | 4.14.0 | +| helm | OK | 3.14.0 | +| podman | OK | 4.9.0 | +| skopeo | OK | 1.14.0 | +| ssh | OK | OpenSSH 9.6 | + +Status indicators: OK (working), MISSING (not in PATH), WARN (optional missing). + +### OpenShift Permissions (oc auth can-i) +| Resource | Action | Status | +|----------|--------|--------| +| deployments | create | OK | +| buildconfigs | create | OK | +| imagestreams | create | OK | + +### Connectivity +- Cluster: Connected (`oc whoami` → admin) +- Podman info: `podman info --format '{{.Host.OS}} {{.Host.Arch}}'` → linux amd64 + +### Ready for +/detect-project, /s2i-build, /deploy, /helm-deploy, /containerize-deploy + +REPORT_EOF diff --git a/evaluation/with_skills/rh-developer__validate-environment/task.toml b/evaluation/with_skills/rh-developer__validate-environment/task.toml new file mode 100644 index 00000000..10df7267 --- /dev/null +++ b/evaluation/with_skills/rh-developer__validate-environment/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__validate-environment" +name = "rh-developer Environment Validation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "validate-environment", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-developer__validate-environment/tests/llm_judge.py b/evaluation/with_skills/rh-developer__validate-environment/tests/llm_judge.py new file mode 100644 index 00000000..3545ef59 --- /dev/null +++ b/evaluation/with_skills/rh-developer__validate-environment/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "skopeo_validation", + "file": "/root/report.md", + "question": "Does the report validate skopeo as a required tool in the environment?", + "reference": "A skilled report checks that skopeo is installed and available, as it is needed for image recommendation workflows. An unskilled report skips skopeo validation." + }, + { + "id": "rbac_permission_checks", + "file": "/root/report.md", + "question": "Does the report verify that the user has permissions to create deployments, buildconfigs, or imagestreams in the target namespace?", + "reference": "A skilled report checks create permissions for deployments, buildconfigs, and imagestreams — either via 'oc auth can-i' commands or via MCP/API queries that verify the same permissions. An unskilled report only checks identity (oc whoami) without verifying specific resource permissions." + }, + { + "id": "structured_validation_report", + "file": "/root/report.md", + "question": "Does the report present environment validation results in a structured pass/fail format with remediation for failures?", + "reference": "A skilled report presents each check as pass/fail with an overall go/no-go assessment and remediation steps. An unskilled report lists tools without structured evaluation." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-developer__validate-environment/tests/test.sh b/evaluation/with_skills/rh-developer__validate-environment/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-developer__validate-environment/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-developer__validate-environment/tests/test_outputs.py b/evaluation/with_skills/rh-developer__validate-environment/tests/test_outputs.py new file mode 100644 index 00000000..8f62b808 --- /dev/null +++ b/evaluation/with_skills/rh-developer__validate-environment/tests/test_outputs.py @@ -0,0 +1,67 @@ +""" +Tests for rh-developer__validate-environment per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_environment(self): + content = read_report().lower() + assert any(t in content for t in ["environment", "cluster", "ready", "validation"]), ( + "report should mention environment validation" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 100, "report should have substantial content" + + +class TestSkillDependent: + def test_skopeo_as_required_tool(self): + """Skill teaches skopeo is a required dependency for image recommendation flows. + Without skill, agents skip skopeo in environment validation.""" + c = read_report().lower() + assert "skopeo" in c, ( + "should validate skopeo as a required tool" + ) + + def test_oc_auth_can_i_checks(self): + """Skill teaches oc auth can-i create deployments/buildconfigs/imagestreams + for permission checks. Without skill, agents only check oc whoami.""" + c = read_report().lower() + has_permission_method = ("auth can-i" in c or "can-i" in c or "permission" in c) + has_resource_type = any(t in c for t in [ + "deployment", "buildconfig", "imagestream", "create" + ]) + assert has_permission_method and has_resource_type, ( + "should verify create permissions for deployments/buildconfigs/imagestreams" + ) + + def test_tool_version_checks(self): + """Skill teaches checking version/availability of oc, helm, podman, git.""" + c = read_report().lower() + tools = ["oc", "helm", "podman", "git", "skopeo"] + mentioned = sum(1 for t in tools if t in c) + assert mentioned >= 3, "should validate multiple CLI tools" + + def test_structured_pass_fail(self): + """Skill teaches presenting results as pass/fail per check.""" + c = read_report().lower() + assert any(t in c for t in ["pass", "fail", "missing", "go", "no-go", "available"]) and any(t in c for t in [ + "tool", "check", "oc", "helm", "result" + ]), "should provide structured pass/fail validation report" diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/Dockerfile b/evaluation/with_skills/rh-sre__cve-impact/environment/Dockerfile new file mode 100644 index 00000000..484ebb33 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/Dockerfile @@ -0,0 +1,52 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + + + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..1092dd1d --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- pass rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/with_skills/rh-sre__cve-impact/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..fe5d549c --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/SKILL.md b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/SKILL.md new file mode 100644 index 00000000..9671bc2d --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/SKILL.md @@ -0,0 +1,409 @@ +--- +name: cve-impact +description: | + **CRITICAL**: Use for ALL CVE discovery and listing. DO NOT call get_cves directly. + + Use when: "show critical CVEs", "CVEs on hostname X", "remediatable vulnerabilities", "impact of CVE-X", risk assessment. + + NOT for remediation (use `/remediation`). + + System-level: FIRST reply = pagination prompt (Step -1). Parsing: references/01-cve-response-parser.py. +--- + +# CVE Impact Analysis Skill + +This skill helps SREs analyze CVE vulnerabilities to understand their impact on systems before creating remediation playbooks. + +**🚨 SYSTEM-LEVEL (CVEs on device X)**: Your **first reply** to the user MUST be the pagination prompt (Step -1). Do NOT call `inventory__find_host_by_name` or `vulnerability__get_system_cves` until the user responds. Do not validate MCP or resolve hostname first—HITL comes first. + +**Integration with Remediation Skill**: The `/remediation` skill orchestrates this skill as part of its Step 1 (Impact Analysis) workflow for complex remediation scenarios. For simple standalone impact analysis, you can invoke this skill directly. + +## Invocation Note (Host-Specific) + +When invoked by another skill (e.g. remediation), use the Skill tool—do NOT use "Task Output" with the skill name as task ID. That causes "No task found with ID: cve-impact". See [skill-invocation.md](../../docs/references/skill-invocation.md). + +## Prerequisites + +**Required MCP Servers**: `lightspeed-mcp` ([setup guide](https://console.redhat.com/)) + +**Required MCP Tools**: +- `get_cves` (from lightspeed-mcp) - List/query CVEs by severity +- `get_cve` (from lightspeed-mcp) - Get specific CVE details +- `get_cve_systems` (from lightspeed-mcp) - Find systems affected by CVEs +- `get_system_cves` (from lightspeed-mcp) - List CVEs affecting a specific system (uses `system_uuid` only) + +**Required Environment Variables**: +- `LIGHTSPEED_CLIENT_ID` - Red Hat Lightspeed service account client ID +- `LIGHTSPEED_CLIENT_SECRET` - Red Hat Lightspeed service account secret + +### Prerequisite Validation + +**CRITICAL**: Before executing any operations, execute the `/mcp-lightspeed-validator` skill to verify MCP server availability. + +**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-lightspeed-validator/SKILL.md#validation-freshness-policy). + +**How to invoke**: Execute the `/mcp-lightspeed-validator` skill + +**Handle validation result**: +- **If validation PASSED**: Continue with CVE impact analysis +- **If validation PARTIAL**: Warn user and ask to proceed +- **If validation FAILED**: Stop execution, provide setup instructions + +## When to Use This Skill + +**Use this skill directly when you need**: +- Standalone CVE impact analysis without remediation planning +- Quick risk assessment for a single CVE +- Understanding which systems are affected by a CVE +- CVE severity assessments for change management documentation +- Risk assessment reports for management + +**Use the `/remediation` skill when you need**: +- CVE analysis followed by remediation playbook generation +- Complex workflows involving multiple CVEs and systems +- Integrated risk assessment + remediation planning + execution guidance +- Batch remediation across infrastructure +- End-to-end CVE management (analysis → validation → remediation → verification) + +**To invoke**: Execute the `/remediation` skill + +**How they work together**: The `/remediation` skill orchestrates this skill as part of its comprehensive workflow, combining impact analysis with context gathering, playbook generation, and execution guidance. + +## Workflow + +### Step -1: System-Level Gate — HITL FIRST (MANDATORY) + +**If the user asked for CVEs on a device** (e.g. "CVEs on ip-172-31-32-201", "remediatable CVEs on hostname X", "most critical CVEs on system Y"): + +**Your first response to the user MUST be the pagination prompt below. Do not run Step 0, do not call `inventory__find_host_by_name`, do not call `vulnerability__get_system_cves` until the user responds.** + +Reply to the user with: + +``` +To fetch remediatable CVEs on this system, I will: +- Paginate through vulnerability__get_system_cves (limit=100 per page) +- Filter each page for advisory_available === true +- Systems often have 1,700+ CVEs (~18 API calls) + +⚠️ First page only often returns 0 remediatable CVEs—they may be on any page. For "remediatable" queries, recommend "all pages". + +Options: +- **First page only**: Fetch 100 CVEs, filter for remediatable (may be 0) +- **All pages**: Fetch until no more results (recommended for remediatable) +- **N pages**: Fetch up to N pages (e.g. "3 pages" = up to 300 CVEs scanned) + +How would you like to proceed? (first page / all pages / N pages) +``` + +**Wait for the user to respond.** Only after they reply may you proceed to Step 0. + +**If account-level** (e.g. "CVEs on my account"): Skip this step, go to Step 0. + +--- + +### Step 0: Validate Lightspeed MCP Prerequisites + +**Action**: Execute the `/mcp-lightspeed-validator` skill + +**Note**: Can skip if validation was performed earlier in this session and succeeded. See [Validation Freshness Policy](../mcp-lightspeed-validator/SKILL.md#validation-freshness-policy). + +**How to invoke**: Execute the `/mcp-lightspeed-validator` skill + +**Handle validation result**: +- **If validation PASSED**: Continue to Step 1 +- **If validation PARTIAL**: Warn user and ask to proceed +- **If validation FAILED**: Stop execution, user must set up MCP server + +### Step 1: CVE Discovery — Choose Flow + +Select the appropriate flow based on user request. + +| Flow | When to Use | Flow File | +|------|-------------|-----------| +| **Account-level** | devices=all (account-wide CVEs) | [01-account-cves.md](flows/01-account-cves.md) | +| **System-level (all CVEs)** | devices=selected, remediation=does not matter | [02-system-all-cves.md](flows/02-system-all-cves.md) | +| **System-level (remediatable)** | devices=selected, remediation=available | [03-system-remediatable-cves.md](flows/03-system-remediatable-cves.md) | + +**Variable mapping**: +- **devices**: all → account flow; selected → system flow +- **severities**: all, most critical, or selected → parameter/filter in each flow +- **remediation**: available → remediatable flow; does not matter → all-cves flow + +--- + +#### CRITICAL: System-Level — HITL FIRST (Before Any Other Action) + +**For system-level flows (02 or 03)**: Your **first** action MUST be to display the HITL prompt below and **wait for user confirmation**. Do NOT resolve hostname, do NOT call any MCP tool, until the user responds. + +**Order of operations**: +1. **STOP. Display HITL prompt. Wait for user.** +2. Only after user confirms → document consultation → resolve hostname → call `vulnerability__get_system_cves` + +*For remediatable CVEs on system (flow 03):* +``` +To fetch remediatable CVEs on this system, I will: +- Paginate through vulnerability__get_system_cves (limit=100 per page) +- Filter each page for advisory_available === true +- Systems often have 1,700+ CVEs (~18 API calls) + +⚠️ First page only often returns 0 remediatable CVEs—they may be on any page. For "remediatable" queries, recommend "all pages". + +Options: +- **First page only**: Fetch 100 CVEs, filter for remediatable (may be 0) +- **All pages**: Fetch until no more results (recommended for remediatable) +- **N pages**: Fetch up to N pages (e.g. "3 pages" = up to 300 CVEs scanned) + +How would you like to proceed? (first page / all pages / N pages) +``` + +*For all CVEs on system (flow 02):* +``` +This system may have many CVEs. I will paginate through vulnerability__get_system_cves (limit=100 per page). + +Options: +- **First page only**: Fetch 100 CVEs, then stop (quick overview) +- **All pages**: Fetch until no more results (systems with 1,700+ CVEs may require ~18 API calls) +- **N pages**: Fetch up to N pages (e.g. "3 pages" = 300 CVEs) + +How would you like to proceed? (first page / all pages / N pages) +``` + +**Handle response**: Wait for user reply. Only after user confirms (and specifies strategy) may you proceed to resolve hostname and call `vulnerability__get_system_cves`. If user says "no" or cancels, stop execution. + +**Anti-pattern**: Do NOT call `vulnerability__get_system_cves` or `inventory__find_host_by_name` before completing HITL. Calling with only the first page (limit=100, no offset loop) misses remediatable CVEs on later pages. + +--- + +**Action**: Read and follow the selected flow file. For system-level, HITL is Step 1 (before all other steps). + +### Step 2: CVE Information Retrieval (For Specific CVE Analysis) + +**CRITICAL**: Document consultation MUST happen BEFORE tool invocation. + +**Document Consultation** (REQUIRED - Execute FIRST): +1. **Action**: Read [cvss-scoring.md](../../docs/references/cvss-scoring.md) using the Read tool to understand CVSS score interpretation and severity mapping +2. **Output to user**: "I consulted [cvss-scoring.md](../../docs/references/cvss-scoring.md) to understand CVSS score interpretation and severity mapping." + +**MCP Tool**: `get_cve` or `vulnerability__get_cve` (from lightspeed-mcp) + +**Parameters**: +- `cve_id`: Exact CVE identifier from user query (format: `"CVE-YYYY-NNNNN"`) + - Example: `"CVE-2024-1234"` +- `include_details`: `true` (retrieve complete metadata including CVSS vector, affected packages, references) + +**Expected Output**: Comprehensive CVE metadata including CVSS score, severity, attack vector, affected packages, remediation status + +Retrieve comprehensive CVE metadata: +``` +CVE ID: CVE-YYYY-NNNNN +CVSS Score: X.X (Base score from NIST) +Severity: Critical/High/Medium/Low +Attack Vector: Network/Adjacent/Local/Physical +Attack Complexity: Low/High +Privileges Required: None/Low/High +User Interaction: None/Required +Scope: Unchanged/Changed +Confidentiality Impact: None/Low/High +Integrity Impact: None/Low/High +Availability Impact: None/Low/High + +Description: [CVE description from NVD/Red Hat] +Affected Packages: [List of packages and versions] +Published Date: YYYY-MM-DD +Last Modified: YYYY-MM-DD +``` + +### Step 3: Affected Systems Identification + +**CRITICAL**: Document consultation MUST happen BEFORE tool invocation. + +**Document Consultation** (REQUIRED - Execute FIRST): +1. **Action**: Read [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) using the Read tool to understand the system identification response format +2. **Output to user**: "I consulted vulnerability-logic.md to understand the system identification response format." + +**MCP Tool**: `get_cve_systems` or `vulnerability__get_cve_systems` (from lightspeed-mcp) + +**Parameters**: +- `cve_id`: Exact CVE identifier (format: `"CVE-YYYY-NNNNN"`) + - Example: `"CVE-2024-1234"` +- `include_patched`: `false` (exclude already-patched systems to focus on vulnerable systems) + +**Expected Output**: List of affected systems with UUID, hostname, IP address, package version, vulnerability status + +Identify which systems in your infrastructure are affected by the CVE: +``` +Affected Systems: +- system-uuid-1 (hostname: web-server-01, IP: 10.0.1.10) + - Package: httpd-2.4.37-1.el8 + - Status: Vulnerable + +- system-uuid-2 (hostname: web-server-02, IP: 10.0.1.11) + - Package: httpd-2.4.37-1.el8 + - Status: Vulnerable + +Total Affected Systems: N +``` + +### Step 4: System Classification + +**CRITICAL**: Document consultation MUST happen BEFORE classification logic. + +**Document Consultation** (REQUIRED - Execute FIRST): +1. **Action**: Read [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) using the Read tool to understand system tagging and classification strategies +2. **Output to user**: "I consulted vulnerability-logic.md to understand system tagging and classification strategies." + +Classify affected systems by environment and criticality based on tags from Red Hat Lightspeed: +``` +System Classification: +- Production systems: N (highest priority) +- Staging systems: M (test before prod) +- Development systems: K (lowest priority) +- Bare metal: X systems +- Virtual machines: Y systems +``` + +### Step 5: Risk Assessment + +**CRITICAL**: Document consultation MUST happen BEFORE risk assessment. + +**Document Consultation** (REQUIRED - Execute FIRST): +1. **Action**: Read [cvss-scoring.md](../../docs/references/cvss-scoring.md) using the Read tool to understand risk prioritization based on CVSS vectors +2. **Output to user**: "I consulted [cvss-scoring.md](../../docs/references/cvss-scoring.md) to understand risk prioritization based on CVSS vectors." + +Provide a risk assessment based on: +- CVSS score and severity +- Number of affected systems +- System criticality (production vs non-production) +- Attack vector and exploitability +- Known exploits in the wild + +**Output Format**: +``` +Risk Assessment Summary +━━━━━━━━━━━━━━━━━━━━━━━ + +Overall Risk Level: Critical/High/Medium/Low + +Factors: +✓ CVSS Score: X.X (Severity: High) +✓ Affected Systems: N systems +✓ Environment: Production (Critical) +✓ Attack Vector: Network (Remote exploitable) +✓ Exploitability: Proof-of-concept exists +✓ User Interaction: None required + +Recommendation: Immediate remediation recommended +Priority: P0 (within 24 hours) / P1 (within 7 days) / P2 (within 30 days) +``` + +### Step 6: Impact Analysis + +Analyze business impact (service, confidentiality/integrity/availability, compliance). + +### Step 7: Remediation Readiness Check + +Check if automated playbook or manual steps are available. + +## Output and Examples + +**Read [references/03-output-templates.md](references/03-output-templates.md)** for report format. +**Read [references/04-examples.md](references/04-examples.md)** for query-type examples and remediation integration. + +## Error Handling + +**Read [references/05-error-handling.md](references/05-error-handling.md)** for CVE not found, no affected systems, and Lightspeed tool failures. + +## Reference Files + +| File | Use When | +|------|----------| +| [01-cve-response-parser.py](references/01-cve-response-parser.py) | Parse/filter MCP vulnerability responses | +| [02-cve-parsing-guide.md](references/02-cve-parsing-guide.md) | Parser invocation, filter options | +| [03-output-templates.md](references/03-output-templates.md) | Report format | +| [04-examples.md](references/04-examples.md) | Query-type examples | +| [05-error-handling.md](references/05-error-handling.md) | CVE not found, no systems, Lightspeed failures | +| [lightspeed-mcp-tool-failures.md](../../docs/references/lightspeed-mcp-tool-failures.md) | explain_cves dnf_modules workaround | + +## Parsing MCP Responses + +**REQUIRED**: Use the skill's parser script for all vulnerability response parsing. Do NOT use jq, inline Python, or other ad-hoc JSON parsing. + +**Do NOT generate inline Python** to aggregate multiple page files—the parser accepts multiple file paths and produces aggregated reports. + +**Read** [references/02-cve-parsing-guide.md](references/02-cve-parsing-guide.md) for: +- Parser location: `references/01-cve-response-parser.py` +- Single page: `python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py ` +- Multiple pages: `python3 .../01-cve-response-parser.py page1.json page2.json page3.json ...` (merges, dedupes, aggregated report) +- Filter options: `FILTER_REMEDIATABLE=1`, `FILTER_IMPACT=Critical,Important` +- Report format: `OUTPUT=report`, `SYSTEM_NAME=hostname` for aggregated multi-page reports + +Save each MCP tool result to a file, then run the parser with one or more paths. Use parser output for summary tables and reports. + +## Best Practices + +1. **Always start with risk assessment** before deciding on remediation +2. **Check for known exploits** using CVE description and references +3. **Consider business impact** not just CVSS scores +4. **Group related CVEs** for batch remediation when possible +5. **Document findings** for compliance and audit purposes +6. **Test in staging first** for high-impact changes + +## Dependencies + +### Required MCP Servers +- `lightspeed-mcp` - Red Hat Lightspeed platform access + +### Required MCP Tools +- `get_cves` (from lightspeed-mcp) - List/query CVEs by severity and filters + - Parameters: severity (array), sort_by (string), limit (number) + - Returns: List of CVEs with CVSS scores, severity, affected systems count + +- `get_cve` (from lightspeed-mcp) - Get specific CVE details + - Parameters: cve_id (string, format CVE-YYYY-NNNNN), include_details (boolean) + - Returns: Complete CVE metadata with CVSS vector, affected packages, references + +- `get_cve_systems` (from lightspeed-mcp) - Find systems affected by CVE + - Parameters: cve_id (string), include_patched (boolean) + - Returns: List of affected systems with UUID, hostname, package version, status + +- `get_system_cves` (from lightspeed-mcp) - List CVEs affecting a specific system + - Parameters: **system_uuid** (string, required) - use `system_uuid`, NOT `system_id` + - Does NOT support: impact, limit, severity filters - filter results client-side + - Returns: List of CVEs affecting the system + +### Related Skills +- `mcp-lightspeed-validator` - **PREREQUISITE** - Validates Lightspeed MCP server before operations + - Use before: ALL cve-impact operations (Step 0 in workflow) + - Purpose: Ensures MCP server is available before attempting tool calls + +- `cve-validation` - Validate CVE IDs before impact analysis + - Use before: Step 2 if CVE format/existence is uncertain + - Purpose: Confirms CVE is valid and remediable before expensive analysis + +- `system-context` - Get detailed system configuration after identifying affected systems + - Use after: Step 3 when deeper system investigation is needed + - Purpose: Understand deployment architecture for remediation planning + +- `fleet-inventory` - Get comprehensive fleet information before CVE analysis + - Use before: Step 1 when starting from fleet discovery + - Purpose: Understand overall infrastructure before assessing CVE impact + +### Reference Documentation +- [cvss-scoring.md](../../docs/references/cvss-scoring.md) - CVSS score interpretation and severity mapping +- [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) - CVE severity classification and filtering +- [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) - CVE severity classification, system identification, and filtering +- [references/02-cve-parsing-guide.md](references/02-cve-parsing-guide.md) - Parse MCP vulnerability responses; use the parser script instead of generating inline Python + +## Tools Reference + +This skill primarily uses: +- `get_cve` (vulnerability toolset) - Get details about a specific CVE +- `get_cve_systems` (vulnerability toolset) - Get list of systems affected by a CVE +- `get_cves` (vulnerability toolset) - Get list of all CVEs affecting the account (optional) +- `get_system_cves` (vulnerability toolset) - Get list of CVEs affecting a specific system + - **CRITICAL**: Use `system_uuid` (required), NOT `system_id` + - Does NOT support `impact`, `limit`, or severity filters - filter client-side +- `inventory__find_host_by_name` (inventory toolset) - Resolve hostname to system UUID before get_system_cves +- `get_host_details` (inventory toolset) - Get detailed system information (optional) + +All tools are provided by the lightspeed-mcp MCP server configured in `.mcp.json`. diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/01-account-cves.md b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/01-account-cves.md new file mode 100644 index 00000000..d981a9e9 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/01-account-cves.md @@ -0,0 +1,92 @@ +# Flow: Account-Level CVEs + +**Scope**: devices=all (account-wide) | severities=all, most critical, or selected | remediation=available or does not matter + +**Tool**: `vulnerability__get_cves` (single request; no offset pagination in API) + +## When to Use + +- "What are the critical vulnerabilities on my account?" +- "Show me high-severity CVEs" +- "List all vulnerabilities affecting my account" +- "Which CVEs can I remediate?" (account-wide) + +## Step 1: Document Consultation (REQUIRED - Execute FIRST) + +1. **Action**: Read [vulnerability-logic.md](../../../docs/insights/vulnerability-logic.md) using the Read tool +2. **Output to user**: "I consulted [vulnerability-logic.md](../../../docs/insights/vulnerability-logic.md) to understand CVE severity classification and filtering." + +## Step 2: HITL Checkpoint — Pagination / Limit + +**Before** calling the API, display and wait for confirmation: + +``` +For account-level CVEs, I will fetch up to {limit} CVEs per request. +The API returns a single page (no offset pagination). + +Options: +- Default: limit=20 (top CVEs by CVSS score) +- You may request a different limit (e.g. 10, 50) before I proceed + +Proceed with limit=20? (yes/no) Or specify a different limit. +``` + +**Handle response**: +- **yes** or no limit specified → use `limit=20` +- **User specifies N** → use `limit=N` +- **no** → Stop execution + +## Step 3: MCP Tool Invocation + +**Tool**: `vulnerability__get_cves` (from lightspeed-mcp) + +**Parameters**: + +| Parameter | Severity=all | Severity=most critical | Severity=selected | +|-----------|-------------|------------------------|-------------------| +| `impact` | `"7,6,5,4"` | `"7,6"` | e.g. `"7"` (Critical only) or `"6,5"` | + +| Parameter | Remediation=available | Remediation=does not matter | +|-----------|----------------------|------------------------------| +| `advisory_available` | `"true"` | `"true,false"` | + +**Common parameters**: +- `sort`: `"-cvss_score"` (descending by CVSS) +- `limit`: From HITL (default 20) + +**Example** (most critical, remediatable): +``` +vulnerability__get_cves( + impact="7,6", + sort="-cvss_score", + limit=20, + advisory_available="true" +) +``` + +**Example** (all severities, remediation doesn't matter): +``` +vulnerability__get_cves( + impact="7,6,5,4", + sort="-cvss_score", + limit=20, + advisory_available="true,false" +) +``` + +## Step 4: After Listing + +- **Parse response** (if needed): Use [references/01-cve-response-parser.py](../references/01-cve-response-parser.py). Do NOT use jq or inline Python. See [02-cve-parsing-guide.md](../references/02-cve-parsing-guide.md). +- Sort by CVSS score (highest first) or by affected system count +- Provide summary table: CVE ID, severity, affected systems count, remediation availability +- Offer to analyze a specific CVE (see [SKILL.md](../SKILL.md) — Step 2: CVE Information Retrieval) +- Offer to create remediation plan (invoke `/remediation` skill) + +## Impact Level Reference + +| impact | Severity | +|--------|----------| +| 7 | Critical | +| 6 | High | +| 5 | Important | +| 4 | Moderate | diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/02-system-all-cves.md b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/02-system-all-cves.md new file mode 100644 index 00000000..6a819454 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/02-system-all-cves.md @@ -0,0 +1,89 @@ +# Flow: System-Level CVEs (All — Remediation Does Not Matter) + +**Scope**: devices=selected | severities=all, most critical, or selected | remediation=does not matter + +**Tool**: `vulnerability__get_system_cves` (paginated via `limit` + `offset`) + +**BLOCKING**: HITL (Step -1 in SKILL.md) MUST be your first action. Reply to the user with the pagination prompt before ANY tool call. Do NOT proceed to this flow until the user has responded to the HITL prompt. + +## When to Use + +- "What CVEs affect hostname X?" +- "What vulnerabilities are on system Y?" +- "Show CVEs for ip-172-31-32-201.eu-west-3.compute.internal" +- (When user does NOT specifically ask for remediatable-only) + +## Step 1: HITL — Pagination Strategy (Done in SKILL.md Step -1) + +**If you have not yet replied to the user with the pagination prompt**: Stop. Go to [SKILL.md](../SKILL.md) Step -1. Your first reply to the user must be the prompt below. Do NOT call `inventory__find_host_by_name` or `vulnerability__get_system_cves` until the user responds. + +``` +This system may have many CVEs. I will paginate through vulnerability__get_system_cves (limit=100 per page). + +Options: +- **First page only**: Fetch 100 CVEs, then stop (quick overview) +- **All pages**: Fetch until no more results (systems with 1,700+ CVEs may require ~18 API calls) +- **N pages**: Fetch up to N pages (e.g. "3 pages" = 300 CVEs) + +How would you like to proceed? (first page / all pages / N pages) +``` + +**Handle response**: +- **first page** → Single call: `limit=100`, `offset=0`; stop after first response +- **all pages** → Loop: `offset=0, 100, 200, ...` until `len(data) < 100` or empty +- **N pages** → Loop N times: `offset=0`, then `offset=100`, ... up to N pages +- **no** or cancel → Stop execution + +## Step 2: Document Consultation (REQUIRED - Execute AFTER HITL) + +1. **Action**: Read [insights-api.md](../../../docs/insights/insights-api.md) using the Read tool +2. **Output to user**: "I consulted [insights-api.md](../../../docs/insights/insights-api.md) to understand system identification." + +## Step 3: Resolve Hostname to System UUID + +**If user provided hostname** (not UUID): +- **Tool**: `inventory__find_host_by_name` (preferred) or `inventory__list_hosts` +- **inventory__list_hosts**: Use `per_page` (integer), NOT `page_size`; pass `display_name=""` if no filter +- If multiple matches: ask user to disambiguate or use first match with a note + +## Step 4: MCP Tool Invocation + +**Tool**: `vulnerability__get_system_cves` (from lightspeed-mcp) + +**Parameters**: +- `system_uuid`: Required (from Step 2) +- `limit`: `100` (fewer pages) +- `offset`: `0`, `100`, `200`, ... per pagination strategy + +**First call** (to get total estimate if available): +``` +vulnerability__get_system_cves(system_uuid="", limit=100, offset=0) +``` +Check `meta.count` in response for total estimate. + +**Pagination loop** (if user chose "all pages" or "N pages"): +``` +offset = 0 +all_cves = [] +while (strategy allows): + result = vulnerability__get_system_cves(system_uuid="...", limit=100, offset=offset) + all_cves.extend(result.data) + if len(result.data) < 100: break + offset += 100 +``` + +## Step 5: Filter by Severity (Client-Side) + +`get_system_cves` does NOT support severity filters. Filter results client-side: + +| Severity choice | Filter | +|-----------------|--------| +| all | No filter | +| most critical | Keep items where severity in (Critical, High) | +| selected | Keep items matching user-specified severity | + +## Step 6: After Retrieval + +- **Parse response**: Use [references/01-cve-response-parser.py](../references/01-cve-response-parser.py). Do NOT use jq or inline Python. See [02-cve-parsing-guide.md](../references/02-cve-parsing-guide.md). +- Sort by CVSS score (highest first) +- Provide summary table; offer to analyze specific CVEs or create remediation plan (`/remediation` skill) diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/03-system-remediatable-cves.md b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/03-system-remediatable-cves.md new file mode 100644 index 00000000..a1351e79 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/03-system-remediatable-cves.md @@ -0,0 +1,96 @@ +# Flow: System-Level CVEs (Remediatable Only) + +**Scope**: devices=selected | severities=all, most critical, or selected | remediation=available + +**Tool**: `vulnerability__get_system_cves` (paginated; filter client-side for `advisory_available === true`) + +**BLOCKING**: HITL (Step -1 in SKILL.md) MUST be your first action. Reply to the user with the pagination prompt before ANY tool call. Do NOT proceed to this flow until the user has responded to the HITL prompt. + +## When to Use + +- "Remediatable CVEs on system X" +- "CVEs with available remediation on device Y" +- "Which CVEs can I fix on hostname Z?" + +**CRITICAL**: `get_system_cves` does NOT support `advisory_available` as a request parameter. We must paginate through ALL CVEs and filter client-side for `attributes.advisory_available === true`. Do NOT use `get_cves` + `get_cve_systems` per CVE—does not scale. + +## Step 1: HITL — Pagination Strategy (Done in SKILL.md Step -1) + +**If you have not yet replied to the user with the pagination prompt**: Stop. Go to [SKILL.md](../SKILL.md) Step -1. Your first reply to the user must be the prompt below. Do NOT call `inventory__find_host_by_name` or `vulnerability__get_system_cves` until the user responds. + +``` +To fetch remediatable CVEs on this system, I will: +- Paginate through vulnerability__get_system_cves (limit=100 per page) +- Filter each page for advisory_available === true +- Systems often have 1,700+ CVEs (~18 API calls) + +⚠️ First page only often returns 0 remediatable CVEs—they may be on any page. For "remediatable" queries, recommend "all pages". + +Options: +- **First page only**: Fetch 100 CVEs, filter for remediatable (may be 0) +- **All pages**: Fetch until no more results (recommended for remediatable) +- **N pages**: Fetch up to N pages (e.g. "3 pages" = up to 300 CVEs scanned) + +How would you like to proceed? (first page / all pages / N pages) +``` + +**Handle response**: +- **first page** → Single call: `limit=100`, `offset=0`; filter for remediatable; stop +- **all pages** → Loop until empty; filter each page for remediatable +- **N pages** → Loop N times; filter each page for remediatable +- **no** or cancel → Stop execution + +## Step 2: Document Consultation (REQUIRED - Execute AFTER HITL) + +1. **Action**: Read [insights-api.md](../../../docs/insights/insights-api.md) using the Read tool +2. **Output to user**: "I consulted [insights-api.md](../../../docs/insights/insights-api.md) to understand system identification." + +## Step 3: Resolve Hostname to System UUID + +**If user provided hostname** (not UUID): +- **Tool**: `inventory__find_host_by_name` (preferred) or `inventory__list_hosts` +- **inventory__list_hosts**: Use `per_page` (integer), NOT `page_size`; pass `display_name=""` if no filter +- If multiple matches: ask user to disambiguate or use first match with a note + +## Step 4: MCP Tool Invocation + +**Tool**: `vulnerability__get_system_cves` (from lightspeed-mcp) + +**Parameters**: +- `system_uuid`: Required (from Step 2) +- `limit`: `100` +- `offset`: `0`, `100`, `200`, ... per pagination strategy + +**First call** (to get total estimate if available): +``` +vulnerability__get_system_cves(system_uuid="", limit=100, offset=0) +``` + +**Pagination loop** (filter for remediatable): +``` +offset = 0 +all_remediatable = [] +while (strategy allows): + result = vulnerability__get_system_cves(system_uuid="...", limit=100, offset=offset) + for item in result.data: + if item.attributes.advisory_available is True: + all_remediatable.append(item) + if len(result.data) < 100: break + offset += 100 +``` + +## Step 5: Filter by Severity (Client-Side) + +After filtering for remediatable, optionally filter by severity: + +| Severity choice | Filter | +|-----------------|--------| +| all | No additional filter | +| most critical | Keep items where severity in (Critical, High) | +| selected | Keep items matching user-specified severity | + +## Step 6: After Retrieval + +- **Parse response**: Use [references/01-cve-response-parser.py](../references/01-cve-response-parser.py) with `FILTER_REMEDIATABLE=1`. Do NOT use jq or inline Python. See [02-cve-parsing-guide.md](../references/02-cve-parsing-guide.md). +- Sort by CVSS score (highest first) +- Provide summary table; offer to analyze specific CVEs or create remediation plan (`/remediation` skill) diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/01-cve-response-parser.py b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/01-cve-response-parser.py new file mode 100644 index 00000000..d9235f25 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/01-cve-response-parser.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +""" +Parse Red Hat Lightspeed vulnerability MCP tool responses. + +Handles: vulnerability__get_system_cves, vulnerability__get_cves + +Usage: + python3 01-cve-response-parser.py < response.json + python3 01-cve-response-parser.py /path/to/response.json + python3 01-cve-response-parser.py page1.json page2.json page3.json # Multiple pages → aggregated report + +Options (env vars or args): + FILTER_REMEDIATABLE=1 - Only CVEs with advisory_available=true + FILTER_IMPACT=Critical,Important - Only these severities (comma-separated) + SORT_BY=cvss|impact|public_date + OUTPUT=summary|table|json|report + SYSTEM_NAME=hostname - For report header (when OUTPUT=report) + PAGES_SCANNED=5 - For report header (when OUTPUT=report, multiple files) +""" +import json +import sys +import os +from pathlib import Path + +# Response structure: {"result": {"data": [...]}, "meta": {"count": N}} +# Each CVE: {"id": "CVE-...", "type": "cve", "url": "...", "attributes": {...}} +# attributes: advisory_available, impact, cvss3_score, cvss2_score, description, synopsis, public_date, business_risk, etc. + +IMPACT_ORDER = {"Critical": 0, "High": 1, "Important": 2, "Moderate": 3, "Low": 4, "None": 5} + + +def load_response(src): + """Load JSON from file path or stdin.""" + if src and src != "-": + with open(src, "r") as f: + return json.load(f) + return json.load(sys.stdin) + + +def extract_cves(data): + """Extract CVE list from Lightspeed response (handles result.data or result.results).""" + result = data.get("result", data) + cves = result.get("data", result.get("results", [])) + meta = data.get("meta", {}) + total = meta.get("count", meta.get("total", len(cves))) + return cves, total + + +def get_attr(cve, key, default=None): + """Get attribute from CVE (handles nested attributes).""" + attrs = cve.get("attributes", cve) + return attrs.get(key, attrs.get(key.replace("_", "-"), default)) + + +def filter_cves(cves, remediatable_only=None, impact_filter=None): + """Filter CVEs by advisory_available and/or impact.""" + filtered = [] + for cve in cves: + if remediatable_only and not get_attr(cve, "advisory_available", False): + continue + if impact_filter: + impact = get_attr(cve, "impact", "") + if impact not in impact_filter: + continue + filtered.append(cve) + return filtered + + +def sort_cves(cves, sort_by="cvss"): + """Sort CVEs by cvss (desc), impact, or public_date.""" + def key_fn(cve): + if sort_by == "cvss": + score = get_attr(cve, "cvss3_score") or get_attr(cve, "cvss2_score") or "0" + return -(float(score) if score else 0) + if sort_by == "impact": + return IMPACT_ORDER.get(get_attr(cve, "impact", "None"), 99) + if sort_by == "public_date": + return get_attr(cve, "public_date", "") or "" + return 0 + return sorted(cves, key=key_fn) + + +def format_summary(cves, total_in_api, remediatable_only=False, impact_filter=None): + """Print summary counts by impact and remediation.""" + by_impact = {} + by_remediation = {"with_remediation": 0, "without": 0} + for cve in cves: + impact = get_attr(cve, "impact", "None") or "None" + by_impact[impact] = by_impact.get(impact, 0) + 1 + if get_attr(cve, "advisory_available", False): + by_remediation["with_remediation"] += 1 + else: + by_remediation["without"] += 1 + + lines = [ + "CVE Response Summary", + "=" * 60, + f"Total in this page/batch: {len(cves)}", + f"Total in API (meta.count): {total_in_api}", + ] + if remediatable_only or impact_filter: + lines.append(f"Filter: remediatable={remediatable_only}, impact={impact_filter}") + lines.append("") + lines.append("By Impact:") + for impact in ["Critical", "Important", "High", "Moderate", "Low", "None"]: + if impact in by_impact: + lines.append(f" {impact}: {by_impact[impact]}") + lines.append("") + lines.append("By Remediation:") + lines.append(f" With advisory: {by_remediation['with_remediation']}") + lines.append(f" Without: {by_remediation['without']}") + return "\n".join(lines) + + +def format_table(cves, limit=20): + """Print CVE table (CVE ID, CVSS, Impact, Remediation).""" + lines = [ + "CVE ID | CVSS | Impact | Remediation", + "-" * 60, + ] + for cve in cves[:limit]: + cve_id = cve.get("id", "?") + cvss = get_attr(cve, "cvss3_score") or get_attr(cve, "cvss2_score") or "-" + impact = get_attr(cve, "impact", "-") or "-" + rem = "Yes" if get_attr(cve, "advisory_available", False) else "No" + lines.append(f"{cve_id:<19} | {str(cvss):<6} | {impact:<9} | {rem}") + if len(cves) > limit: + lines.append(f"... and {len(cves) - limit} more") + return "\n".join(lines) + + +def format_report(cves, total_in_api, system_name=None, pages_scanned=None): + """Print aggregated report format (for multi-page results).""" + lines = ["=" * 80] + title = "CVEs WITH AVAILABLE REMEDIATION" + if system_name: + title = f"{title.upper()} — System: {system_name}" + lines.append(title) + if pages_scanned: + lines.append(f"Scanned: {pages_scanned} page(s)") + lines.append("=" * 80) + lines.append("") + if not cves: + lines.append("No CVEs with available remediation found.") + if pages_scanned: + lines.append("") + lines.append("Note: Try scanning more pages or use FILTER_IMPACT=Critical,Important for severity filter.") + else: + lines.append(f"Found {len(cves)} CVE(s) with available remediation:\n") + for i, cve in enumerate(cves, 1): + cve_id = cve.get("id", "?") + cvss = get_attr(cve, "cvss3_score") or get_attr(cve, "cvss2_score") or "-" + impact = get_attr(cve, "impact", "-") or "-" + synopsis = get_attr(cve, "synopsis", "") or cve_id + url = cve.get("url", "") + lines.append(f"{i}. CVE ID: {cve_id}") + lines.append(f" CVSS v3 Score: {cvss}") + lines.append(f" Severity: {impact}") + lines.append(f" Synopsis: {synopsis}") + if url: + lines.append(f" View in Insights: {url}") + lines.append("") + return "\n".join(lines) + + +def load_and_merge_files(paths): + """Load multiple JSON files, extract CVEs, merge and dedupe by id.""" + all_cves = {} + max_total = 0 + for p in paths: + if not os.path.exists(p): + continue + with open(p, "r") as f: + try: + data = json.load(f) + except json.JSONDecodeError: + continue + cves, total = extract_cves(data) + max_total = max(max_total, total) + for cve in cves: + cid = cve.get("id") + if cid and cid not in all_cves: + all_cves[cid] = cve + return list(all_cves.values()), max_total + + +def main(): + # Parse args — multiple files = aggregated multi-page mode + paths = [a for a in sys.argv[1:] if not a.startswith("-")] + if not paths and not sys.stdin.isatty(): + paths = ["-"] + + remediatable_only = os.environ.get("FILTER_REMEDIATABLE", "").lower() in ("1", "true", "yes") + impact_str = os.environ.get("FILTER_IMPACT", "") + impact_filter = [s.strip() for s in impact_str.split(",") if s.strip()] if impact_str else None + sort_by = os.environ.get("SORT_BY", "cvss") + output = os.environ.get("OUTPUT", "report" if len(paths) > 1 else "summary") + system_name = os.environ.get("SYSTEM_NAME", "") + pages_scanned = os.environ.get("PAGES_SCANNED", str(len(paths)) if len(paths) > 1 else None) + + if len(paths) > 1 and "-" not in paths: + cves, total = load_and_merge_files(paths) + else: + src = paths[0] if paths else "-" + data = load_response(src) + cves, total = extract_cves(data) + + cves = filter_cves(cves, remediatable_only=remediatable_only, impact_filter=impact_filter) + cves = sort_cves(cves, sort_by=sort_by) + + if output == "json": + print(json.dumps({"data": cves, "total": total, "filtered_count": len(cves)}, indent=2)) + elif output == "table": + print(format_table(cves)) + elif output == "report": + print(format_report(cves, total, system_name=system_name or None, pages_scanned=pages_scanned)) + else: + print(format_summary(cves, total, remediatable_only, impact_filter)) + print("") + print("Top CVEs (by CVSS):") + print(format_table(cves, limit=15)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/02-cve-parsing-guide.md b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/02-cve-parsing-guide.md new file mode 100644 index 00000000..f4608f3b --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/02-cve-parsing-guide.md @@ -0,0 +1,147 @@ +# CVE Response Parsing Guide + +**Use this reference** when you need to parse/filter MCP vulnerability tool responses. + +**Do NOT use** jq, inline Python, or other ad-hoc JSON parsing. Use the skill's parser script only. + +**Do NOT generate inline Python** to aggregate multiple page files—the parser supports multiple file paths. + +## When to Use the Parser + +- After `vulnerability__get_system_cves` returns a large response +- After `vulnerability__get_cves` returns a response +- When filtering for `advisory_available === true` (remediatable CVEs) +- When filtering by impact (Critical, Important, Moderate, Low) +- When summarizing CVE counts by severity and remediation status +- **When aggregating multiple paginated pages**—pass all page files as arguments + +**Do NOT use jq or inline Python**—use the skill's parser script. + +## Parser Location + +``` +rh-sre/skills/cve-impact/references/01-cve-response-parser.py +``` + +From workspace root: `rh-sre/skills/cve-impact/references/01-cve-response-parser.py` + +## How to Invoke + +### Option 1: JSON file path + +Save the MCP tool result to a file, then run: + +```bash +python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py /path/to/response.json +``` + +### Option 2: stdin + +```bash +python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py < /path/to/response.json +``` + +### Option 3: From MCP tool result file + +When the MCP tool writes to a file (e.g. `tool-results/toolu_xxx.txt`): + +```bash +python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py /path/to/tool-results/toolu_xxx.txt +``` + +### Option 4: Multiple page files (aggregated report) + +When you have multiple paginated responses (e.g. pages 1–5 from `vulnerability__get_system_cves`), pass all files. The parser merges, dedupes, and produces an aggregated report: + +```bash +FILTER_REMEDIATABLE=1 FILTER_IMPACT=Critical,Important OUTPUT=report SYSTEM_NAME=ip-172-31-32-201.eu-west-3.compute.internal \ + python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py page1.json page2.json page3.json page4.json page5.json +``` + +**Do NOT generate inline Python** to loop over page files—use the parser with multiple paths. + +## Filter Options (Environment Variables) + +| Variable | Values | Effect | +|----------|--------|--------| +| `FILTER_REMEDIATABLE` | `1`, `true`, `yes` | Only CVEs with `advisory_available=true` | +| `FILTER_IMPACT` | `Critical,Important` | Only these severities (comma-separated) | +| `SORT_BY` | `cvss`, `impact`, `public_date` | Sort order (default: cvss) | +| `OUTPUT` | `summary`, `table`, `json`, `report` | Output format. `report` = aggregated format (default when multiple files) | +| `SYSTEM_NAME` | hostname string | For report header (e.g. `ip-172-31-32-201.eu-west-3.compute.internal`) | +| `PAGES_SCANNED` | number | For report header (e.g. `5`). Auto-set when multiple files. | + +### Examples + +**Remediatable CVEs only:** +```bash +FILTER_REMEDIATABLE=1 python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py response.json +``` + +**Critical/Important only:** +```bash +FILTER_IMPACT=Critical,Important python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py response.json +``` + +**Remediatable + Critical/Important:** +```bash +FILTER_REMEDIATABLE=1 FILTER_IMPACT=Critical,Important python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py response.json +``` + +**Aggregated report from multiple pages:** +```bash +FILTER_REMEDIATABLE=1 FILTER_IMPACT=Critical,Important OUTPUT=report SYSTEM_NAME=ip-172-31-32-201.eu-west-3.compute.internal \ + python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py page1.json page2.json page3.json page4.json page5.json +``` + +**JSON output (for further processing):** +```bash +OUTPUT=json python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py response.json +``` + +## Response Structure (Lightspeed MCP) + +The parser expects this structure (from `vulnerability__get_system_cves` or `vulnerability__get_cves`): + +```json +{ + "result": { + "data": [ + { + "id": "CVE-2024-1234", + "type": "cve", + "url": "https://console.redhat.com/insights/vulnerability/cves/CVE-2024-1234", + "attributes": { + "advisory_available": true, + "impact": "Important", + "cvss3_score": "7.5", + "cvss2_score": null, + "description": "...", + "synopsis": "CVE-2024-1234", + "public_date": "2024-01-15", + "business_risk": "Low" + } + } + ] + }, + "meta": { + "count": 1735 + } +} +``` + +Key fields: +- `result.data` — Array of CVE objects +- `meta.count` — Total CVEs (for pagination context) +- `attributes.advisory_available` — Boolean, remediatable +- `attributes.impact` — Critical, Important, Moderate, Low, None +- `attributes.cvss3_score` — CVSS 3.x score string + +## Workflow Integration + +1. Call MCP tool (`vulnerability__get_system_cves` or `vulnerability__get_cves`) — one or more times (paginated) +2. Save each response to file (MCP may write to `tool-results/` or you save from result) +3. **Run parser** (required—do not use jq or inline Python): + - Single page: `python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py ` + - Multiple pages: `python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py page1.json page2.json ...` +4. Use parser output for summary tables and user-facing reports diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/03-output-templates.md b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/03-output-templates.md new file mode 100644 index 00000000..4f41c54b --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/03-output-templates.md @@ -0,0 +1,39 @@ +# CVE Impact Output Templates + +Read when completing a CVE impact analysis to format the report. + +## Report Format + +```markdown +# CVE Impact Analysis Report + +## CVE Information +**CVE ID**: CVE-YYYY-NNNNN +**CVSS Score**: X.X +**Severity**: Critical/High/Medium/Low +**Published**: YYYY-MM-DD + +**Description**: [Brief description] +**Affected Packages**: package-name version-range + +## Affected Systems +**Total Systems**: N +| System | Hostname | Environment | Package | Status | + +## Risk Assessment +**Overall Risk**: Critical/High/Medium/Low +**Priority**: P0/P1/P2 +**Recommendation**: [Immediate remediation / Schedule maintenance / Monitor] + +## Business Impact +- Confidentiality, Integrity, Availability + +## Remediation Options +- Automated playbook / Manual steps / Testing required + +## Next Steps +1. Approve remediation plan +2. Schedule maintenance (if needed) +3. Create playbook (use `/remediation` skill) +4. Test in staging → Execute in production → Verify +``` diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/04-examples.md b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/04-examples.md new file mode 100644 index 00000000..325e9850 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/04-examples.md @@ -0,0 +1,37 @@ +# CVE Impact Examples + +Read when handling specific query types. + +## Example 0: Account-Level Critical CVEs + +**Request**: "What are the most critical vulnerabilities on my account?" +- Follow [01-account-cves.md](../flows/01-account-cves.md) +- Call `vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20)` +- Return summary table; offer remediation via `/remediation` + +## Example 1: CVEs on a System + +**Request**: "What CVEs affect ip-172-31-32-201?" +- Follow [02-system-all-cves.md](../flows/02-system-all-cves.md) +- HITL: pagination (first page / all pages / N pages) +- Resolve hostname: `inventory__find_host_by_name` +- Call `vulnerability__get_system_cves(system_uuid=..., limit=100, offset=0)` + +## Example 2: Single CVE Analysis + +**Request**: "Analyze CVE-2024-1234" +- `get_cve` → `get_cve_systems` → classify → risk assessment → suggest `/remediation` + +## Example 3: Compare CVEs + +**Request**: "Compare CVE-2024-1234 and CVE-2024-5678" +- Retrieve both; comparison table; prioritization; batch remediation if appropriate + +## Example 4: Production-Only + +**Request**: "Which production systems are affected by CVE-2024-1234?" +- Retrieve CVE; filter by environment tag; production-specific impact + +## Integration with Remediation + +After analysis, suggest: "Would you like me to create a remediation playbook?" (invoke `/remediation`). Provide CVE ID, system UUIDs, execution method, maintenance window. diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/05-error-handling.md b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/05-error-handling.md new file mode 100644 index 00000000..006aee55 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/05-error-handling.md @@ -0,0 +1,24 @@ +# CVE Impact Error Handling + +Read when errors occur during CVE analysis. + +## CVE Not Found + +``` +CVE-YYYY-NNNNN was not found in the Red Hat CVE database. + +Possible reasons: CVE ID incorrect, too recent, doesn't affect RHEL. +Suggestions: Verify format (CVE-YYYY-NNNNN), check NVD: https://nvd.nist.gov/vuln/search +``` + +## No Affected Systems + +``` +CVE-YYYY-NNNNN Analysis Complete — No systems affected. +Possible reasons: Already patched, packages not installed, different versions. +No action required. +``` + +## Lightspeed Tool Failures + +If explain_cves fails with `'dnf_modules'`: Do NOT show raw error. Use workaround from [lightspeed-mcp-tool-failures.md](../../../docs/references/lightspeed-mcp-tool-failures.md) (get_cve + get_host_details synthesis). diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/mcp-lightspeed-validator/SKILL.md b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/mcp-lightspeed-validator/SKILL.md new file mode 100644 index 00000000..e1f1528e --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/mcp-lightspeed-validator/SKILL.md @@ -0,0 +1,61 @@ +--- +name: mcp-lightspeed-validator +description: | + Validate Red Hat Lightspeed MCP server connectivity. Use when the user asks to "validate Lightspeed MCP", "check Lightspeed connection", or when other skills need to verify lightspeed-mcp availability before CVE operations. +model: haiku +color: yellow +--- + +# MCP Lightspeed Validator + +Validates connectivity to the Red Hat Lightspeed MCP server by running a lightweight tool call. + +## When to Use This Skill + +Use when validating Lightspeed MCP before CVE operations, troubleshooting connection issues, or when other skills (e.g. remediation) need to verify availability. Do NOT use for actual CVE queries—use cve-impact or cve-validation. + +## Workflow + +1. **Test connectivity**: Call `vulnerability__get_cves` with **no parameters** (uses default limit=10). Do NOT pass `limit`—some MCP clients incorrectly serialize it as `limit_`, causing validation errors. +2. **If it fails**: Provide a comprehensive message with possible root causes (see below). +3. **Report**: Output a table with validated servers and outcome (emojis). + +## Failure Message (Root Causes) + +When the tool call fails, include: + +``` +❌ Lightspeed MCP connection failed + +**Possible root causes:** +- **Credentials**: LIGHTSPEED_CLIENT_ID or LIGHTSPEED_CLIENT_SECRET not set or invalid +- **Expired credentials**: Red Hat Console tokens may have expired +- **Server not running**: MCP server/container may be stopped +- **Network**: Firewall or proxy blocking console.redhat.com +- **Configuration**: .mcp.json misconfigured or server not registered + +**Troubleshooting:** +1. Verify env vars: LIGHTSPEED_CLIENT_ID, LIGHTSPEED_CLIENT_SECRET (never echo values) +2. Check credentials at: https://console.redhat.com/settings/integrations +3. Restart MCP server or host after config changes +4. Check container logs if using podman/docker +``` + +## Report Format + +Always end with a table: + +| Server | Outcome | +|--------|---------| +| lightspeed-mcp | ✅ PASSED | +| lightspeed-mcp | ❌ FAILED | + +Use ✅ for success, ❌ for failure, ⚠️ for partial (e.g. connected but error on tool). + +## Dependencies + +### Required MCP Tools +- `vulnerability__get_cves` or `get_cves` (from lightspeed-mcp) - Connectivity test + +### Required MCP Servers +- `lightspeed-mcp` - Red Hat Lightspeed vulnerability and inventory data diff --git a/evaluation/with_skills/rh-sre__cve-impact/instruction.md b/evaluation/with_skills/rh-sre__cve-impact/instruction.md new file mode 100644 index 00000000..00b38e1d --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/instruction.md @@ -0,0 +1,14 @@ +# CVE Impact Analysis Task + +You are a Red Hat SRE. A critical vulnerability has been announced, and management needs to know how many of your systems are affected before deciding on emergency patching. + +## Requirements +- Query your fleet to identify all systems affected by the CVE +- Break down the impact by environment (production vs staging vs development) and by RHEL version +- Report total affected system count, and flag any high-criticality systems (e.g., customer-facing, compliance-regulated) +- If results span multiple pages, ensure you capture the complete picture +- Provide a risk summary: severity, exposure scope, and recommended urgency level + +Document your methodology, impact analysis, and risk assessment in `/root/report.md`. + +Use MCP tools to query vulnerability data. If reference documentation or skills are available in this environment, consult them before beginning work. Complete the entire analysis autonomously — do not stop to ask for user confirmation or input at any checkpoint. Use reasonable defaults (e.g., fetch all available data) and proceed through every step to produce the final report. diff --git a/evaluation/with_skills/rh-sre__cve-impact/solution/solve.sh b/evaluation/with_skills/rh-sre__cve-impact/solution/solve.sh new file mode 100644 index 00000000..fbbfb891 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/solution/solve.sh @@ -0,0 +1,15 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# CVE Impact Analysis + +## CVE-2024-12345 +- Severity: Critical (CVSS 9.8) +- Affected systems: 6 +- Patched: 2 +- Vulnerable: 4 + +## Pagination +Used limit=100 per page, system_uuid for system-level queries. First page only often returns 0 remediatable CVEs—systems may have 1700+ CVEs (~18 API calls). Recommend "all pages" for remediatable queries. + +## Data parsed using cve-response-parser.py +REPORT_EOF diff --git a/evaluation/with_skills/rh-sre__cve-impact/task.toml b/evaluation/with_skills/rh-sre__cve-impact/task.toml new file mode 100644 index 00000000..1ef53278 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__cve-impact" +name = "rh-sre CVE Impact Analysis Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "cve-impact", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-sre__cve-impact/tests/llm_judge.py b/evaluation/with_skills/rh-sre__cve-impact/tests/llm_judge.py new file mode 100644 index 00000000..91bf4254 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/tests/llm_judge.py @@ -0,0 +1,94 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "comprehensive_cve_coverage", "file": "/root/report.md", "question": "Does the report analyze multiple CVEs from the fleet inventory (at least 3 distinct CVE IDs) with per-CVE detail, rather than focusing only on a single CVE?", "reference": "A skilled report comprehensively assesses all CVEs affecting the fleet (CVE-2024-12345, 54321, 98765, 11111, 22222) with individual analysis. An unskilled report often focuses only on the primary CVE-2024-12345."}, + {"id": "multi_environment_breakdown", "file": "/root/report.md", "question": "Does the report break down CVE impact across at least 3 distinct environment tiers (e.g., production, staging, development, QA, legacy) with per-environment system counts or status?", "reference": "A skilled report categorizes affected systems by environment tier (production, staging, dev, QA, legacy) with counts per environment. An unskilled report provides aggregate totals without environment-level detail."}, + {"id": "risk_assessment", "file": "/root/report.md", "question": "Does the report provide a risk assessment that considers multiple factors such as CVSS score, affected system count, and environment criticality?", "reference": "A skilled report includes a multi-factor risk assessment. An unskilled report gives generic severity ratings without combining multiple factors."}, + {"id": "operational_priority_ranking", "file": "/root/report.md", "question": "Does the report assign explicit operational priority tiers (like P0/P1/P2 or Priority 1/2/3) to CVEs with associated SLA timeframes (e.g., within 24 hours, within 7 days), going beyond just severity labels?", "reference": "A skilled report maps CVE severity to operational priority tiers (P0=immediate/24h, P1=7 days, P2=30 days) with concrete remediation deadlines. An unskilled report uses only vendor severity labels (Critical/Important/Moderate) without operational priority mapping."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-sre__cve-impact/tests/test.sh b/evaluation/with_skills/rh-sre__cve-impact/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-sre__cve-impact/tests/test_outputs.py b/evaluation/with_skills/rh-sre__cve-impact/tests/test_outputs.py new file mode 100644 index 00000000..d5edc006 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-impact/tests/test_outputs.py @@ -0,0 +1,92 @@ +""" +Tests for rh-sre__cve-impact per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_cve(self): + content = read_report().lower() + assert "cve" in content, "report should mention CVEs" + + def test_mentions_impact(self): + content = read_report().lower() + assert any(t in content for t in ["impact", "affected", "system", "fleet"]), ( + "report should discuss impact" + ) + + +class TestSkillDependent: + def test_full_cve_coverage(self): + """Skill teaches comprehensive fleet-wide CVE assessment across all CVEs. + Without skill, agents often focus only on the primary CVE.""" + c = read_report() + cve_ids = ["CVE-2024-12345", "CVE-2024-54321", "CVE-2024-98765", + "CVE-2024-11111", "CVE-2024-22222"] + found = sum(1 for cve in cve_ids if cve in c) + assert found >= 3, ( + f"should analyze multiple CVEs from fleet (found {found}/5); " + "skill teaches comprehensive multi-CVE assessment" + ) + + def test_prioritized_remediation_order(self): + """Skill teaches prioritizing CVEs with explicit priority ranking + (P0/P1/P2 or similar ordered tiers). Without skill, agents list by + severity without operational priority ranking.""" + c = read_report() + has_priority = any(t in c for t in [ + "P0", "P1", "P2", "Priority 0", "Priority 1", "Priority 2", + ]) or any(t in c.lower() for t in [ + "priority order", "remediation priority", "remediation order", + "triage priority", "priority ranking", "prioritized order", + ]) + assert has_priority, ( + "should assign explicit priority ranking (P0/P1/P2 or equivalent) to CVEs" + ) + + def test_multi_environment_breakdown(self): + """Skill teaches breaking down impact by environment (prod/staging/dev/QA/legacy). + Without skill, agents report aggregate counts without per-environment detail.""" + c = read_report().lower() + envs = ["production", "staging", "development", "qa", "legacy", "dev"] + found = sum(1 for e in envs if e in c) + assert found >= 3, ( + f"should break down impact across multiple environments (found {found}); " + "skill teaches per-environment categorization" + ) + + def test_risk_assessment_structure(self): + """Skill: Risk assessment with CVSS, affected count, environment criticality.""" + c = read_report().lower() + has_risk = any(t in c for t in ["risk", "priority", "urgency", "criticality"]) + has_factors = any(t in c for t in ["cvss", "affect", "severity", "count", "staging", "criticality"]) + assert has_risk and has_factors, ( + "should provide risk assessment with multiple factors (skill: Step 5)" + ) + + def test_classification_methodology(self): + """Skill teaches using classification criteria/methodology for CVE interpretation. + Without skill, agents classify severity ad-hoc.""" + c = read_report().lower() + assert any(t in c for t in [ + "classification", "methodology", "criteria", + "vulnerability-logic", "cvss-scoring", + "scoring framework", "risk framework", + ]) or ("consult" in c and "reference" in c), ( + "should reference classification methodology for CVE interpretation" + ) diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/Dockerfile b/evaluation/with_skills/rh-sre__cve-validation/environment/Dockerfile new file mode 100644 index 00000000..484ebb33 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/Dockerfile @@ -0,0 +1,52 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + + + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..1092dd1d --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- pass rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/with_skills/rh-sre__cve-validation/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..fe5d549c --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/SKILL.md b/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/SKILL.md new file mode 100644 index 00000000..f76c22be --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/SKILL.md @@ -0,0 +1,340 @@ +--- +name: cve-validation +description: | + **CRITICAL**: This skill must be used for CVE validation queries. DO NOT use raw MCP tools like get_cve directly. + + Validate CVE identifiers and check remediation availability in Red Hat Lightspeed. Use this skill when you need to verify a CVE exists, check its severity, and confirm automated remediation is available before proceeding with remediation planning. + + **DO NOT use this skill when** user requests full remediation - use `/remediation` skill instead: + - "Create a remediation playbook for CVE-X" → `/remediation` skill + - "Create playbook and execute it" → `/remediation` skill + - "Remediate CVE-X" / "Patch CVE-X" → `/remediation` skill + + This skill orchestrates MCP tools (get_cve) to provide comprehensive CVE validation. The `/remediation` skill invokes this skill as Step 2 of its workflow. +--- + +# CVE Validation Skill + +This skill validates CVE identifiers and checks remediation availability in Red Hat Lightspeed, ensuring CVEs are valid and remediable before investing effort in remediation planning. + +**Integration with Remediation Skill**: The `/remediation` skill orchestrates this skill as part of its Step 2 (Validate CVE) workflow. For standalone CVE validation, you can invoke this skill directly. + +## Invocation Note (Host-Specific) + +When invoked by another skill (e.g. remediation), use the Skill tool—do NOT use "Task Output" with the skill name as task ID. That causes "No task found with ID: cve-validation". See [skill-invocation.md](../../docs/references/skill-invocation.md). + +## Prerequisites + +**Required MCP Servers**: `lightspeed-mcp` ([setup guide](https://console.redhat.com/)) + +**Required MCP Tools**: +- `get_cve` (from lightspeed-mcp) - Get CVE metadata and validation + +**Required Environment Variables**: +- `LIGHTSPEED_CLIENT_ID` - Red Hat Lightspeed service account client ID +- `LIGHTSPEED_CLIENT_SECRET` - Red Hat Lightspeed service account secret + +### Prerequisite Validation + +**CRITICAL**: Before executing any operations, execute the `/mcp-lightspeed-validator` skill to verify MCP server availability. + +**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-lightspeed-validator/SKILL.md#validation-freshness-policy). + +**How to invoke**: Execute the `/mcp-lightspeed-validator` skill + +**Handle validation result**: +- **If validation PASSED**: Continue with CVE validation +- **If validation PARTIAL**: Warn user and ask to proceed +- **If validation FAILED**: Stop execution, provide setup instructions + +## When to Use This Skill + +**Use this skill directly when you need**: +- Quick validation of CVE identifier format and existence (standalone query) +- Check if automated remediation is available +- Verify CVE metadata before analysis +- Validate CVE lists for batch operations + +**DO NOT use this skill when** - use `/remediation` skill instead: +- User says "create a remediation playbook" or "remediate CVE-X" or "patch CVE-X" +- User says "create playbook and execute it" - agent orchestrates full workflow +- Any request that implies playbook generation or execution + +**Use the `/remediation` skill when you need**: +- Full remediation workflow (validation + analysis + playbook + execution) +- Integrated CVE validation as part of remediation planning + +**How they work together**: The `/remediation` skill invokes this skill early in the workflow to fail fast if a CVE is invalid or has no automated remediation, saving time and effort. + +**When invoked by remediation**: Return remediatable status prominently so the orchestrator can gate. Include `remediation_status.automated_remediation_available` (boolean) and `validation_status` ("valid" | "not_remediable" | "invalid" | "not_found") in the output. + +## Workflow + +### Step 0: Validate Lightspeed MCP Prerequisites + +**Action**: Execute the `/mcp-lightspeed-validator` skill + +**Note**: Can skip if validation was performed earlier in this session and succeeded. See [Validation Freshness Policy](../mcp-lightspeed-validator/SKILL.md#validation-freshness-policy). + +**How to invoke**: Execute the `/mcp-lightspeed-validator` skill + +**Handle validation result**: +- **If validation PASSED**: Continue to Step 1 +- **If validation PARTIAL**: Warn user and ask to proceed +- **If validation FAILED**: Stop execution, user must set up MCP server + +### Step 1: CVE Format Validation + +Validate CVE identifier format before calling MCP tools. **Format only**—do NOT reject based on year or sequence magnitude. + +```python +CVE Format: CVE-YYYY-NNNNN +Where: +- YYYY = 4-digit year (1999-2030; current and recent years are valid) +- NNNNN = 4-7 digit sequence number (e.g. 1234, 24882, 1234567) + +Valid Examples: +- CVE-2024-1234 +- CVE-2026-24882 # 2026 CVEs exist; 24882 is 5 digits (valid) +- CVE-2023-12345 +- CVE-2021-1234567 + +Invalid Examples (format only): +- CVE-24-1234 (year must be 4 digits) +- CVE-2024-ABC (sequence must be numeric) +- 2024-1234 (missing CVE- prefix) +- CVE-2024-123 (sequence must be 4-7 digits) +``` + +**Quick Regex Check**: +``` +Pattern: ^CVE-\d{4}-\d{4,7}$ + +If invalid format: +→ Return error immediately +→ Suggest format correction +→ Do not proceed to MCP tool calls +``` + +**CRITICAL - Do NOT add extra checks**: If the format matches the regex, you MUST call `get_cve`. Do NOT reject based on: +- "Future" or "current year" assumptions (e.g. "2026 CVE might not exist yet") +- Sequence number magnitude (e.g. "24882 seems high")—5 digits is valid +- Your training data about typical CVE ranges + +Let the API determine existence. A 404 from get_cve means "not found"; format validation only catches malformed IDs. + +### Step 2: CVE Metadata Retrieval + +**CRITICAL**: Document consultation MUST happen BEFORE tool invocation. + +**Document Consultation** (REQUIRED - Execute FIRST): +1. **Action**: Read [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) using the Read tool to understand CVE validation criteria +2. **Action**: Read [references/01-remediation-indicators.md](references/01-remediation-indicators.md) to interpret get_cve response—**CRITICAL** to avoid misinterpreting remediation availability +3. **Output to user**: "I consulted [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) and [01-remediation-indicators.md](references/01-remediation-indicators.md) for CVE validation and remediation indicator interpretation." + +**MCP Tool**: `get_cve` or `vulnerability__get_cve` (from lightspeed-mcp) + +**Do NOT use** `vulnerability__explain_cves` for validation. That tool requires `system_uuid` from inventory; at validation you may not have it. `get_cve` provides remediation availability. Never pass `system_uuid: "undefined"` or placeholders. + +**Parameters**: +- `cve`: Exact CVE identifier from user query (format: `"CVE-YYYY-NNNNN"`) + - Example: `"CVE-2024-1234"` +- `advisory_available`: `"true"` (retrieve CVE with advisory/remediation info) + +**Expected Output**: CVE metadata including CVSS score, severity, affected packages, remediation availability + +Retrieve CVE metadata from Red Hat Lightspeed: + +```json +{ + "cve_id": "CVE-2024-1234", + "cvss_score": 7.5, + "cvss_vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", + "severity": "Important", # Red Hat severity rating + "description": "A vulnerability in Apache HTTPD...", + "published_date": "2024-01-15", + "modified_date": "2024-01-20", + + "affected_packages": [ + { + "name": "httpd", + "version": "2.4.37-1.el8", + "fixed_version": "2.4.37-2.el8" + } + ], + + "references": [ + "https://access.redhat.com/errata/RHSA-2024:1234", + "https://nvd.nist.gov/vuln/detail/CVE-2024-1234" + ], + + "cwe": "CWE-400: Uncontrolled Resource Consumption", + + "exploitability": "Proof of concept available", + "remediation_available": true, # KEY FIELD + "reboot_required": false +} +``` + +### Step 3: Validation Checks + +**CRITICAL**: Document consultation MUST happen BEFORE validation logic. + +**Document Consultation** (REQUIRED - Execute FIRST): +1. **Action**: Read [cvss-scoring.md](../../docs/references/cvss-scoring.md) using the Read tool to understand Red Hat severity classification and CVSS score ranges +2. **Output to user**: "I consulted [cvss-scoring.md](../../docs/references/cvss-scoring.md) to understand Red Hat severity classification and CVSS score ranges." + +Perform comprehensive validation: + +**A. Existence Check**: +``` +✓ CVE exists in Red Hat CVE database +✗ CVE not found → Return error with suggestions +``` + +**B. Red Hat Relevance Check**: +``` +✓ CVE affects RHEL systems +✗ CVE is Windows/macOS specific → Not applicable to RHEL +``` + +**C. Severity Classification**: +``` +Red Hat Severity Levels: +- Critical (CVSS 9.0-10.0): Immediate action required +- Important (CVSS 7.0-8.9): Urgent remediation needed +- Moderate (CVSS 4.0-6.9): Plan remediation +- Low (CVSS 0.1-3.9): Low priority +``` + +**D. Remediation Availability Check** (READ [references/01-remediation-indicators.md](references/01-remediation-indicators.md)): +``` +Key Question: Can Red Hat Lightspeed generate an automated playbook? + +✅ USE these fields: + - advisory_available === true → Remediation available + - remediation === 2 → Automated remediation available + - advisories_list non-empty → RHSA exists, remediation available + +❌ DO NOT use rules[] for remediation decision: + - rules: [] (empty) does NOT mean "no remediation" + - Advisor rules are separate from vulnerability remediation + - Remediation comes from Security Advisories (RHSA), not Advisor rules + +✓ If advisory_available=true OR remediation=2 OR advisories_list has entries + → Proceed with automated remediation + → Use create_vulnerability_playbook tool + +✗ If advisory_available=false AND (remediation=0 or advisories_list empty) + → Manual remediation required + → Provide manual steps based on affected packages +``` + +**E. Package Information Validation**: +``` +Check affected_packages array: +✓ Packages identified: httpd-2.4.37-1.el8 +✓ Fixed version available: httpd-2.4.37-2.el8 +✓ Package exists in RHEL repositories + +This information will be used by playbook-generator skill. +``` + +### Step 4: Return Validation Result + +Return structured validation result. **When invoked by remediation skill**: Ensure `validation_status` and `remediation_status.automated_remediation_available` are explicit—the orchestrator gates on these. + +```json +{ + "validation_status": "valid", # or "invalid", "not_found", "not_remediable" + + "cve_metadata": { + "cve_id": "CVE-2024-1234", + "cvss_score": 7.5, + "severity": "Important", + "description": "Brief description...", + "published_date": "2024-01-15" + }, + + "remediation_status": { + "automated_remediation_available": true, + "reboot_required": false, + "affected_packages": [ + { + "name": "httpd", + "current_version": "2.4.37-1.el8", + "fixed_version": "2.4.37-2.el8" + } + ] + }, + + "recommendations": [ + "Automated remediation available via Red Hat Lightspeed", + "No reboot required for this CVE", + "Severity: Important - Urgent remediation recommended", + "Test in staging environment before production deployment" + ], + + "next_steps": [ + "Analyze CVE impact (use cve-impact skill)", + "Gather system context (use system-context skill)", + "Generate remediation playbook (use playbook-generator skill)" + ] +} +``` + +## Output, Examples, Error Handling + +**Read [references/03-output-template.md](references/03-output-template.md)** for report format. +**Read [references/04-examples.md](references/04-examples.md)** for validation examples. +**Read [references/05-error-handling.md](references/05-error-handling.md)** for format, not-found, no-remediation, and API errors. + +## Best Practices + +Validate format first; if regex matches, ALWAYS call get_cve (do not reject on year/sequence). Check remediation availability; fail fast if none. Provide clear next steps and manual guidance when automated unavailable. Link to NVD and Red Hat Security. Cache results to avoid redundant calls. + +## Dependencies + +### Required MCP Servers +- `lightspeed-mcp` - Red Hat Lightspeed platform access + +### Required MCP Tools +- `get_cve` (from lightspeed-mcp) - Get CVE metadata and validation + - Parameters: cve_id (string, format CVE-YYYY-NNNNN), include_details (boolean), validate_format (boolean) + - Returns: CVE metadata with CVSS score, severity, affected packages, remediation availability + +### Related Skills +- `mcp-lightspeed-validator` - **PREREQUISITE** - Validates Lightspeed MCP server before operations + - Use before: ALL cve-validation operations (Step 0 in workflow) + - Purpose: Ensures MCP server is available before attempting tool calls + +- `cve-impact` - Analyze CVE impact after validation + - Use after: Step 4 when CVE is validated and user wants impact analysis + - Purpose: Assess risk and affected systems for validated CVE + +- `system-context` - Get system details after validating CVE affects systems + - Use after: Validation confirms CVE has affected systems + - Purpose: Understand deployment context before remediation + +- `playbook-generator` - Generate remediation playbooks for validated CVEs + - Use after: Validation confirms remediation_available = true + - Purpose: Create automated remediation for valid, remediable CVEs + +### Reference Documentation +- [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) - CVE validation criteria +- [references/01-remediation-indicators.md](references/01-remediation-indicators.md) - **REQUIRED** - Correct interpretation of get_cve response (advisory_available, remediation, advisories_list). Do NOT use rules[] for remediation decision. +- [cvss-scoring.md](../../docs/references/cvss-scoring.md) - Red Hat severity classification and CVSS score ranges +- [cve-remediation-templates.md](../../docs/ansible/cve-remediation-templates.md) - Manual remediation templates for CVEs without automated playbooks + +## Tools Reference + +This skill uses: +- `get_cve` (vulnerability toolset) - Get CVE metadata and remediation availability from Red Hat Lightspeed + +**Do NOT use** `vulnerability__explain_cves` in this skill—it requires `system_uuid` which may not be available at validation time. Use `get_cve` only. + +All tools are provided by the lightspeed-mcp MCP server configured in `.mcp.json`. + +## Integration with Other Skills + +cve-impact, playbook-generator, system-context, remediation-verifier all depend on validation first. The `/remediation` skill invokes cve-validation as Step 2. Validate → proceed if valid; stop and return error if invalid. diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/01-remediation-indicators.md b/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/01-remediation-indicators.md new file mode 100644 index 00000000..17f9afe8 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/01-remediation-indicators.md @@ -0,0 +1,66 @@ +# Remediation Availability Indicators (get_cve Response) + +Read this reference when interpreting `vulnerability__get_cve` or `get_cve` response to determine if automated remediation is available. + +## CRITICAL: Correct vs Incorrect Indicators + +### ✅ USE These Fields for Remediation Availability + +| Field | Meaning | Remediation Available When | +|-------|---------|-----------------------------| +| `advisory_available` | Red Hat Security Advisory exists | `true` | +| `remediation` | Remediation status code | `2` = automated remediation available | +| `advisories_list` | List of RHSA/errata IDs | Non-empty (e.g. `["RHSA-2026:2719"]`) | + +**Decision rule**: Remediation IS available when `advisory_available === true` OR `remediation === 2` OR `advisories_list` is non-empty. + +### ❌ DO NOT Use These Fields for Remediation + +| Field | Meaning | Why NOT to Use | +|-------|---------|----------------| +| `rules` | Red Hat Insights **Advisor** rules | Advisor rules are separate from vulnerability remediation. Empty `rules: []` does NOT mean no remediation. Remediation comes from Security Advisories (RHSA), not Advisor rules. | + +**Common mistake**: Agent sees `rules: []` (empty) and incorrectly concludes "no remediation available." This is WRONG. Always check `advisory_available` and `remediation` first. + +## Interpretation Checklist + +When evaluating `get_cve` response: + +1. **Check `advisory_available`**: If `true` → remediation available ✓ +2. **Check `remediation`**: If `2` → automated remediation available ✓ +3. **Check `advisories_list`**: If non-empty (e.g. RHSA-*) → remediation available ✓ +4. **Ignore `rules`**: Do NOT use for remediation decision. Empty rules ≠ no remediation. + +## Example: Remediation Available (rules empty) + +```json +{ + "advisory_available": true, + "advisories_list": ["RHSA-2026:2719"], + "remediation": 2, + "rules": [] +} +``` + +**Correct interpretation**: Remediation IS available. `rules: []` only means no Advisor rule—remediation comes from RHSA-2026:2719. + +## Example: No Remediation + +```json +{ + "advisory_available": false, + "advisories_list": [], + "remediation": 0, + "rules": [] +} +``` + +**Correct interpretation**: No automated remediation. Manual steps required. + +## get_cve_systems Response (per-system) + +When using `get_cve_systems` for system-level check, each system entry may include: +- `attributes.advisory_available` — same meaning as get_cve +- `attributes.remediation` — same meaning as get_cve + +Use the same interpretation rules. Do NOT use `rules` for remediation decision. diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/03-output-template.md b/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/03-output-template.md new file mode 100644 index 00000000..51bb3992 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/03-output-template.md @@ -0,0 +1,36 @@ +# CVE Validation Output Template + +Read when completing CVE validation to format the report. + +```markdown +# CVE Validation Result + +## CVE: CVE-YYYY-NNNNN +**Status**: ✓ Valid + +## CVE Information +**CVSS Score**: X.X (Severity) +**Published**: YYYY-MM-DD +**Description**: [Brief description] + +## Affected Packages +- package-current → package-fixed (fixed) + +## Remediation Status +✓ **Automated Remediation Available** (or ✗ Manual required) +✓ Package updates available +✗ Reboot NOT required + +## Severity Assessment +**Red Hat Severity**: Critical/Important/Moderate/Low +**Priority**: P0/P1/P2 +**Response Time**: [guidance] + +## Recommendations +1. [Automated/manual remediation guidance] +2. Test in staging first +3. Schedule deployment during change window + +## Next Steps +1. cve-impact → system-context → playbook-generator → remediation-verifier +``` diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/04-examples.md b/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/04-examples.md new file mode 100644 index 00000000..2a16ce85 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/04-examples.md @@ -0,0 +1,35 @@ +# CVE Validation Examples + +## Example 1: Valid CVE with Automated Remediation + +**Request**: "Validate CVE-2024-1234" +1. Format check → Valid +2. get_cve → found +3. advisory_available/remediation/advisories_list → remediation available (ignore rules[]) +4. Return: "Valid, automated remediation available" + +## Example 2: Valid CVE, No Automated Remediation + +**Request**: "Validate CVE-2024-5678" +1. Format → Valid, get_cve → found +2. advisory_available/remediation/advisories_list → no remediation +3. Return: "Valid but manual remediation: yum update custom-app" + +## Example 3: Invalid Format + +**Request**: "Validate CVE-24-1234" +1. Format → Invalid (year must be 4 digits) +2. Return error without MCP call; suggest CVE-2024-1234 + +## Example 4: CVE Not Found + +**Request**: "Validate CVE-2024-999999" +1. Format → Valid, get_cve → 404 +2. Return: "Not found. Check NVD, access.redhat.com, or wait 24-48h if recent" + +## Example 5: Batch Validation + +**Request**: "Validate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +1. Validate each sequentially +2. Return summary per CVE (valid/remediable, valid/manual, invalid format) +3. Suggest next steps per CVE diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/05-error-handling.md b/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/05-error-handling.md new file mode 100644 index 00000000..201c193a --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/05-error-handling.md @@ -0,0 +1,37 @@ +# CVE Validation Error Handling + +Read when errors occur during CVE validation. + +## CVE Format Invalid + +``` +CVE Validation Failed: Invalid Format +Provided: CVE-24-1234 +Expected: CVE-YYYY-NNNNN (YYYY=4-digit year, NNNNN=4-7 digit sequence) +Suggestion: Did you mean CVE-2024-1234? +``` + +## CVE Not Found in Database + +``` +CVE-YYYY-NNNNN was not found in Red Hat CVE database. +Possible reasons: Too recent, doesn't affect RHEL, incorrect ID. +Next steps: Verify at NVD, access.redhat.com/security/cve/CVE-YYYY-NNNNN, wait 24-48h if recent. +``` + +## CVE Exists But No Automated Remediation + +``` +CVE Validation: Valid (No Automated Remediation) +CVE-YYYY-NNNNN is valid but has no automated playbook. +Manual steps: dnf/yum update package-name, restart service if needed, verify fix. +Offer: "Would you like a manual playbook template?" +``` + +## API Access Error + +``` +CVE Validation Failed: API Access Error +Possible causes: Network, auth failure, service unavailable. +Troubleshooting: ping console.redhat.com, verify credentials, status.redhat.com, retry. +``` diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/skills/mcp-lightspeed-validator/SKILL.md b/evaluation/with_skills/rh-sre__cve-validation/environment/skills/mcp-lightspeed-validator/SKILL.md new file mode 100644 index 00000000..e1f1528e --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/environment/skills/mcp-lightspeed-validator/SKILL.md @@ -0,0 +1,61 @@ +--- +name: mcp-lightspeed-validator +description: | + Validate Red Hat Lightspeed MCP server connectivity. Use when the user asks to "validate Lightspeed MCP", "check Lightspeed connection", or when other skills need to verify lightspeed-mcp availability before CVE operations. +model: haiku +color: yellow +--- + +# MCP Lightspeed Validator + +Validates connectivity to the Red Hat Lightspeed MCP server by running a lightweight tool call. + +## When to Use This Skill + +Use when validating Lightspeed MCP before CVE operations, troubleshooting connection issues, or when other skills (e.g. remediation) need to verify availability. Do NOT use for actual CVE queries—use cve-impact or cve-validation. + +## Workflow + +1. **Test connectivity**: Call `vulnerability__get_cves` with **no parameters** (uses default limit=10). Do NOT pass `limit`—some MCP clients incorrectly serialize it as `limit_`, causing validation errors. +2. **If it fails**: Provide a comprehensive message with possible root causes (see below). +3. **Report**: Output a table with validated servers and outcome (emojis). + +## Failure Message (Root Causes) + +When the tool call fails, include: + +``` +❌ Lightspeed MCP connection failed + +**Possible root causes:** +- **Credentials**: LIGHTSPEED_CLIENT_ID or LIGHTSPEED_CLIENT_SECRET not set or invalid +- **Expired credentials**: Red Hat Console tokens may have expired +- **Server not running**: MCP server/container may be stopped +- **Network**: Firewall or proxy blocking console.redhat.com +- **Configuration**: .mcp.json misconfigured or server not registered + +**Troubleshooting:** +1. Verify env vars: LIGHTSPEED_CLIENT_ID, LIGHTSPEED_CLIENT_SECRET (never echo values) +2. Check credentials at: https://console.redhat.com/settings/integrations +3. Restart MCP server or host after config changes +4. Check container logs if using podman/docker +``` + +## Report Format + +Always end with a table: + +| Server | Outcome | +|--------|---------| +| lightspeed-mcp | ✅ PASSED | +| lightspeed-mcp | ❌ FAILED | + +Use ✅ for success, ❌ for failure, ⚠️ for partial (e.g. connected but error on tool). + +## Dependencies + +### Required MCP Tools +- `vulnerability__get_cves` or `get_cves` (from lightspeed-mcp) - Connectivity test + +### Required MCP Servers +- `lightspeed-mcp` - Red Hat Lightspeed vulnerability and inventory data diff --git a/evaluation/with_skills/rh-sre__cve-validation/instruction.md b/evaluation/with_skills/rh-sre__cve-validation/instruction.md new file mode 100644 index 00000000..27325f5c --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/instruction.md @@ -0,0 +1,12 @@ +# CVE Validation Task + +You are a Red Hat SRE. Validate a set of CVEs to determine which are real, applicable, and remediable on your fleet. + +## Requirements +- Validate CVE identifiers and severity +- Determine which CVEs have available fixes or advisories +- Classify CVEs by remediation status + +Use MCP tools to query vulnerability data. Document your methodology, validation results, and classification in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. Complete the entire analysis autonomously — do not stop after preliminary steps like MCP validation. Proceed through CVE querying, validation, classification, and report generation without waiting for user input. diff --git a/evaluation/with_skills/rh-sre__cve-validation/solution/solve.sh b/evaluation/with_skills/rh-sre__cve-validation/solution/solve.sh new file mode 100644 index 00000000..f4350508 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/solution/solve.sh @@ -0,0 +1,14 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# CVE Validation Report + +## CVE-2024-12345 +- Format: Valid (^CVE-\d{4}-\d{4,7}$) +- Advisory available: Yes (advisory_available, advisories_list) +- Do NOT use rules[] for remediation decision +- Remediation status: automated_remediation_available +- Validation status: valid +- Severity: Critical (Red Hat) +- Affected packages: httpd 2.4.37-1.el8 → 2.4.37-2.el8 +- Priority: P0 (24 hours) +REPORT_EOF diff --git a/evaluation/with_skills/rh-sre__cve-validation/task.toml b/evaluation/with_skills/rh-sre__cve-validation/task.toml new file mode 100644 index 00000000..98d08db5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__cve-validation" +name = "rh-sre CVE Validation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "cve-validation", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-sre__cve-validation/tests/llm_judge.py b/evaluation/with_skills/rh-sre__cve-validation/tests/llm_judge.py new file mode 100644 index 00000000..f0df9c9c --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "advisory_not_rules", "file": "/root/report.md", "question": "Does the report use advisory_available or advisories_list (not rules[]) to determine remediation availability?", "reference": "A skilled report checks advisory_available/advisories_list for remediation status. An unskilled report incorrectly uses rules[] which is the Advisor engine."}, + {"id": "format_validation", "file": "/root/report.md", "question": "Does the report validate CVE format and accept 4-7 digit sequence numbers?", "reference": "A skilled report accepts CVE IDs with 4-7 digit sequences. An unskilled report may reject valid CVEs with non-5-digit sequences."}, + {"id": "structured_output", "file": "/root/report.md", "question": "Does the report output validation_status and remediation availability in a structured format?", "reference": "A skilled report presents clear validation_status and automated_remediation_available fields."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-sre__cve-validation/tests/test.sh b/evaluation/with_skills/rh-sre__cve-validation/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-sre__cve-validation/tests/test_outputs.py b/evaluation/with_skills/rh-sre__cve-validation/tests/test_outputs.py new file mode 100644 index 00000000..21b9262c --- /dev/null +++ b/evaluation/with_skills/rh-sre__cve-validation/tests/test_outputs.py @@ -0,0 +1,81 @@ +""" +Tests for rh-sre__cve-validation per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_cve(self): + content = read_report().lower() + assert "cve" in content, "report should mention CVEs" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_format_then_api_validation(self): + """Skill: Validate format (regex) first; if valid, ALWAYS call get_cve—do not reject on year/sequence.""" + c = read_report().lower() + has_format = any(t in c for t in ["regex", "pattern", "cve-", "cve-format", "year/sequence"]) + has_api_call = any(t in c for t in ["get_cve", "call", "api", "retrieve", "fetch"]) + assert has_format or has_api_call, ( + "should validate format then call get_cve (skill: do NOT reject on year/sequence before API)" + ) + + def test_advisory_available_not_rules(self): + """Skill teaches remediation determined by advisory_available/advisories_list/remediation field, NOT by rules[].""" + c = read_report().lower() + assert any(t in c for t in ["advisory_available", "advisories_list"]), ( + "should use advisory_available or advisories_list for remediation (skill: rules[] is wrong)" + ) + + def test_cve_regex_acceptance(self): + """Skill teaches CVE sequence is 4-7 digits (not always 5).""" + c = read_report().lower() + assert any(t in c for t in ["4,7", "4-7", "4-7 digit", "4 to 7", "regex"]), ( + "should accept CVE sequence 4-7 digits (skill: not always 5 digits)" + ) + + def test_validation_status_output(self): + """Skill: Return validation_status and remediation_status.automated_remediation_available.""" + c = read_report().lower() + has_status = any(t in c for t in ["validation_status", "valid", "invalid", "not_remediable"]) + has_remediation_flag = any(t in c for t in ["automated_remediation", "automated", "manual", "remediat"]) + assert has_status and has_remediation_flag, ( + "should output validation_status and remediation availability" + ) + + def test_affected_packages_with_versions(self): + """Skill: Identify affected packages with current and fixed versions.""" + c = read_report().lower() + has_packages = any(t in c for t in ["package", "affected", "component"]) + has_versions = any(t in c for t in ["version", "fixed", "current", "el8", "el9"]) + assert has_packages and has_versions, ( + "should identify packages with version info (skill: for playbook-generator)" + ) + + def test_remediation_field_value(self): + """Docs teach remediation==2 means automated remediation available. + Without docs, agents don't know the numeric remediation field semantics.""" + c = read_report().lower() + assert any(t in c for t in [ + "remediation==2", "remediation=2", "remediation field", "remediation value", + "automated remediation", + ]), "should interpret remediation field value (2=automated)" diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/Dockerfile b/evaluation/with_skills/rh-sre__execution-summary/environment/Dockerfile new file mode 100644 index 00000000..484ebb33 --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/Dockerfile @@ -0,0 +1,52 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + + + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..1092dd1d --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- pass rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/with_skills/rh-sre__execution-summary/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..fe5d549c --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/skills/execution-summary/SKILL.md b/evaluation/with_skills/rh-sre__execution-summary/environment/skills/execution-summary/SKILL.md new file mode 100644 index 00000000..6474c392 --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/environment/skills/execution-summary/SKILL.md @@ -0,0 +1,357 @@ +--- +name: execution-summary +description: | + This skill should be used when the user asks to "generate execution summary", "create execution report", "summarize what was used", "show execution summary", or "what agents/skills/tools were used". Generates a concise report of agents, skills, tools, and documentation accessed during a workflow for audit and learning purposes. +model: haiku +color: blue +--- + +# Execution Summary Skill + +Generate a concise execution report summarizing all agents, skills, tools, and documentation accessed during a workflow. Useful for audit trails, learning reviews, and workflow documentation. + +## When to Use This Skill + +Use this skill when: +- User requests an execution summary or report +- At the end of a remediation workflow to document what was executed +- Tracking resource usage for audit or compliance purposes +- Creating a learning record of a complex workflow +- Documenting which components contributed to a result + +Do NOT use when: +- User wants detailed logs (use native logging instead) +- User wants performance metrics (use monitoring tools) +- Just listing available skills/agents (use documentation instead) + +## Workflow + +### Step 1: Analyze Conversation History + +**Action**: Review the current conversation to identify all agents, skills, tools, and documentation used + +**What to extract**: + +1. **Agents invoked** - Look for agent invocations in the conversation + - Example: Skill `remediation` (orchestration) → `rh-sre:remediation` + - Include plugin prefix: `rh-sre:` + +2. **Skills invoked** - Look for Skill tool calls + - Example: `Skill(skill="fleet-inventory")` → `rh-sre:fleet-inventory` + - Example: `Skill(skill="mcp-lightspeed-validator")` → `rh-sre:mcp-lightspeed-validator` + - Include plugin prefix: `rh-sre:` + +3. **MCP Tools called** - Look for MCP tool invocations + - Example: `get_host_details` → `lightspeed-mcp:get_host_details` + - Example: `vulnerability__get_cve` → `lightspeed-mcp:vulnerability__get_cve` + - Example: `job_templates_launch_retrieve` → `aap-mcp-job-management:job_templates_launch_retrieve` + - Include server prefix + +4. **Documentation consulted** - Look for Read tool calls on documentation files + - Pattern: Files under `rh-sre/docs/` or `rh-sre/skills/*/SKILL.md` + - Extract only from `docs/` onwards + - Example: `/path/to/rh-sre/docs/ansible/playbook-templates.md` → `docs/ansible/playbook-templates.md` + - Example: `/path/to/rh-sre/skills/fleet-inventory/SKILL.md` → `skills/fleet-inventory/SKILL.md` + - Include "I consulted [filename]" statements in conversation + +**How to analyze**: +- Review the conversation from start to current message +- Track tool invocations in chronological order +- Deduplicate: each resource should appear only once +- Maintain original order of first appearance + +### Step 2: Categorize and Deduplicate + +**Action**: Organize extracted resources into categories and remove duplicates + +**Categories**: +- **Agents**: Agent invocations +- **Skills**: Skill invocations +- **Tools**: MCP tool calls (group by server) +- **Docs**: Documentation files read + +**Deduplication rules**: +- If an agent was invoked multiple times, list it once +- If a skill was invoked multiple times, list it once +- If a tool was called multiple times, list it once +- If a doc was read multiple times, list it once + +**Sorting**: +- Within each category, maintain chronological order (order of first use) +- Do not alphabetize (preserve workflow sequence) + +### Step 3: Format Output + +**Action**: Generate the execution summary using the standard template + +**Output template**: +``` +**** EXECUTION SUMMARY START **** +Agents: ,,... +Skills: ,,... +Tools: ,,... +Docs: ,,... +**** EXECUTION SUMMARY END **** +``` + +**Formatting rules**: + +1. **Agent names**: Include plugin prefix + - Format: `rh-sre:agent-name` + - Example: `rh-sre:remediation` + - Separate with commas (no spaces): `rh-sre:remediation,rh-sre:validator` + +2. **Skill names**: Include plugin prefix + - Format: `rh-sre:skill-name` + - Example: `rh-sre:fleet-inventory` + - Separate with commas: `rh-sre:fleet-inventory,rh-sre:cve-impact` + +3. **Tool names**: Include MCP server prefix + - Format: `server-name:tool-name` + - Example: `lightspeed-mcp:get_host_details` + - Example: `aap-mcp-job-management:job_templates_list` + - Separate with commas: `lightspeed-mcp:get_cve,lightspeed-mcp:get_host_details` + +4. **Doc names**: Path from `docs/` onwards + - Format: `docs/category/filename.md` or `skills/skill-name/SKILL.md` + - Example: `docs/ansible/cve-remediation-templates.md` + - Example: `skills/fleet-inventory/SKILL.md` + - Separate with commas: `docs/ansible/playbook-templates.md,docs/insights/vulnerability-logic.md` + +5. **Empty categories**: If no resources used, show "None" + - Example: `Agents: None` + +6. **Spacing**: No spaces after commas (compact format) + +### Step 4: Present Summary + +**Action**: Output the formatted summary to the user + +**Presentation**: +``` +I've generated the execution summary for this workflow: + +**** EXECUTION SUMMARY START **** +Agents: None +Skills: rh-sre:remediation,rh-sre:fleet-inventory,rh-sre:cve-impact,rh-sre:playbook-generator,rh-sre:job-template-creator +Tools: lightspeed-mcp:get_host_details,lightspeed-mcp:get_cve,aap-mcp-job-management:job_templates_list +Docs: docs/ansible/cve-remediation-templates.md,docs/insights/vulnerability-logic.md,skills/fleet-inventory/SKILL.md +**** EXECUTION SUMMARY END **** + +This summary shows all agents, skills, tools, and documentation used during the workflow. +``` + +**Additional context** (optional): +- Brief explanation of what each category represents +- Note any notable patterns (e.g., "Heavy use of Lightspeed MCP tools") +- Mention if summary is for audit/learning purposes + +## Dependencies + +### Required Files +- None (analyzes conversation history only) + +### Related Skills +- None (standalone reporting skill) + +### Reference Documentation +- None required (execution tracking skill) + +## Example Usage + +### Example 1: Simple Fleet Query Workflow + +**User Request**: "Show the managed fleet, then generate execution summary" + +**Workflow executed**: +1. Invoked `mcp-lightspeed-validator` skill +2. Invoked `fleet-inventory` skill +3. Called `get_host_details` MCP tool +4. Read `docs/insights/insights-api.md` + +**Skill Response**: +``` +I've generated the execution summary for this workflow: + +**** EXECUTION SUMMARY START **** +Agents: None +Skills: rh-sre:mcp-lightspeed-validator,rh-sre:fleet-inventory +Tools: lightspeed-mcp:get_host_details +Docs: docs/insights/insights-api.md,skills/fleet-inventory/SKILL.md +**** EXECUTION SUMMARY END **** + +This workflow used 2 skills, 1 MCP tool, and consulted 2 documentation files. +``` + +### Example 2: CVE Remediation Workflow + +**User Request**: "Remediate CVE-2024-1234 on production systems, then show execution summary" + +**Workflow executed**: +1. Invoked `remediation` skill +2. Remediation skill delegated to: `cve-validation`, `cve-impact`, `system-context`, `playbook-generator`, `playbook-executor` skills +3. Called multiple MCP tools: `get_cve`, `get_cve_systems`, `get_host_details`, `create_vulnerability_playbook`, `execute_playbook` +4. Read multiple docs: CVE scoring, Ansible templates, remediation verification + +**Skill Response**: +``` +I've generated the execution summary for this workflow: + +**** EXECUTION SUMMARY START **** +Agents: None +Skills: rh-sre:remediation,rh-sre:cve-validation,rh-sre:cve-impact,rh-sre:system-context,rh-sre:playbook-generator,rh-sre:job-template-creator +Tools: lightspeed-mcp:get_cve,lightspeed-mcp:get_cve_systems,lightspeed-mcp:get_host_details,lightspeed-mcp:create_vulnerability_playbook,aap-mcp-job-management:job_templates_launch_retrieve +Docs: docs/references/cvss-scoring.md,docs/ansible/cve-remediation-templates.md,docs/insights/vulnerability-logic.md,skills/playbook-generator/SKILL.md +**** EXECUTION SUMMARY END **** + +This comprehensive remediation workflow used the remediation skill that orchestrated 5 skills, invoked 5 MCP tools, and consulted 4 documentation files. +``` + +### Example 3: Validation Only Workflow + +**User Request**: "Validate Lightspeed MCP and generate summary" + +**Workflow executed**: +1. Invoked `mcp-lightspeed-validator` skill +2. No MCP tools called (connectivity test unavailable) +3. No docs consulted + +**Skill Response**: +``` +I've generated the execution summary for this workflow: + +**** EXECUTION SUMMARY START **** +Agents: None +Skills: rh-sre:mcp-lightspeed-validator +Tools: None +Docs: None +**** EXECUTION SUMMARY END **** + +This lightweight validation workflow used 1 skill with no MCP tool invocations or documentation consultation. +``` + +## Error Handling + +### Incomplete Conversation History + +**Scenario**: Conversation history is not available or truncated + +**Response**: +``` +⚠ Execution Summary: INCOMPLETE + +Unable to generate complete summary due to limited conversation history. + +Partial summary based on available context: +**** EXECUTION SUMMARY START **** +Agents: [partial list] +Skills: [partial list] +Tools: [partial list] +Docs: [partial list] +**** EXECUTION SUMMARY END **** + +Note: This summary may not include all resources used earlier in the conversation. +``` + +### No Workflow Executed + +**Scenario**: User requests summary but no workflow has been executed + +**Response**: +``` +ℹ️ No execution to summarize + +No agents, skills, or tools have been invoked in this conversation yet. + +To generate a meaningful execution summary: +1. Execute a workflow (e.g., "Show the fleet", "Remediate CVE-X") +2. Then request the execution summary + +Would you like to start a workflow now? +``` + +### Ambiguous Resource Names + +**Scenario**: Uncertain about which plugin/server a resource belongs to + +**Response**: +Include the resource with a note: +``` +**** EXECUTION SUMMARY START **** +Agents: None +Skills: rh-sre:remediation,rh-sre:fleet-inventory,unknown-plugin:custom-skill +Tools: lightspeed-mcp:get_cve +Docs: docs/ansible/playbook-templates.md +**** EXECUTION SUMMARY END **** + +Note: "unknown-plugin:custom-skill" origin unclear - verify plugin source. +``` + +## Best Practices + +1. **Analyze entire conversation** - Don't miss early invocations +2. **Deduplicate resources** - Each resource appears once +3. **Maintain chronological order** - Preserve workflow sequence +4. **Use exact prefixes** - `rh-sre:`, `lightspeed-mcp:`, etc. +5. **Compact format** - No spaces after commas +6. **Include all categories** - Even if "None" +7. **Extract docs from "I consulted" statements** - These indicate documentation usage +8. **Path from docs/ onwards** - Not full filesystem paths +9. **Brief explanation** - Help user understand the summary +10. **Handle edge cases gracefully** - Empty workflows, incomplete history + +## Use Cases + +**Audit Trail**: +- Document which components were used for compliance +- Track MCP tool access patterns +- Record skill usage for billing/metrics + +**Learning & Training**: +- Show new users which resources solve specific problems +- Demonstrate skill orchestration patterns +- Illustrate skill orchestration workflows + +**Troubleshooting**: +- Identify which tools were called before an error +- Trace skill invocation sequence +- Document successful workflows for reproduction + +**Workflow Documentation**: +- Create records of complex remediation processes +- Document resource usage for similar future tasks +- Build a library of workflow patterns + +## Integration with Other Skills + +This skill complements other rh-sre skills: + +**After `/remediation` skill**: +``` +User: "Remediate CVE-X" +→ `/remediation` skill executes full workflow (invoked) +User: "Generate execution summary" +→ execution-summary shows complete resource usage +``` + +**For learning workflows**: +``` +User: "Show the fleet" +→ fleet-inventory skill executes +User: "What did you use to do that?" +→ execution-summary shows skills and tools invoked +``` + +**For audit purposes**: +``` +User: "Create playbook for CVE-X and generate audit trail" +→ Workflow executes +→ execution-summary provides compliance record +``` + +The summary output format is designed to be: +- **Machine-readable**: Parseable by scripts/tools +- **Human-readable**: Clear and concise for users +- **Compact**: Minimal token usage +- **Complete**: All resource categories represented +- **Auditable**: Chronological order preserved diff --git a/evaluation/with_skills/rh-sre__execution-summary/instruction.md b/evaluation/with_skills/rh-sre__execution-summary/instruction.md new file mode 100644 index 00000000..5521bb63 --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/instruction.md @@ -0,0 +1,15 @@ +# Execution Summary Task + +You are a Red Hat SRE. Your team just completed an emergency remediation of a critical CVE across your managed fleet. Management needs a structured post-incident execution summary. + +## Scenario +A critical kernel vulnerability was announced. Your team used automation tools to identify affected systems, generate remediation playbooks, execute patching, and verify the fix. Now you need to document what was done. + +## Requirements +- Use MCP tools to query the current state of the fleet, identify which systems were affected, and gather evidence of remediation actions taken +- Produce an execution summary that includes: what was done, which tools and automation were used, the sequence of steps, results and verification outcomes, and any remaining gaps +- Structure the summary so it can be reviewed by management and used for future incident response improvement + +Document the full execution summary, including your methodology and the tools used, in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-sre__execution-summary/solution/solve.sh b/evaluation/with_skills/rh-sre__execution-summary/solution/solve.sh new file mode 100644 index 00000000..68891309 --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/solution/solve.sh @@ -0,0 +1,13 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Execution Summary + +**** EXECUTION SUMMARY START **** +Agents: None +Skills: rh-sre:fleet-inventory,rh-sre:cve-impact +Tools: lightspeed-mcp:get_host_details,lightspeed-mcp:get_cves +Docs: docs/references/cvss-scoring.md,docs/insights/vulnerability-logic.md +**** EXECUTION SUMMARY END **** + +This summary shows all agents, skills, tools, and documentation used during the workflow. +REPORT_EOF diff --git a/evaluation/with_skills/rh-sre__execution-summary/task.toml b/evaluation/with_skills/rh-sre__execution-summary/task.toml new file mode 100644 index 00000000..a983e99f --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__execution-summary" +name = "rh-sre Execution Summary Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "execution-summary", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-sre__execution-summary/tests/llm_judge.py b/evaluation/with_skills/rh-sre__execution-summary/tests/llm_judge.py new file mode 100644 index 00000000..c426d912 --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "tool_purpose_attribution", "file": "/root/report.md", "question": "Does the report pair each tool or MCP call with an explicit purpose statement explaining why it was invoked, rather than just listing tools used or describing actions narratively?", "reference": "A skilled report explicitly pairs each tool (e.g., get_host_details, get_cves) with a 'Purpose:' or 'used to' statement. An unskilled report lists tools in a table or mentions them inline without structured per-tool purpose mapping."}, + {"id": "tool_purpose_categorization", "file": "/root/report.md", "question": "Does the report organize MCP tools into named functional categories (e.g., 'Vulnerability Assessment Tools', 'Fleet Management Tools', 'Remediation Tools') rather than listing all tools in a single flat list?", "reference": "A skilled report groups tools by their functional purpose into labeled sub-sections (e.g., 'Assessment Tools: get_cves, get_cve', 'Fleet Tools: get_host_details'). An unskilled report lists all tools sequentially without categorizing them by function."}, + {"id": "methodology_phases", "file": "/root/report.md", "question": "Does the report organize the execution into at least two named methodology phases (e.g., 'Data Collection', 'Evidence Gathering', 'Triage', 'Verification') as distinct sections or headings?", "reference": "A skilled report structures execution into named phases as headings or sections. An unskilled report presents a flat narrative or simple numbered list without phase-level organization."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-sre__execution-summary/tests/test.sh b/evaluation/with_skills/rh-sre__execution-summary/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-sre__execution-summary/tests/test_outputs.py b/evaluation/with_skills/rh-sre__execution-summary/tests/test_outputs.py new file mode 100644 index 00000000..6cd1228a --- /dev/null +++ b/evaluation/with_skills/rh-sre__execution-summary/tests/test_outputs.py @@ -0,0 +1,55 @@ +""" +Tests for rh-sre__execution-summary per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: skill-specific patterns (not generic report quality). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['summary', 'execution', 'remediation']), ( + "report should mention execution summary or remediation" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 300, "execution summary should be substantial" + + +class TestSkillDependent: + def test_methodology_phases(self): + """Structuring execution into methodology phases + (data collection, evidence gathering, etc.).""" + c = read_report().lower() + phase_terms = [ + "data collection", "evidence gathering", "discovery", + "triage", "assessment", "verification", + "phase 1", "phase 2", "step 1", "step 2", + ] + found = sum(1 for t in phase_terms if t in c) + assert found >= 2, ( + f"should organize execution into methodology phases (found {found})" + ) + + def test_docs_from_consulted(self): + """Extract docs from 'I consulted' statements; path from docs/ or skills/ onwards.""" + c = read_report().lower() + has_docs = any(t in c for t in ["docs/", "skills/", "consult", "documentation"]) + assert has_docs, ( + "should list documentation consulted" + ) diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/Dockerfile b/evaluation/with_skills/rh-sre__fleet-inventory/environment/Dockerfile new file mode 100644 index 00000000..484ebb33 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/Dockerfile @@ -0,0 +1,52 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + + + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..1092dd1d --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- pass rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/with_skills/rh-sre__fleet-inventory/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..fe5d549c --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/SKILL.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/SKILL.md new file mode 100644 index 00000000..530bf58b --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/SKILL.md @@ -0,0 +1,254 @@ +--- +name: fleet-inventory +description: | + Query and display Red Hat Lightspeed managed system inventory. Use this skill for information-gathering requests about the fleet, registered systems, or inventory queries. This skill focuses on discovery and listing only - for remediation actions, transition to the `/remediation` skill. + + **When to use this skill**: + - "Show the managed fleet" + - "List all systems registered in Lightspeed" + - "What systems are affected by CVE-X?" + - "How many RHEL 8 systems do we have?" + - "Show me production systems" + + **When NOT to use this skill** (use `/remediation` skill instead): + - "Remediate CVE-X on these systems" + - "Create a playbook for..." + - "Patch system Y" + + This skill orchestrates MCP tools from lightspeed-mcp to provide comprehensive fleet visibility and system inventory management. +model: inherit +color: blue +--- + +# Fleet Inventory Skill + +This skill queries Red Hat Lightspeed to retrieve and display information about managed systems, registered hosts, and fleet inventory. + +## Prerequisites + +**Required MCP Servers**: `lightspeed-mcp` ([setup guide](https://console.redhat.com/)) + +**Required MCP Tools**: +- `get_host_details` (from lightspeed-mcp) - Retrieve system inventory +- `get_cve_systems` (from lightspeed-mcp) - Find CVE-affected systems + +**Required Environment Variables**: +- `LIGHTSPEED_CLIENT_ID` - Red Hat Lightspeed service account client ID +- `LIGHTSPEED_CLIENT_SECRET` - Red Hat Lightspeed service account secret + +### Prerequisite Validation + +**CRITICAL**: Before executing any operations, execute the `/mcp-lightspeed-validator` skill to verify MCP server availability. + +See **Step 0** in the Workflow section below for implementation details. + +**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-lightspeed-validator/SKILL.md#validation-freshness-policy). + +## When to Use This Skill + +**Use this skill directly when you need**: +- List all systems registered in Red Hat Lightspeed +- Show systems affected by specific CVEs +- Display system details (OS version, tags, last check-in) +- Filter systems by environment, RHEL version, or tags +- Count systems matching criteria +- Verify system registration status + +**Use the `/remediation` skill when you need**: +- Remediate vulnerabilities on systems +- Generate or execute playbooks +- Perform infrastructure changes +- End-to-end CVE remediation workflows + +**How they work together**: Use this skill for discovery ("What systems are affected?"), then transition to the `/remediation` skill for action ("Remediate those systems"). + +## Workflow + +### Step 0: Validate Lightspeed MCP Prerequisites + +**Action**: Execute the `/mcp-lightspeed-validator` skill + +**Note**: Can skip if validation was performed earlier in this session and succeeded. See [Validation Freshness Policy](../mcp-lightspeed-validator/SKILL.md#validation-freshness-policy). + +**How to invoke**: Execute the `/mcp-lightspeed-validator` skill + +**Handle validation result**: +- **If validation PASSED**: Continue to Step 1 +- **If validation PARTIAL** (connectivity test unavailable): + - Warn user: "Configuration appears correct but connectivity could not be tested" + - Ask: "Do you want to proceed? (yes/no)" + - If yes: Continue to Step 1 + - If no: Stop execution +- **If validation FAILED**: + - The validator provides error details and setup instructions + - Wait for user decision (setup/skip/abort) + - If user chooses "skip": Attempt Step 1 anyway (may fail) + - If user chooses "setup" or "abort": Stop execution + +**Example**: +``` +Before retrieving fleet inventory, I'll validate the Lightspeed MCP server configuration. + +[Invoke mcp-lightspeed-validator skill] + +✓ Lightspeed MCP validation successful. +Proceeding with fleet inventory query... +``` + +### Step 1: Retrieve System Inventory + +**Document Consultation** (REQUIRED - Execute FIRST): +1. **Action**: Read [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) using the Read tool to understand the `get_host_details` response format and pagination handling +2. **Output to user**: "I consulted vulnerability-logic.md to understand the `get_host_details` response format and pagination handling." + +**MCP Tool**: `get_host_details` (from lightspeed-mcp) + +**Purpose**: Query Lightspeed for comprehensive system information + +**Parameters**: See [references/01-parameter-reference.md](references/01-parameter-reference.md) for get_host_details/get_cve_systems parameters and response fields. + +**Verification Checklist**: +- ✓ Systems list returned with metadata +- ✓ Total count matches expectation +- ✓ System details include RHEL version, tags, status +- ✓ No authentication errors (401/403) + +**Key Fields to Extract**: +- `id`: Unique system identifier (use for remediation workflows) +- `display_name` / `fqdn`: Human-readable hostname +- `rhel_version`: OS version (critical for remediation compatibility) +- `tags`: Environment labels (production, staging, dev) +- `stale`: Whether system recently checked in (< 7 days) +- `last_seen`: Last Lightspeed client run timestamp + +### Step 2: Filter and Organize Systems + +**Document Consultation** (REQUIRED - Execute FIRST): +1. **Action**: Read [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) using the Read tool to understand fleet inventory reporting structure and best practices +2. **Output to user**: "I consulted vulnerability-logic.md to structure this inventory report." + +Apply user-requested filters and grouping. See [references/01-parameter-reference.md](references/01-parameter-reference.md) for filtering and sorting patterns. + +### Step 3: Query CVE-Affected Systems + +**MCP Tool**: `get_cve_systems` (from lightspeed-mcp) + +**Purpose**: Find systems affected by specific CVEs + +**Parameters**: `cve_id` (CVE-YYYY-NNNNN, uppercase). See [references/01-parameter-reference.md](references/01-parameter-reference.md). + +**Verification Checklist**: +- ✓ CVE ID matches request exactly +- ✓ System list includes remediation status for each +- ✓ Counts are accurate (affected, remediated, still vulnerable) +- ✓ `remediation_available` flag is present + +**Status Interpretation**: +``` +Status: "Vulnerable" +→ CVE affects this system, patch not applied +→ Action: Suggest remediation via `/remediation` skill + +Status: "Patched" +→ CVE previously affected, now remediated +→ Action: No action needed, informational only + +Status: "Not Affected" +→ System not vulnerable to this CVE +→ Action: Exclude from affected count +``` + +### Step 4: Generate Fleet Summary + +Create organized output. **Read [references/03-output-templates.md](references/03-output-templates.md)** for report format (Overview, RHEL/Environment breakdown, System Details, Stale Systems). + +### Step 5: Offer Remediation Transition + +When appropriate, suggest transitioning to the `/remediation` skill: + +```markdown +## Next Steps + +**For CVE Remediation**: +If you need to remediate vulnerabilities on any of these systems, I can help using the `/remediation` skill: + +Examples: +- "Remediate CVE-2024-1234 on web-server-01" +- "Create playbook for all RHEL 8 production systems affected by CVE-2024-5678" +- "Batch remediate critical CVEs on staging environment" + +**For System Investigation**: +- "Show CVEs affecting web-server-01" (use cve-impact skill) +- "Analyze risk for production systems" (use cve-impact skill) +- "List critical vulnerabilities across the fleet" (use cve-impact skill) +``` + +## Dependencies + +### Required MCP Servers +- `lightspeed-mcp` - Red Hat Lightspeed platform access for system inventory and CVE data + +### Required MCP Tools +- `get_host_details` (from lightspeed-mcp) - Retrieve all registered systems with metadata + - Parameters: Optional filters (system_id, hostname_pattern, tags, operating_system) + - Returns: List of systems with id, display_name, fqdn, rhel_version, tags, stale status + +- `get_cve_systems` (from lightspeed-mcp) - Find systems affected by specific CVEs + - Parameters: cve_id (string, format: CVE-YYYY-NNNNN) + - Returns: List of affected systems with vulnerability and remediation status + +### Related Skills +- `mcp-lightspeed-validator` - **PREREQUISITE** - Validates Lightspeed MCP server configuration and connectivity + - Use before: ALL fleet-inventory operations (Step 0 in workflow) + - Purpose: Ensures MCP server is available before attempting tool calls + - Prevents errors from missing configuration or credentials + +- `cve-impact` - Analyze CVE severity and risk after identifying affected systems + - Use after: "What systems are affected by CVE-X?" → "What's the risk of CVE-X?" + +- `cve-validation` - Validate CVE IDs before querying affected systems + - Use before: If CVE ID format is unclear, validate first + +- `system-context` - Get detailed system configuration for specific hosts + - Use after: Fleet discovery identifies systems needing deeper investigation + +- `/remediation` (skill) - Transition to remediation workflows after discovery + - Use after: "Show affected systems" → "Remediate those systems" + +### Reference Documentation +- [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) - CVE severity classification, API response patterns, and filtering strategies + +### Skill Orchestration Pattern + +**Information-First Workflow**: +``` +User Query: "Show the managed fleet" + ↓ +fleet-inventory skill (discovery) + ↓ +Systems identified: 42 total, 15 affected by CVE-2024-1234 + ↓ +User: "What's the risk of CVE-2024-1234?" + ↓ +cve-impact skill (analysis) + ↓ +CVSS 8.1, Critical severity, affects httpd package + ↓ +User: "Remediate CVE-2024-1234 on all production systems" + ↓ +`/remediation` skill (action) + ↓ +Playbook generated and executed +``` + +**Key Principle**: Always start with discovery before taking remediation actions. This ensures informed decisions based on actual fleet state. + +## Output, Examples, Error Handling + +**Read [references/03-output-templates.md](references/03-output-templates.md)** for report format. +**Read [references/04-examples.md](references/04-examples.md)** for fleet, CVE-affected, and environment-filter examples. +**Read [references/05-error-handling.md](references/05-error-handling.md)** for no-results, API errors, and stale system handling. + +## Best Practices + +Start broad then filter; group by environment/RHEL/tier; highlight stale systems; offer `/remediation` transitions; use tables and percentages; declare document consultations; verify prerequisites first. diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/01-parameter-reference.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/01-parameter-reference.md new file mode 100644 index 00000000..6909aa85 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/01-parameter-reference.md @@ -0,0 +1,49 @@ +# Fleet Inventory Parameter Reference + +Read when calling `get_host_details` or `get_cve_systems` to ensure correct parameters. + +## get_host_details + +**Parameters** (based on user query): + +```python +# No filters +get_host_details() + +# Specific system +get_host_details(system_id="abc-123") + +# Hostname pattern +get_host_details(hostname_pattern="web-*") + +# Tag filter +get_host_details(tags=["production"]) + +# RHEL version filter +get_host_details(operating_system__version__startswith="8") + +# Combined +get_host_details(tags=["production", "web-tier"], operating_system__version__startswith="8") +``` + +**Response fields**: id, display_name, fqdn, rhel_version, last_seen, tags, stale, satellite_managed + +## get_cve_systems + +**Parameters**: `cve_id` (string, format CVE-YYYY-NNNNN, uppercase) + +```python +get_cve_systems(cve_id="CVE-2024-1234") +``` + +**Response fields**: cve_id, affected_systems (system_id, display_name, status, remediation_available), total_affected, total_remediated, total_vulnerable + +**Status values**: Vulnerable (patch needed), Patched (no action), Not Affected (exclude) + +## Filtering and Sorting + +**By RHEL**: `[s for s in systems if s['rhel_version'].startswith("8")]` +**By tag**: `[s for s in systems if "production" in s.get('tags', [])]` +**By stale**: `[s for s in systems if not s.get('stale', False)]` +**Sort by last_seen**: `sorted(systems, key=lambda s: s['last_seen'], reverse=True)` +**Sort by display_name**: `sorted(systems, key=lambda s: s['display_name'])` diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/03-output-templates.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/03-output-templates.md new file mode 100644 index 00000000..b0337ed8 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/03-output-templates.md @@ -0,0 +1,80 @@ +# Fleet Inventory Output Templates + +Read when completing a fleet inventory report to format the output. + +## Template 1: Full Fleet Listing + +**User Request**: "Show the managed fleet" + +```markdown +# Managed Fleet Inventory + +I consulted [fleet-management.md](../../../docs/insights/fleet-management.md) to structure this inventory report. + +Retrieved from Red Hat Lightspeed on YYYY-MM-DDTHH:MM:SSZ + +## Fleet Overview +- **Total Registered Systems**: N +- **Active (< 24h)**: N +- **Stale (> 7 days)**: N + +## RHEL Version Distribution +| Version | Count | Percentage | + +## Environment Breakdown +| Environment | Count | Systems | + +## Top 20 Systems (by last check-in) +[Table: display_name, rhel_version, tags, last_seen] + +**Would you like to**: Filter by environment/RHEL, view CVEs, create remediation plans +``` + +## Template 2: CVE-Affected Systems + +**User Request**: "What systems are affected by CVE-X?" + +```markdown +# CVE-X Impact Analysis + +## Affected Systems Summary +- **Total Vulnerable**: N +- **Already Patched**: N +- **Impact Rate**: X% of fleet + +## Vulnerable Systems +| System Name | RHEL Version | Environment | Remediation Available | + +## Already Patched (No Action Needed) +[list] + +## Next Steps +- Use `/remediation` skill for remediation +- Use cve-impact for severity analysis +``` + +## Template 3: Environment-Filtered View + +**User Request**: "Show me production systems" + +```markdown +# Production Systems Inventory + +Filtered by tag: "production" + +## Production Fleet Summary +- **Total**: N +- **RHEL 9.x / 8.x / 7.x** breakdown +- **Active / Stale** counts + +## System Tiers +### Web Tier, Database Tier, Application Tier +[grouped lists] + +## Stale System Alert ⚠️ +[list with action: investigate Lightspeed client] + +## Next Steps +- "Show CVEs affecting production systems" +- "Remediate CVE-X on production web tier" +``` diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/04-examples.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/04-examples.md new file mode 100644 index 00000000..2d08d77a --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/04-examples.md @@ -0,0 +1,32 @@ +# Fleet Inventory Examples + +## Example 1: General Fleet Query + +**User Request**: "Show the managed fleet" + +1. Invoke mcp-lightspeed-validator (Step 0) → PASSED +2. Call `get_host_details()` with no filters +3. Consult fleet-management.md for grouping +4. Group by RHEL version, environment tags +5. Generate Template 1 output +6. Offer next steps (CVE analysis, remediation) + +## Example 2: CVE Impact Query + +**User Request**: "What systems are affected by CVE-2024-1234?" + +1. Invoke mcp-lightspeed-validator (Step 0) → PASSED +2. Call `get_cve_systems(cve_id="CVE-2024-1234")` +3. Separate vulnerable vs. patched systems +4. Generate Template 2 output +5. Suggest /remediation for next steps + +## Example 3: Environment Filter + +**User Request**: "Show me staging systems" + +1. Invoke mcp-lightspeed-validator (Step 0) → PARTIAL +2. Ask user: "Proceed? (yes/no)" → yes +3. Call `get_host_details()` → filter by tag "staging" +4. Group by tier (hostname patterns) +5. Generate Template 3 output diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/05-error-handling.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/05-error-handling.md new file mode 100644 index 00000000..e295f0e6 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/05-error-handling.md @@ -0,0 +1,45 @@ +# Fleet Inventory Error Handling + +Read when errors occur during fleet inventory queries. + +## No Systems Found + +``` +Fleet Inventory Query: No Results + +Query: [user's filter criteria] +Result: No systems match the specified criteria + +Possible reasons: No systems registered, filter too restrictive, tag mismatch +Troubleshooting: Verify at console.redhat.com/insights/inventory, try broader filters +Suggested: "Show the managed fleet" (no filters) +``` + +## Lightspeed API Error + +``` +❌ Fleet Inventory Query: API Error + +Possible causes: MCP not running, auth failure, network, service outage + +Troubleshooting: +1. Run /mcp-lightspeed-validator skill +2. Check LIGHTSPEED_CLIENT_ID and LIGHTSPEED_CLIENT_SECRET (never echo values) +3. Verify at console.redhat.com/settings/service-accounts +4. Check status.redhat.com + +Options: retry | setup | abort +``` + +## Stale System Warning + +``` +⚠️ Stale Systems Detected + +Systems not checked in > 7 days: [list] + +Impact: Vulnerability data may be outdated + +Actions: Verify insights-client, check connectivity, review logs, re-register if needed +Note: Stale systems included but may have outdated CVE data +``` diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/mcp-lightspeed-validator/SKILL.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/mcp-lightspeed-validator/SKILL.md new file mode 100644 index 00000000..e1f1528e --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/mcp-lightspeed-validator/SKILL.md @@ -0,0 +1,61 @@ +--- +name: mcp-lightspeed-validator +description: | + Validate Red Hat Lightspeed MCP server connectivity. Use when the user asks to "validate Lightspeed MCP", "check Lightspeed connection", or when other skills need to verify lightspeed-mcp availability before CVE operations. +model: haiku +color: yellow +--- + +# MCP Lightspeed Validator + +Validates connectivity to the Red Hat Lightspeed MCP server by running a lightweight tool call. + +## When to Use This Skill + +Use when validating Lightspeed MCP before CVE operations, troubleshooting connection issues, or when other skills (e.g. remediation) need to verify availability. Do NOT use for actual CVE queries—use cve-impact or cve-validation. + +## Workflow + +1. **Test connectivity**: Call `vulnerability__get_cves` with **no parameters** (uses default limit=10). Do NOT pass `limit`—some MCP clients incorrectly serialize it as `limit_`, causing validation errors. +2. **If it fails**: Provide a comprehensive message with possible root causes (see below). +3. **Report**: Output a table with validated servers and outcome (emojis). + +## Failure Message (Root Causes) + +When the tool call fails, include: + +``` +❌ Lightspeed MCP connection failed + +**Possible root causes:** +- **Credentials**: LIGHTSPEED_CLIENT_ID or LIGHTSPEED_CLIENT_SECRET not set or invalid +- **Expired credentials**: Red Hat Console tokens may have expired +- **Server not running**: MCP server/container may be stopped +- **Network**: Firewall or proxy blocking console.redhat.com +- **Configuration**: .mcp.json misconfigured or server not registered + +**Troubleshooting:** +1. Verify env vars: LIGHTSPEED_CLIENT_ID, LIGHTSPEED_CLIENT_SECRET (never echo values) +2. Check credentials at: https://console.redhat.com/settings/integrations +3. Restart MCP server or host after config changes +4. Check container logs if using podman/docker +``` + +## Report Format + +Always end with a table: + +| Server | Outcome | +|--------|---------| +| lightspeed-mcp | ✅ PASSED | +| lightspeed-mcp | ❌ FAILED | + +Use ✅ for success, ❌ for failure, ⚠️ for partial (e.g. connected but error on tool). + +## Dependencies + +### Required MCP Tools +- `vulnerability__get_cves` or `get_cves` (from lightspeed-mcp) - Connectivity test + +### Required MCP Servers +- `lightspeed-mcp` - Red Hat Lightspeed vulnerability and inventory data diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/instruction.md b/evaluation/with_skills/rh-sre__fleet-inventory/instruction.md new file mode 100644 index 00000000..3074bb9c --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/instruction.md @@ -0,0 +1,17 @@ +# Fleet Inventory Task + +You are a Red Hat SRE. Your manager has asked for a current snapshot of all RHEL systems in your managed fleet ahead of an upcoming compliance audit. + +## Scenario +The compliance team needs to know exactly what systems you manage, their RHEL versions, patch levels, and any outstanding vulnerability exposure. They need this by end of day. + +## Requirements +- Query the fleet to enumerate all managed RHEL systems +- For each system, report: hostname, RHEL version, last check-in date, and patch status +- Identify which systems have outstanding CVEs, grouped by severity +- Flag any systems that are stale (not checking in) or running unsupported RHEL versions +- Summarize the fleet's overall health and compliance readiness + +Document your methodology, findings, and fleet assessment in `/root/report.md`. + +Use MCP tools to query data. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/solution/solve.sh b/evaluation/with_skills/rh-sre__fleet-inventory/solution/solve.sh new file mode 100644 index 00000000..dc994408 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/solution/solve.sh @@ -0,0 +1,25 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Fleet Inventory Report + +## Systems Summary +| Hostname | RHEL | Environment | Status | Last Seen | +|----------|------|-------------|--------|-----------| +| web-01 | 9.3 | Production | Active | 2024-01-15 | +| db-01 | 9.3 | Production | Active | 2024-01-15 | +| dev-01 | 8.9 | Development | Stale | 2024-01-01 | + +## Data Source +Queried via `get_host_details` with pagination. Key fields: rhel_version, tags, stale, last_seen. + +## CVE-Affected Systems +Use `get_cve_systems` with cve_id (CVE-YYYY-NNNNN). Check remediation_available flag. + +## Status Interpretation +- **Vulnerable**: CVE affects system, patch not applied → suggest /remediation +- **Patched**: Previously affected, now remediated → no action +- **Not Affected**: Exclude from affected count + +## Next Steps +For CVE remediation, transition to /remediation skill. +REPORT_EOF diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/task.toml b/evaluation/with_skills/rh-sre__fleet-inventory/task.toml new file mode 100644 index 00000000..cff6fe66 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__fleet-inventory" +name = "rh-sre Fleet Inventory Query Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "fleet-inventory", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/tests/llm_judge.py b/evaluation/with_skills/rh-sre__fleet-inventory/tests/llm_judge.py new file mode 100644 index 00000000..977611c9 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/tests/llm_judge.py @@ -0,0 +1,92 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "system_id_for_remediation", "file": "/root/report.md", "question": "Does the report track individual system identifiers (system_uuid, system_id, or host UUID) and link them to specific remediation follow-up actions, rather than just listing hostnames?", "reference": "A skilled report captures system UUIDs or identifiers to enable programmatic remediation API calls. An unskilled report lists hostnames or display names without machine-usable identifiers for follow-up."}, + {"id": "classification_methodology", "file": "/root/report.md", "question": "Does the report reference a classification methodology, classification criteria, or vulnerability classification framework for interpreting CVE status, rather than using ad-hoc severity labeling?", "reference": "A skilled report consults or references CVE classification criteria or methodology documentation before interpreting vulnerability data. An unskilled report classifies CVEs based on general knowledge without referencing established criteria."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/tests/test.sh b/evaluation/with_skills/rh-sre__fleet-inventory/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/tests/test_outputs.py b/evaluation/with_skills/rh-sre__fleet-inventory/tests/test_outputs.py new file mode 100644 index 00000000..f8c232d0 --- /dev/null +++ b/evaluation/with_skills/rh-sre__fleet-inventory/tests/test_outputs.py @@ -0,0 +1,67 @@ +""" +Tests for rh-sre__fleet-inventory per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['system', 'host', 'fleet', 'inventory']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_system_identifier_tracking(self): + """Skill teaches tracking system identifiers for follow-up actions. + Without skill, agents list systems without identifiers for remediation.""" + c = read_report().lower() + assert any(t in c for t in [ + "system id", "system_id", "system_uuid", "uuid", "identifier", + ]) and any(t in c for t in [ + "remediat", "follow-up", "subsequent", "action", "track", + ]), ( + "should track system identifiers for follow-up remediation actions" + ) + + def test_remediation_transition_offer(self): + """Skill: Offer transition to a remediation workflow for CVE remediation.""" + c = read_report().lower() + assert any(t in c for t in [ + "next step", "remediate", "playbook", + "remediation workflow", "remediation action", + ]), "should offer next steps for remediation" + + def test_classification_criteria_reference(self): + """Skill/docs teach consulting classification criteria or reference + documentation before interpreting vulnerability data. Without skill, + agents classify CVEs based on general knowledge alone.""" + c = read_report().lower() + assert any(t in c for t in [ + "classification criteria", "classification methodology", + "vulnerability classification", "cve classification", + ]) or ( + "classification" in c and any(t in c for t in [ + "consult", "reference", "methodology", "criteria", + ]) + ), "should reference CVE classification criteria or methodology" diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/Dockerfile b/evaluation/with_skills/rh-sre__job-template-creator/environment/Dockerfile new file mode 100644 index 00000000..d5c9e7b7 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/Dockerfile @@ -0,0 +1,56 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + + + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + }, \ + "aap-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-aap-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..1092dd1d --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- pass rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/mcp-servers/mock-aap-mcp.py b/evaluation/with_skills/rh-sre__job-template-creator/environment/mcp-servers/mock-aap-mcp.py new file mode 100644 index 00000000..d8ae4fd5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/mcp-servers/mock-aap-mcp.py @@ -0,0 +1,1048 @@ +#!/usr/bin/env python3 +""" +Mock AAP (Ansible Automation Platform) MCP Server + +Simulates the AAP MCP gateway for per-skill evaluation tasks. Implements +the full set of tools used by rh-sre skills: + - job_templates_list / job_templates_retrieve + - projects_list + - job_templates_launch_retrieve + - jobs_retrieve / jobs_stdout_retrieve + - jobs_job_events_list / jobs_job_host_summaries_list + - jobs_relaunch_retrieve + - inventories_list / hosts_list + +Data mirrors a realistic AAP deployment: + - 6 job templates (3 remediation, 1 compliance, 1 patching, 1 reporting) + - 3 projects (remediation, compliance, reporting) + - 3 inventories (production 30 hosts, staging 15 hosts, all-managed 63 hosts) + - 12 recent jobs with varied statuses + +Follows the same mock-server pattern as mock-lightspeed-mcp.py. +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +mcp = FastMCP("aap-mcp") + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +def _ts(delta: timedelta) -> str: + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +# --------------------------------------------------------------------------- +# Mock data: Projects +# --------------------------------------------------------------------------- + +MOCK_PROJECTS = [ + { + "id": 6, + "type": "project", + "name": "Remediation Playbooks", + "description": "CVE and security remediation playbooks managed via Git", + "scm_type": "git", + "scm_url": "https://github.com/org/remediation-playbooks.git", + "scm_branch": "main", + "scm_revision": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + "status": "successful", + "last_job_run": _ts(timedelta(hours=2)), + "last_update_failed": False, + "created": _ts(timedelta(days=90)), + "modified": _ts(timedelta(hours=2)), + }, + { + "id": 7, + "type": "project", + "name": "Compliance Checks", + "description": "STIG and CIS compliance scanning playbooks", + "scm_type": "git", + "scm_url": "https://github.com/org/compliance-playbooks.git", + "scm_branch": "main", + "scm_revision": "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3", + "status": "successful", + "last_job_run": _ts(timedelta(days=1)), + "last_update_failed": False, + "created": _ts(timedelta(days=120)), + "modified": _ts(timedelta(days=1)), + }, + { + "id": 8, + "type": "project", + "name": "Fleet Reporting", + "description": "System inventory and health reporting playbooks", + "scm_type": "git", + "scm_url": "https://github.com/org/fleet-reports.git", + "scm_branch": "main", + "scm_revision": "c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "status": "successful", + "last_job_run": _ts(timedelta(days=3)), + "last_update_failed": False, + "created": _ts(timedelta(days=180)), + "modified": _ts(timedelta(days=3)), + }, +] + +# --------------------------------------------------------------------------- +# Mock data: Inventories & Hosts +# --------------------------------------------------------------------------- + +MOCK_INVENTORIES = [ + { + "id": 1, + "type": "inventory", + "name": "Production Systems", + "description": "All production RHEL systems across data centers", + "total_hosts": 30, + "has_active_failures": False, + "hosts_with_active_failures": 0, + "total_groups": 5, + "groups_with_active_failures": 0, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=365)), + "modified": _ts(timedelta(days=1)), + }, + { + "id": 2, + "type": "inventory", + "name": "Staging Systems", + "description": "Pre-production staging environment", + "total_hosts": 15, + "has_active_failures": False, + "hosts_with_active_failures": 0, + "total_groups": 3, + "groups_with_active_failures": 0, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=300)), + "modified": _ts(timedelta(days=7)), + }, + { + "id": 3, + "type": "inventory", + "name": "All Managed Systems", + "description": "Complete fleet: production, staging, development, QA, legacy", + "total_hosts": 63, + "has_active_failures": True, + "hosts_with_active_failures": 2, + "total_groups": 8, + "groups_with_active_failures": 1, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=365)), + "modified": _ts(timedelta(hours=6)), + }, +] + + +def _generate_hosts(inventory_id: int) -> list[dict]: + """Generate realistic hosts for an inventory.""" + hosts: list[dict] = [] + if inventory_id == 1: + roles = ["web", "db", "app", "lb", "monitoring", "cache"] + for i, role in enumerate(roles): + for j in range(5 if role in ("web", "app") else 4 if role == "db" else 3 if role == "monitoring" else 2): + hosts.append({ + "id": len(hosts) + 1, + "type": "host", + "name": f"{role}-{j+1:02d}.prod.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "production", "role": "{role}"}}', + }) + if len(hosts) >= 30: + break + if len(hosts) >= 30: + break + elif inventory_id == 2: + for i in range(15): + role = ["web", "db", "app"][i % 3] + hosts.append({ + "id": 100 + i, + "type": "host", + "name": f"{role}-{i+1:02d}.staging.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "staging", "role": "{role}"}}', + }) + elif inventory_id == 3: + for i in range(30): + hosts.append({ + "id": 200 + i, + "type": "host", + "name": f"host-{i+1:02d}.prod.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": i in (45, 58), + "variables": f'{{"rhel_version": "9.3", "environment": "production"}}', + }) + for i in range(15): + hosts.append({ + "id": 230 + i, + "type": "host", + "name": f"host-{i+1:02d}.staging.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "staging"}}', + }) + for i in range(10): + hosts.append({ + "id": 245 + i, + "type": "host", + "name": f"dev-{i+1:02d}.dev.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "8.9", "environment": "development"}}', + }) + for i in range(5): + hosts.append({ + "id": 255 + i, + "type": "host", + "name": f"qa-{i+1:02d}.qa.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.2", "environment": "qa"}}', + }) + for i in range(3): + hosts.append({ + "id": 260 + i, + "type": "host", + "name": f"legacy-{i+1:02d}.corp.example.com", + "inventory": inventory_id, + "enabled": i < 2, + "has_active_failures": i == 2, + "variables": f'{{"rhel_version": "7.9", "environment": "legacy"}}', + }) + return hosts + + +# --------------------------------------------------------------------------- +# Mock data: Job Templates +# --------------------------------------------------------------------------- + +MOCK_JOB_TEMPLATES = [ + { + "id": 10, + "type": "job_template", + "name": "CVE Remediation - Kernel Update", + "description": "Kernel update with boom snapshot for rollback safety", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": True, + "job_type": "check", + "verbosity": 1, + "timeout": 3600, + "forks": 5, + "status": "successful", + "last_job_run": _ts(timedelta(hours=4)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1001, "status": "successful", "finished": _ts(timedelta(hours=4))}, + }, + "created": _ts(timedelta(days=60)), + "modified": _ts(timedelta(days=2)), + }, + { + "id": 11, + "type": "job_template", + "name": "CVE Remediation - Package Update", + "description": "General package update for CVE remediation with needs-restarting check", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-package-update.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": False, + "job_type": "check", + "verbosity": 1, + "timeout": 1800, + "forks": 10, + "status": "successful", + "last_job_run": _ts(timedelta(hours=12)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1005, "status": "successful", "finished": _ts(timedelta(hours=12))}, + }, + "created": _ts(timedelta(days=45)), + "modified": _ts(timedelta(days=5)), + }, + { + "id": 12, + "type": "job_template", + "name": "CVE Remediation - Generic", + "description": "Generic CVE remediation template for ad-hoc patches", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-remediation.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": True, + "job_type": "check", + "verbosity": 1, + "timeout": 3600, + "forks": 5, + "status": "never updated", + "last_job_run": None, + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + }, + "created": _ts(timedelta(days=30)), + "modified": _ts(timedelta(days=30)), + }, + { + "id": 20, + "type": "job_template", + "name": "Compliance Check - STIG", + "description": "Run STIG compliance scan across fleet", + "inventory": 3, + "project": 7, + "playbook": "playbooks/compliance/check-all.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": False, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 7200, + "forks": 20, + "status": "successful", + "last_job_run": _ts(timedelta(days=1)), + "summary_fields": { + "project": {"id": 7, "name": "Compliance Checks", "status": "successful"}, + "inventory": {"id": 3, "name": "All Managed Systems", "total_hosts": 63}, + "credentials": [ + {"id": 2, "name": "compliance-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1010, "status": "successful", "finished": _ts(timedelta(days=1))}, + }, + "created": _ts(timedelta(days=180)), + "modified": _ts(timedelta(days=14)), + }, + { + "id": 25, + "type": "job_template", + "name": "Emergency Patching", + "description": "Emergency patch application — NO become enabled (misconfigured)", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/emergency-patch.yml", + "become_enabled": False, + "ask_job_type_on_launch": False, + "ask_variables_on_launch": False, + "ask_limit_on_launch": False, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 600, + "forks": 25, + "status": "failed", + "last_job_run": _ts(timedelta(days=7)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1020, "status": "failed", "finished": _ts(timedelta(days=7))}, + }, + "created": _ts(timedelta(days=200)), + "modified": _ts(timedelta(days=200)), + }, + { + "id": 30, + "type": "job_template", + "name": "Fleet Health Report", + "description": "Generate fleet health and inventory report", + "inventory": 3, + "project": 8, + "playbook": "playbooks/reporting/fleet-health.yml", + "become_enabled": False, + "ask_job_type_on_launch": False, + "ask_variables_on_launch": True, + "ask_limit_on_launch": False, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 1800, + "forks": 30, + "status": "successful", + "last_job_run": _ts(timedelta(hours=6)), + "summary_fields": { + "project": {"id": 8, "name": "Fleet Reporting", "status": "successful"}, + "inventory": {"id": 3, "name": "All Managed Systems", "total_hosts": 63}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1025, "status": "successful", "finished": _ts(timedelta(hours=6))}, + }, + "created": _ts(timedelta(days=120)), + "modified": _ts(timedelta(days=14)), + }, +] + +# --------------------------------------------------------------------------- +# Mock data: Jobs (recent runs) +# --------------------------------------------------------------------------- + +PROD_HOSTS = [ + "web-01.prod.example.com", + "web-02.prod.example.com", + "db-01.prod.example.com", + "db-02.prod.example.com", + "app-01.prod.example.com", + "app-02.prod.example.com", +] + +MOCK_JOBS = [ + { + "id": 1001, + "type": "job", + "name": "CVE Remediation - Kernel Update", + "job_type": "check", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=4, minutes=30)), + "finished": _ts(timedelta(hours=4)), + "elapsed": 1800.0, + "job_template": 10, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "limit": "web-01.prod.example.com,web-02.prod.example.com,db-01.prod.example.com", + "extra_vars": '{"target_cve": "CVE-2024-12345", "remediation_mode": "automated", "verify_after": true}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 10, "name": "CVE Remediation - Kernel Update"}, + }, + }, + { + "id": 1002, + "type": "job", + "name": "CVE Remediation - Kernel Update", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=3, minutes=45)), + "finished": _ts(timedelta(hours=3)), + "elapsed": 2700.0, + "job_template": 10, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "limit": "web-01.prod.example.com,web-02.prod.example.com,db-01.prod.example.com", + "extra_vars": '{"target_cve": "CVE-2024-12345", "remediation_mode": "automated", "verify_after": true}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 10, "name": "CVE Remediation - Kernel Update"}, + }, + }, + { + "id": 1005, + "type": "job", + "name": "CVE Remediation - Package Update", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=12, minutes=20)), + "finished": _ts(timedelta(hours=12)), + "elapsed": 1200.0, + "job_template": 11, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-package-update.yml", + "limit": "", + "extra_vars": '{"target_cve": "CVE-2024-54321"}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 11, "name": "CVE Remediation - Package Update"}, + }, + }, + { + "id": 1010, + "type": "job", + "name": "Compliance Check - STIG", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(days=1, hours=2)), + "finished": _ts(timedelta(days=1)), + "elapsed": 7200.0, + "job_template": 20, + "inventory": 3, + "project": 7, + "playbook": "playbooks/compliance/check-all.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "scheduled", + "summary_fields": { + "job_template": {"id": 20, "name": "Compliance Check - STIG"}, + }, + }, + { + "id": 1020, + "type": "job", + "name": "Emergency Patching", + "job_type": "run", + "status": "failed", + "failed": True, + "started": _ts(timedelta(days=7, hours=1)), + "finished": _ts(timedelta(days=7)), + "elapsed": 3600.0, + "job_template": 25, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/emergency-patch.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 25, "name": "Emergency Patching"}, + }, + }, + { + "id": 1025, + "type": "job", + "name": "Fleet Health Report", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=6, minutes=30)), + "finished": _ts(timedelta(hours=6)), + "elapsed": 1800.0, + "job_template": 30, + "inventory": 3, + "project": 8, + "playbook": "playbooks/reporting/fleet-health.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "scheduled", + "summary_fields": { + "job_template": {"id": 30, "name": "Fleet Health Report"}, + }, + }, +] + +_next_job_id = 2000 + + +# --------------------------------------------------------------------------- +# Mock stdout generators +# --------------------------------------------------------------------------- + +def _generate_stdout(job: dict) -> str: + """Generate realistic Ansible playbook stdout for a job.""" + playbook_name = job.get("name", "Unknown") + job_type = job.get("job_type", "run") + status = job.get("status", "successful") + limit = job.get("limit", "") + hosts = limit.split(",") if limit else PROD_HOSTS[:3] + hosts = [h.strip() for h in hosts if h.strip()] + extra_vars = job.get("extra_vars", "{}") + mode = " (CHECK MODE)" if job_type == "check" else "" + + lines = [] + lines.append(f"PLAY [{playbook_name}] *****") + lines.append("") + + lines.append(f"TASK [Gathering Facts{mode}] *****") + for h in hosts: + lines.append(f"ok: [{h}]") + lines.append("") + + if "kernel" in playbook_name.lower(): + lines.append(f"TASK [Create boom snapshot for rollback{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}] => {{\"msg\": \"boom create --title pre-remediation-CVE-2024-12345\"}}") + lines.append("") + + lines.append(f"TASK [Check disk space for kernel update{mode}] *****") + for h in hosts: + lines.append(f"ok: [{h}] => {{\"msg\": \"Disk space OK: 45% used\"}}") + lines.append("") + + lines.append(f"TASK [Update kernel package{mode}] *****") + for h in hosts: + result = "changed" if status == "successful" else "fatal" + if result == "changed": + lines.append(f'changed: [{h}] => {{"msg": "kernel-5.14.0-362.24.1.el9_3 -> kernel-5.14.0-362.24.2.el9_3"}}') + else: + lines.append(f'fatal: [{h}]: FAILED! => {{"msg": "Permission denied", "rc": 1}}') + lines.append("") + + lines.append(f"TASK [Check if reboot is needed (needs-restarting -r){mode}] *****") + for h in hosts: + lines.append(f'changed: [{h}] => {{"rc": 1, "msg": "Reboot is required to fully utilize updates."}}') + lines.append("") + + elif "package" in playbook_name.lower(): + lines.append(f"TASK [Update target packages for CVE remediation{mode}] *****") + for h in hosts: + lines.append(f'changed: [{h}] => {{"msg": "httpd-2.4.53-7.el9 -> httpd-2.4.57-8.el9"}}') + lines.append("") + + lines.append(f"TASK [Restart affected services{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}]") + lines.append("") + + lines.append(f"TASK [Verify service health{mode}] *****") + for h in hosts: + lines.append(f'ok: [{h}] => {{"msg": "Service httpd is running"}}') + lines.append("") + + elif "emergency" in playbook_name.lower() and status == "failed": + lines.append(f"TASK [Apply emergency patch{mode}] *****") + for h in hosts: + lines.append(f'fatal: [{h}]: FAILED! => {{"msg": "Missing sudo password (become_enabled not set)", "rc": 1}}') + lines.append("") + lines.append("NO MORE HOSTS LEFT *****") + lines.append("") + + else: + lines.append(f"TASK [Execute playbook tasks{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}]") + lines.append("") + + lines.append("PLAY RECAP *****") + for h in hosts: + if status == "successful": + ok_count = random.randint(3, 6) + changed_count = random.randint(1, 3) + lines.append(f"{h:<45} : ok={ok_count} changed={changed_count} unreachable=0 failed=0 skipped=0 rescued=0 ignored=0") + elif status == "failed": + lines.append(f"{h:<45} : ok=1 changed=0 unreachable=0 failed=1 skipped=0 rescued=0 ignored=0") + lines.append("") + + return "\n".join(lines) + + +def _generate_events(job: dict) -> list[dict]: + """Generate realistic Ansible task events for a job.""" + hosts = (job.get("limit", "").split(",") if job.get("limit") else PROD_HOSTS[:3]) + hosts = [h.strip() for h in hosts if h.strip()] + events: list[dict] = [] + eid = 1 + + task_names = ["Gathering Facts"] + if "kernel" in job.get("name", "").lower(): + task_names += [ + "Create boom snapshot for rollback", + "Check disk space for kernel update", + "Update kernel package", + "Check if reboot is needed (needs-restarting -r)", + ] + elif "package" in job.get("name", "").lower(): + task_names += [ + "Update target packages for CVE remediation", + "Restart affected services", + "Verify service health", + ] + else: + task_names += ["Execute playbook tasks"] + + for task_name in task_names: + for host in hosts: + is_failed = job.get("status") == "failed" and task_name != "Gathering Facts" + events.append({ + "id": eid, + "type": "job_event", + "event": "runner_on_ok" if not is_failed else "runner_on_failed", + "task": task_name, + "host": host, + "host_name": host, + "play": job.get("name", ""), + "changed": task_name != "Gathering Facts" and not is_failed, + "failed": is_failed, + "event_data": { + "task": task_name, + "host": host, + "res": { + "changed": task_name != "Gathering Facts" and not is_failed, + "msg": "Task completed" if not is_failed else "Permission denied", + }, + }, + "created": _ts(timedelta(hours=4, minutes=30 - eid)), + }) + eid += 1 + + return events + + +def _generate_host_summaries(job: dict) -> list[dict]: + """Generate per-host summaries for a job.""" + hosts = (job.get("limit", "").split(",") if job.get("limit") else PROD_HOSTS[:3]) + hosts = [h.strip() for h in hosts if h.strip()] + summaries: list[dict] = [] + + for i, host in enumerate(hosts): + is_failed = job.get("status") == "failed" + summaries.append({ + "id": i + 1, + "type": "job_host_summary", + "host": i + 1, + "host_name": host, + "ok": 1 if is_failed else random.randint(3, 6), + "changed": 0 if is_failed else random.randint(1, 3), + "dark": 0, + "failures": 1 if is_failed else 0, + "skipped": 0, + "processed": 1, + "failed": is_failed, + }) + + return summaries + + +# --------------------------------------------------------------------------- +# MCP Tools: Job Management +# --------------------------------------------------------------------------- + +@mcp.tool() +def job_templates_list( + page_size: int = 10, + search: Optional[str] = None, +) -> dict: + """List available job templates in AAP. + + Args: + page_size: Number of results per page (default 10, max 200). + search: Optional search string to filter templates by name. + """ + results = MOCK_JOB_TEMPLATES + if search: + s = search.lower() + results = [t for t in results if s in t["name"].lower() or s in t.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def job_templates_retrieve(id: str) -> dict: + """Retrieve detailed information about a specific job template. + + Args: + id: Job template ID (as string). + """ + tid = int(id) + template = next((t for t in MOCK_JOB_TEMPLATES if t["id"] == tid), None) + if not template: + return {"detail": f"Not found. Job template {id} does not exist."} + return template + + +@mcp.tool() +def projects_list( + page_size: int = 50, + search: Optional[str] = None, +) -> dict: + """List available projects in AAP. + + Args: + page_size: Number of results per page. + search: Optional search string to filter projects by name. + """ + results = MOCK_PROJECTS + if search: + s = search.lower() + results = [p for p in results if s in p["name"].lower() or s in p.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def job_templates_launch_retrieve( + id: str, + requestBody: Optional[dict] = None, +) -> dict: + """Launch a job from a job template. + + Args: + id: Job template ID to launch. + requestBody: Optional launch parameters including job_type ('run' or 'check'), + extra_vars (dict), and limit (comma-separated host list). + """ + global _next_job_id + tid = int(id) + template = next((t for t in MOCK_JOB_TEMPLATES if t["id"] == tid), None) + if not template: + return {"detail": f"Not found. Job template {id} does not exist."} + + body = requestBody or {} + job_type = body.get("job_type", template.get("job_type", "run")) + + if not template.get("ask_job_type_on_launch") and job_type != template.get("job_type"): + return { + "error": f"Cannot override job_type: ask_job_type_on_launch is disabled on template {id}", + } + + job_id = _next_job_id + _next_job_id += 1 + + new_job = { + "id": job_id, + "type": "job", + "name": template["name"], + "job_type": job_type, + "status": "pending", + "failed": False, + "started": _ts(timedelta(seconds=0)), + "finished": None, + "elapsed": 0.0, + "job_template": tid, + "inventory": template["inventory"], + "project": template["project"], + "playbook": template["playbook"], + "limit": body.get("limit", ""), + "extra_vars": str(body.get("extra_vars", {})), + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": tid, "name": template["name"]}, + }, + } + MOCK_JOBS.append(new_job) + + # Simulate job completion after launch + new_job["status"] = "successful" + new_job["finished"] = _ts(timedelta(seconds=-300)) + new_job["elapsed"] = 300.0 + + return { + "job": job_id, + "status": "pending", + "type": "job", + "url": f"/api/controller/v2/jobs/{job_id}/", + "related": { + "stdout": f"/api/controller/v2/jobs/{job_id}/stdout/", + "job_events": f"/api/controller/v2/jobs/{job_id}/job_events/", + "job_host_summaries": f"/api/controller/v2/jobs/{job_id}/job_host_summaries/", + }, + } + + +@mcp.tool() +def jobs_retrieve(id: int) -> dict: + """Get the status and details of a job run. + + Args: + id: Job ID to retrieve. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + return job + + +@mcp.tool() +def jobs_list(page_size: int = 10) -> dict: + """List recent job runs. + + Args: + page_size: Number of results to return. + """ + results = sorted(MOCK_JOBS, key=lambda j: j.get("started", ""), reverse=True) + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def jobs_stdout_retrieve(id: int, format: str = "txt") -> dict: + """Get the stdout (console output) from a job run. + + Args: + id: Job ID. + format: Output format ('txt' or 'json'). Default 'txt'. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + return { + "content": _generate_stdout(job), + "range": {"start": 0, "end": 1}, + } + + +@mcp.tool() +def jobs_job_events_list(id: int, page_size: int = 50) -> dict: + """Get task-level events for a job run. + + Args: + id: Job ID. + page_size: Number of events to return. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + events = _generate_events(job) + return { + "count": len(events), + "next": None, + "previous": None, + "results": events[:page_size], + } + + +@mcp.tool() +def jobs_job_host_summaries_list(id: int) -> dict: + """Get per-host execution summaries for a job run. + + Args: + id: Job ID. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + summaries = _generate_host_summaries(job) + return { + "count": len(summaries), + "next": None, + "previous": None, + "results": summaries, + } + + +@mcp.tool() +def jobs_relaunch_retrieve( + id: int, + hosts: str = "all", + job_type: str = "run", +) -> dict: + """Relaunch a previously completed or failed job. + + Args: + id: Original job ID to relaunch. + hosts: Which hosts to target ('all' or 'failed'). + job_type: Job type for relaunch ('run' or 'check'). + """ + global _next_job_id + original = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not original: + return {"detail": f"Not found. Job {id} does not exist."} + + new_id = _next_job_id + _next_job_id += 1 + + new_job = { + **original, + "id": new_id, + "job_type": job_type, + "status": "successful", + "failed": False, + "started": _ts(timedelta(seconds=0)), + "finished": _ts(timedelta(seconds=-300)), + "elapsed": 300.0, + "launch_type": "relaunch", + } + MOCK_JOBS.append(new_job) + + return { + "job": new_id, + "status": "pending", + "type": "job", + "url": f"/api/controller/v2/jobs/{new_id}/", + } + + +# --------------------------------------------------------------------------- +# MCP Tools: Inventory Management +# --------------------------------------------------------------------------- + +@mcp.tool() +def inventories_list( + page_size: int = 10, + search: Optional[str] = None, +) -> dict: + """List available inventories in AAP. + + Args: + page_size: Number of results per page. + search: Optional search string to filter inventories. + """ + results = MOCK_INVENTORIES + if search: + s = search.lower() + results = [inv for inv in results if s in inv["name"].lower() or s in inv.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def hosts_list( + inventory_id: Optional[int] = None, + page_size: int = 50, + search: Optional[str] = None, +) -> dict: + """List hosts in an inventory. + + Args: + inventory_id: Filter by inventory ID. If not provided, lists hosts from all inventories. + page_size: Number of results per page. + search: Optional search string to filter hosts by name. + """ + inv_id = inventory_id or 1 + hosts = _generate_hosts(inv_id) + if search: + s = search.lower() + hosts = [h for h in hosts if s in h["name"].lower()] + return { + "count": len(hosts), + "next": None if len(hosts) <= page_size else f"/api/controller/v2/hosts/?page=2", + "previous": None, + "results": hosts[:page_size], + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/with_skills/rh-sre__job-template-creator/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..fe5d549c --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/SKILL.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/SKILL.md new file mode 100644 index 00000000..510776ca --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/SKILL.md @@ -0,0 +1,321 @@ +--- +name: job-template-creator +description: | + Create AAP (Ansible Automation Platform) job templates for executing playbooks. Use when users request: + - "Create a job template for this playbook" + - "Set up a template to run remediation playbooks" + - "Configure AAP to execute this playbook" + - "Add a new job template for CVE remediation" + + This skill guides through adding playbooks to Git projects and creating job templates via AAP Web UI. +model: inherit +color: blue +--- + +# AAP Job Template Creator Skill + +This skill helps SREs create AAP job templates for executing Ansible playbooks, particularly for CVE remediation workflows. + +## Prerequisites + +**Required AAP Components**: +- AAP (Ansible Automation Platform) instance with API access +- Projects configured with playbooks +- Inventories with target hosts +- Credentials for authentication + +**Required MCP Servers**: `aap-mcp-job-management` ([setup guide](https://docs.redhat.com/)) + +**Currently Available MCP Tools** (read-only): +- `job_templates_list` - List existing templates +- `job_templates_retrieve` - Get template details +- `projects_list` - List available projects +- `inventories_list` - List available inventories + +**Missing MCP Tools** (needed for creation): +- ⚠️ `job_templates_create` - **NOT CURRENTLY AVAILABLE** +- ⚠️ `job_templates_update` - **NOT CURRENTLY AVAILABLE** + +**Required Environment Variables**: +- `AAP_MCP_SERVER` - Base URL for the MCP endpoint of the AAP server (must point to the AAP MCP gateway) +- `AAP_API_TOKEN` - AAP API authentication token + +### Prerequisite Validation + +**CRITICAL**: Before executing operations, execute the `/mcp-aap-validator` skill to verify AAP MCP server availability. + +**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-aap-validator/SKILL.md#validation-freshness-policy). + +**How to invoke**: Execute the `/mcp-aap-validator` skill + +**Handle validation result**: +- **If validation PASSED**: Continue with job template creation workflow +- **If validation PARTIAL**: Warn user and ask to proceed +- **If validation FAILED**: Stop execution, provide setup instructions from validator + +## Current Limitation: No Create Tools Available + +⚠️ **IMPORTANT**: The current AAP MCP implementation does **NOT** include tools to create job templates programmatically. The available MCP tools are read-only (list, retrieve, launch). + +**Current Approach**: +- Job templates must be created through the **AAP Web UI** +- This skill provides step-by-step instructions for Web UI creation +- Future MCP tool additions will enable programmatic template creation + +This skill documents both the **current manual workflow** and the **intended automated workflow** for when creation tools become available. + +## When to Use This Skill + +**Use this skill when you need**: +- Create a new job template for a remediation playbook +- Configure AAP to execute dynamically generated playbooks +- Set up templates for CVE remediation workflows +- Automate job template creation as part of remediation setup + +**Do NOT use this skill when**: +- Job templates already exist (use `/playbook-executor` skill instead) +- Only need to execute existing templates (use `job_templates_launch_retrieve`) +- Need to modify existing templates (requires AAP Web UI currently) + +## Invocation from playbook-executor + +When invoked from the [playbook-executor](../playbook-executor/SKILL.md) skill (Scenario 3 - No suitable template), this skill receives playbook content in context. The playbook-executor invokes with an instruction such as: + +``` +Create a job template for this remediation playbook. Playbook: [content]. Filename: [filename]. Path: [our_playbook_path]. CVE: [cve_id]. Target systems: [list]. +``` + +**When playbook content is provided**: +- Use the provided content for Phase 1 (Prepare Playbook in Git) instead of asking the user to supply it +- Write the playbook to the specified path in the user's Git repository (ask for repo path if not provided) +- Follow the Git flow: add, commit (with checkpoint for confirmation), push +- Then guide template creation via AAP Web UI (Phase 4) +- **Output**: Include the created template ID and name in the final report so playbook-executor can retrieve and validate it + +**Phase 0 - Check context**: If playbook content is provided by the invoking skill, execute the git flow (write, add, commit with confirmation checkpoint, push) before guiding template creation. Otherwise, use the existing manual flow where the user supplies the playbook. + +## Workflow + +### Phase 0: Validate AAP MCP Prerequisites + +**Action**: Execute the `/mcp-aap-validator` skill + +**Note**: Can skip if validation was performed earlier in this session and succeeded. See [Validation Freshness Policy](../mcp-aap-validator/SKILL.md#validation-freshness-policy). + +**How to invoke**: Execute the `/mcp-aap-validator` skill + +**Handle validation result**: +- **If validation PASSED**: Continue to Phase 1 +- **If validation PARTIAL**: Warn user and ask to proceed +- **If validation FAILED**: Stop execution, user must set up AAP MCP servers + +### Phase 1: Prepare Playbook in Git Project + +**Goal**: Add playbook to a Git repository AAP can access. + +**Read [references/01-git-setup.md](references/01-git-setup.md)** for Option A (existing repo) and Option B (new repo). + +**Verification**: Playbook committed, pushed, AAP synced, playbook path noted. + +### Phase 2: Gather Required Information + +Before creating a job template, collect: + +1. **Playbook Information**: + - Playbook name/path (e.g., `remediation-CVE-2025-49794.yml`) + - Project where playbook is stored + - Required variables/parameters + +2. **Target Information**: + - Inventory containing target hosts + - Host groups or specific hosts to target + - Any host limits or filters + +3. **Credentials**: + - SSH credentials for host access + - Vault passwords (if playbook uses Ansible Vault) + - Cloud credentials (if targeting cloud resources) + +4. **Execution Settings**: + - Job type (run/check) + - Verbosity level + - Concurrent execution limits + - Timeout settings + +### Phase 3: Verify Prerequisites + +**Step 1: List Available Projects** + +**MCP Tool**: `projects_list` (from aap-mcp-job-management) + +**Parameters**: +- `page_size`: 50 (retrieve up to 50 projects) +- `search`: "remediation" (optional - filter by keyword) + +**Expected Output**: +```json +{ + "count": 1, + "results": [ + { + "id": 6, + "name": "Remediation Playbooks", + "scm_type": "git", + "scm_url": "https://github.com/org/playbooks.git", + "status": "successful" + } + ] +} +``` + +**Action**: Identify the project ID where your playbook is stored. + +**Step 2: List Available Inventories** + +**MCP Tool**: `inventories_list` (from aap-mcp-inventory-management) + +**Parameters**: +- `page_size`: 50 +- `search`: "production" (optional - filter by keyword) + +**Expected Output**: +```json +{ + "count": 1, + "results": [ + { + "id": 1, + "name": "Production Inventory", + "total_hosts": 150, + "has_active_failures": false + } + ] +} +``` + +**Action**: Identify the inventory ID containing your target hosts. + +**Step 3: Verify Credentials** + +**Note**: The current AAP MCP doesn't expose credential listing tools. You'll need credential IDs from AAP Web UI or administrator. + +### Phase 4: Create Job Template via AAP Web UI + +⚠️ **CURRENT LIMITATION**: AAP MCP has no create tools. Template creation must be done via AAP Web UI. + +**Read [references/02-web-ui-form.md](references/02-web-ui-form.md)** for form fields and steps. + +**Required**: Name, Inventory, Project, Playbook, Credentials. Enable Privilege Escalation. Prompt on Launch: Job Type (REQUIRED), Variables, Limit. + +### Phase 5: Verify Template Creation + +**MCP Tool**: `job_templates_list` (from aap-mcp-job-management) + +**Parameters**: +- `search`: "CVE-2025-49794" (search for your template) +- `page_size`: 10 + +**Expected Output**: +```json +{ + "results": [ + { + "id": 42, + "name": "Remediate CVE-2025-49794", + "playbook": "remediation-CVE-2025-49794.yml", + "project": 6, + "inventory": 1, + "status": "never updated" + } + ] +} +``` + +**Success Criteria**: +- ✓ Template appears in search results +- ✓ Playbook path matches your playbook +- ✓ Project and inventory IDs are correct +- ✓ Template status is valid + +### Phase 6: Test Template Execution (Optional) + +**MCP Tool**: `job_templates_launch_retrieve` (from aap-mcp-job-management) + +**Parameters**: +- `id`: "42" (template ID from Phase 5) + +**Expected Output**: +```json +{ + "job": 1234, + "status": "pending", + "url": "/api/controller/v2/jobs/1234/" +} +``` + +**Follow-up**: Use `playbook-executor` skill to track job execution. + +## Output and Examples + +**Read [references/03-output-template.md](references/03-output-template.md)** for report format. +**Read [references/04-examples.md](references/04-examples.md)** for CVE remediation and dynamic variable examples. + +## Dependencies + +### Required MCP Servers +- `aap-mcp-job-management` - AAP job management API access + +### Required MCP Tools (Current) +- `job_templates_list` - List existing templates (verification) +- `job_templates_retrieve` - Get template details (verification) +- `projects_list` - List available projects (prerequisite) +- `inventories_list` (from aap-mcp-inventory-management) - List inventories (prerequisite) + +### Missing MCP Tools (Needed for Full Automation) +- `job_templates_create` - Create new job templates +- `job_templates_update` - Modify existing templates +- `credentials_list` - List available credentials + +### Related Skills +- `mcp-aap-validator` - **PREREQUISITE** - Validates AAP MCP server before creation (invoke in Phase 0 if not validated in session) +- `job-template-remediation-validator` - Validates created template meets remediation requirements +- `playbook-executor` - Execute templates after creation +- `playbook-generator` - Generate remediation playbooks for templates +- `system-context` - Identify target systems for inventory selection + +### Reference Documentation +- [AAP 2.6 Job Templates Documentation](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates) +- [AAP 2.6 Creating Projects](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects) + +## Best Practices + +1. **Use descriptive template names** - Include CVE ID or purpose: "Remediate CVE-2025-49794" +2. **Enable variable prompts for flexibility** - Check "Variables" in the "Prompt on Launch" section for dynamic values +3. **Set appropriate timeouts** - CVE remediation can take time; set generous timeouts +4. **Use privilege escalation** - Most remediation requires sudo/root access +5. **Document template purpose** - Use description field to explain usage +6. **Version playbooks** - Keep playbooks in Git for change tracking +7. **Test templates first** - Use check mode or test inventory before production +8. **Set concurrent limits** - Prevent overwhelming infrastructure with simultaneous jobs +9. **Enable notifications** - Configure email/webhook alerts for job completion +10. **Regular template audits** - Review and update templates as playbooks evolve + +## Human-in-the-Loop Requirements + +This skill requires user confirmation for: + +1. **Git Operations** (adding playbook to repository): + - Display: "I'll help you add the playbook to your Git repository" + - Ask: "Proceed with Git operations (clone, commit, push)?" + - Wait for confirmation + +2. **Manual Template Creation** (AAP Web UI): + - Display: "Template creation requires using the AAP Web UI" + - Ask: "I'll provide step-by-step instructions. Ready to proceed?" + - Wait for confirmation + +3. **Test Execution** (optional verification): + - Ask: "Should I test the template by launching a job?" + - Wait for confirmation before launching + +**Never assume approval** - always wait for explicit user confirmation. diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/01-git-setup.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/01-git-setup.md new file mode 100644 index 00000000..1a367a2e --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/01-git-setup.md @@ -0,0 +1,25 @@ +# Git Setup for Playbooks + +Read when guiding user through Phase 1 (Prepare Playbook in Git). + +## Option A: Add to Existing Project + +1. Ask: repo URL, local path, or "I don't have one" +2. Clone or `cd` to repo +3. `mkdir -p playbooks/remediation`; copy playbook; `git add`; `git commit`; `git push` +4. Sync AAP project (Automation Execution → Projects → Sync) +5. Note playbook path: `playbooks/remediation/remediation-CVE-YYYY-NNNNN.yml` + +## Option B: Create New Repository + +1. `mkdir ansible-remediation-playbooks`; `git init`; `mkdir -p playbooks/remediation` +2. Copy playbook; create README, .gitignore; `git add .`; `git commit` +3. Create remote repo; `git remote add origin `; `git push -u origin main` +4. Add project in AAP Web UI (Automation Execution → Projects → Add) +5. Note playbook path + +## Verification Checklist + +- Playbook committed and pushed +- AAP project synced +- Playbook path noted for template creation diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/02-web-ui-form.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/02-web-ui-form.md new file mode 100644 index 00000000..690d63ec --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/02-web-ui-form.md @@ -0,0 +1,24 @@ +# AAP Web UI Job Template Form + +Read when guiding Phase 4 (Create Template via Web UI). AAP MCP has no create tools—use Web UI. + +## Form Fields + +**Required**: Name, Inventory, Project, Playbook, Credentials (Machine/SSH) +**Job Type**: Run (or Check for dry-run) +**Options**: Enable Privilege Escalation: Yes +**Prompt on Launch** (check): Job Type (REQUIRED), Variables, Limit + +**Extra Variables** (optional): +```yaml +target_cve: "CVE-YYYY-NNNNN" +remediation_mode: "automated" +verify_after: true +``` + +## Steps + +1. Automation Execution → Templates → Add → Job Template +2. Fill form; Save +3. Note template ID from URL or details +4. Verify via `job_templates_list(search="CVE-ID")` diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/03-output-template.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/03-output-template.md new file mode 100644 index 00000000..496d2c45 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/03-output-template.md @@ -0,0 +1,20 @@ +# Job Template Creation Output + +Read when completing template creation. + +## Report Format + +```markdown +# AAP Job Template Created + +**Name**: Remediate CVE-YYYY-NNNNN +**ID**: [template_id] +**Project**: [name] (ID: [id]) +**Playbook**: playbooks/remediation/remediation-CVE-YYYY-NNNNN.yml +**Inventory**: [name] (ID: [id]) + +## Next Steps +1. Execute via AAP Web UI or job_templates_launch_retrieve +2. Monitor via jobs_retrieve, jobs_stdout_retrieve +3. Verify via remediation-verifier skill +``` diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/04-examples.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/04-examples.md new file mode 100644 index 00000000..d19c66c6 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/04-examples.md @@ -0,0 +1,19 @@ +# Job Template Creator Examples + +Read when handling specific request types. + +## Example 1: CVE Remediation Template + +**Request**: "Create a job template for CVE-2025-49794 playbook" +- Phase 1: Git setup (see 01-git-setup.md)—add playbook, commit, push, sync AAP +- Phase 2: Gather playbook path, project, inventory +- Phase 3: projects_list, inventories_list +- Phase 4: Web UI instructions (see 02-web-ui-form.md) +- Phase 5: job_templates_list to verify + +## Example 2: Dynamic CVE Template + +**Request**: "Template with variable CVE ID" +- Enable "Prompt on Launch" → Variables +- Extra vars: cve_id, remediation_mode, verify_after +- Override at launch for different CVEs diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/mcp-aap-validator/SKILL.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/mcp-aap-validator/SKILL.md new file mode 100644 index 00000000..a1c4f708 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/mcp-aap-validator/SKILL.md @@ -0,0 +1,66 @@ +--- +name: mcp-aap-validator +description: | + Validate AAP (Ansible Automation Platform) MCP server connectivity. Use when the user asks to "validate AAP MCP", "check AAP connection", or when other skills need to verify AAP MCP availability before job management or inventory operations. +model: haiku +color: yellow +--- + +# MCP AAP Validator + +Validates connectivity to AAP MCP servers by running lightweight tool calls. + +## When to Use This Skill + +Use when validating AAP MCP before job template operations, troubleshooting connection issues, or when other skills (e.g. playbook-executor) need to verify availability. Do NOT use for creating templates—use job-template-creator. + +## Workflow + +1. **Test connectivity**: Call these tools to verify each server responds: + - `job_templates_list` (page_size: 10) from aap-mcp-job-management + - `inventories_list` (page_size: 10) from aap-mcp-inventory-management +2. **If any fails**: Provide a comprehensive message with possible root causes (see below). +3. **Report**: Output a table with validated servers and outcome (emojis). + +## Failure Message (Root Causes) + +When a tool call fails, include: + +``` +❌ AAP MCP connection failed + +**Possible root causes:** +- **Credentials**: AAP_MCP_SERVER or AAP_API_TOKEN not set or invalid +- **401 Unauthorized**: Token expired or invalid → regenerate in AAP Web UI +- **403 Forbidden**: Token lacks RBAC permissions (need Job Templates, Inventories) +- **404 Not Found**: Wrong AAP_MCP_SERVER URL (must point to MCP gateway, not main AAP UI) +- **Connection timeout**: Server unreachable, firewall, or network issue +- **SSL/TLS error**: Certificate verification problem + +**Troubleshooting:** +1. Verify env vars: AAP_MCP_SERVER, AAP_API_TOKEN (never echo values) +2. Get token: AAP Web UI → Users → [Your User] → Tokens → Create +3. Ensure AAP_MCP_SERVER points to MCP gateway endpoint +4. Restart host after config changes +``` + +## Report Format + +Always end with a table: + +| Server | Outcome | +|--------|---------| +| aap-mcp-job-management | ✅ PASSED | +| aap-mcp-inventory-management | ✅ PASSED | + +Use ✅ for success, ❌ for failure, ⚠️ for partial (e.g. one server OK, one failed). + +## Dependencies + +### Required MCP Tools +- `job_templates_list` (from aap-mcp-job-management) - Connectivity test +- `inventories_list` (from aap-mcp-inventory-management) - Connectivity test + +### Required MCP Servers +- `aap-mcp-job-management` - AAP job template and execution +- `aap-mcp-inventory-management` - AAP inventory management diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/SKILL.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/SKILL.md new file mode 100644 index 00000000..a29c9443 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/SKILL.md @@ -0,0 +1,499 @@ +--- +name: playbook-executor +description: | + **CRITICAL**: Use for Ansible playbook execution via AAP. DO NOT call AAP MCP tools directly. + + Execute remediation playbooks with job management, dry-run, and reporting. Use after playbook-generator. + + **Git Flow**: If template playbook path ≠ generated playbook, perform Git Flow (commit, push, sync) BEFORE launch. +--- + +# AAP Playbook Executor Skill + +This skill executes Ansible remediation playbooks through AAP (Ansible Automation Platform) with full job management capabilities. + +**Integration with Remediation Skill**: The `/remediation` skill orchestrates this skill as part of its Step 5 (Execute Playbook) workflow. For standalone playbook execution, you can invoke this skill directly. + +## Prerequisites + +**Required MCP Servers**: `aap-mcp-job-management`, `aap-mcp-inventory-management` ([setup guide](https://docs.redhat.com/)) + +**Required MCP Tools**: +- `job_templates_list` (from aap-mcp-job-management) - List job templates +- `job_templates_retrieve` (from aap-mcp-job-management) - Get template details +- `projects_list` (from aap-mcp-job-management) - Get project name and scm_url for Git Flow +- `job_templates_launch_retrieve` (from aap-mcp-job-management) - Launch jobs +- `jobs_retrieve` (from aap-mcp-job-management) - Get job status +- `jobs_stdout_retrieve` (from aap-mcp-job-management) - Get console output +- `jobs_job_events_list` (from aap-mcp-job-management) - Get task events +- `jobs_job_host_summaries_list` (from aap-mcp-job-management) - Get host statistics +- `inventories_list` (from aap-mcp-inventory-management) - List inventories +- `hosts_list` (from aap-mcp-inventory-management) - List inventory hosts + +**Required Environment Variables**: +- `AAP_MCP_SERVER` - Base URL for the MCP endpoint of the AAP server (must point to the AAP MCP gateway) +- `AAP_API_TOKEN` - AAP API authentication token + +### Prerequisite Validation + +**CRITICAL**: Before executing operations, execute the `/mcp-aap-validator` skill to verify AAP MCP server availability. + +**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-aap-validator/SKILL.md#validation-freshness-policy). + +**How to invoke**: Execute the `/mcp-aap-validator` skill + +**Handle validation result**: +- **If validation PASSED**: Continue with playbook execution workflow +- **If validation PARTIAL**: Warn user and ask to proceed +- **If validation FAILED**: Stop execution, provide setup instructions from validator + +**Human Notification on Failure**: +If prerequisites are not met: +- ❌ "Cannot proceed: AAP MCP servers are not available" +- 📋 "Setup required: Configure AAP_MCP_SERVER and AAP_API_TOKEN environment variables" +- ❓ "How would you like to proceed? (setup now / skip / abort)" +- ⏸️ Wait for user decision + +## When to Use This Skill + +**Use this skill directly when you need**: +- Execute a previously generated Ansible playbook via AAP +- Track the status of a running AAP job +- Monitor playbook job completion +- Run dry-run (check mode) before production execution +- Verify playbook execution succeeded + +**Use the `/remediation` skill when you need**: +- Full remediation workflow including playbook execution +- Integrated CVE analysis → playbook generation → execution → verification +- End-to-end remediation orchestration + +**How they work together**: The `/remediation` skill invokes this skill after generating a remediation playbook, managing the full workflow from analysis to verification. + +## Workflow + +**Git Flow is MANDATORY**: When the job template's playbook path differs from the generated playbook (or content must be updated), you MUST perform Git Flow (write, commit, push, sync) and receive "sync complete" from the user BEFORE launching any job. Do NOT skip this—launching without it executes the wrong playbook. + +### Phase 0: Validate AAP MCP Prerequisites + +**Action**: Execute the `/mcp-aap-validator` skill + +**Note**: Can skip if validation was performed earlier in this session and succeeded. + +**How to invoke**: Execute the `/mcp-aap-validator` skill + +**Handle validation result**: +- **If validation PASSED**: Continue to Phase 1 +- **If validation PARTIAL**: Warn user and ask to proceed +- **If validation FAILED**: Stop execution, user must set up AAP MCP servers + +### Phase 1: Job Template Selection and Playbook Preparation + +**Goal**: Identify an AAP job template suitable for executing the remediation playbook. **Git Flow is MANDATORY** before Phase 3 when the template points to a different playbook or when content must be updated. + +**Input**: Playbook content and metadata from playbook-generator (filename, CVE ID, target systems). The playbook YAML is already in context—do NOT regenerate it during Git Flow. Playbook path is derived from metadata: `playbooks/remediation/` (e.g., `playbooks/remediation/remediation-CVE-2025-49794.yml`). + +**BLOCKING**: You MUST NOT launch any job (dry-run or production) until the playbook is in the Git repo and the user has confirmed "sync complete". AAP executes from the synced project—there is no "override at launch". Launching without Git Flow executes the WRONG playbook. + +#### Step 1.1: Derive Playbook Path + +From playbook metadata (filename from playbook-generator): +- Use convention `playbooks/remediation/` +- Support both `remediation-CVE-*.yml` and `remediation-CVE-*-playbook.yml` patterns. +- Example: CVE-2026-26103 → `playbooks/remediation/remediation-CVE-2026-26103.yml` + +#### Step 1.2: List Templates and Validate Each Candidate + +**MCP Tool**: `job_templates_list` (from aap-mcp-job-management) + +**Parameters**: +- `page_size`: 50 (retrieve up to 50 templates) +- `search`: "" (search for all templates) + +**REQUIRED**: For each template in results: +1. Call `job_templates_retrieve(id)` to get full details +2. **Invoke the `/job-template-remediation-validator` skill** with the template ID to verify it meets remediation requirements (inventory, project, playbook, credentials, become_enabled) +3. Only include templates that PASS validation in the lists below + +Build two lists: +- **exact_match**: `template.playbook` equals `our_playbook_path` (normalize slashes; match if equal or basenames match) +- **compatible_other**: Passes job-template-remediation-validator but **different playbook path** (template points to e.g. `cve-remediation.yml` while we have `remediation-CVE-2026-26103.yml`) + +**Path normalization**: Normalize slashes, handle `playbooks/remediation/` prefix. Match if `template.playbook` equals `our_playbook_path` or if basenames match. **Different filenames = different path = Scenario 2.** + +#### Step 1.3: Scenario Selection (MANDATORY - Do Not Skip) + +**Scenario 1 - Same playbook path** (exact_match not empty): + +The template already points to our playbook path. The project may need the latest content. **Read [references/05-git-flow-prompts.md](references/05-git-flow-prompts.md)** for Scenario 1 prompt, options (A/B), and Git Flow steps. + +- **If A**: Execute Git Flow (see Git Flow section below). **BLOCK Phase 3** until user confirms "sync complete" or "done". +- **If B**: Wait for user confirmation. **BLOCK Phase 3** until user confirms. + +**Scenario 2 - Different playbook path** (compatible_other not empty, exact_match empty): + +**CRITICAL**: The template points to a DIFFERENT playbook than our generated playbook. You MUST NOT launch the job without Git Flow—AAP executes from synced content; there is no override at launch. **Read [references/05-git-flow-prompts.md](references/05-git-flow-prompts.md)** for Scenario 2 prompt and Git Flow steps. + +- **If yes**: Execute Git Flow. **BLOCK Phase 3** until Git Flow completes and user confirms "sync complete". +- **If no**: Fall through to Scenario 3. + +**Anti-pattern**: Do NOT say "I'll override with our playbook" and then launch—that is impossible. The playbook MUST be in the repo before launch. + +**Scenario 3 - No suitable template** (exact_match and compatible_other both empty, or user chose "no" in Scenario 2): + +Execute the `/job-template-creator` skill with instruction: +``` +"Create a job template for this remediation playbook. Playbook: [content]. Filename: [filename]. Path: [our_playbook_path]. CVE: [cve_id]. Target systems: [list]." +``` + +The job-template-creator skill guides the user through: (1) Adding playbook to Git repository, (2) Syncing AAP project, (3) Creating job template via AAP Web UI with correct path, inventory, credentials, privilege escalation. + +After `/job-template-creator` completes, retrieve the template ID (from skill output or user confirmation). Execute `/job-template-remediation-validator` to validate the newly created template. If passed, proceed to Phase 3 (Dry-Run). If failed, report issues and ask user to fix in AAP Web UI. + +**Multiple matches**: If multiple exact matches, present list and ask user to choose by number. If multiple different-path matches, prefer by project name containing "remediation" or "CVE", else first. + +**Phase 1 Checkpoint** (BLOCKING - must pass before Phase 3): +- **Git Flow required**: If Scenario 1 or 2, you MUST complete Git Flow and receive "sync complete" from the user before proceeding. Do NOT skip. +- **No override**: There is no way to "override" the playbook at launch. AAP runs whatever is in the synced project. +- **Never launch** if the playbook has not been committed, pushed, and synced + +#### Git Flow (for Scenario 1 Override and Scenario 2) - MANDATORY HITL + +**When**: Scenario 1 (same path, update content) or Scenario 2 (different path, replace playbook). **Do not skip**—execution with wrong playbook content will remediate the wrong CVE. + +**Target path**: +- Scenario 1: `our_playbook_path` (e.g. `playbooks/remediation/remediation-CVE-2026-26103.yml`) +- Scenario 2: `template.playbook` (e.g. `playbooks/remediation/cve-remediation.yml`)—we replace the template's playbook with our generated content + +**Prerequisite**: Ask user for the local path to the Git repository. Use `projects_list` for project name and `scm_url`. **Read [references/05-git-flow-prompts.md](references/05-git-flow-prompts.md)** for repo path question, HITL checkpoint text, and after-push message. + +**Steps** (execute in order; HITL at checkpoint): +1. **Write playbook to file** (FAST—do NOT regenerate): + - The playbook content is ALREADY in context from playbook-generator (or remediation skill). Use it directly. + - **⚠️ ABSOLUTE PATH REQUIRED**: The Write path MUST start with `/`. Use: `/`. Example: `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project/playbooks/remediation/cve-remediation.yml` + - **WRONG** (causes "Error writing file"): `test-aap-project/playbooks/...` or `playbooks/remediation/...` — these are relative and fail when repo is outside workspace. + - **Before Write**: Confirm path starts with `/`. If not, prepend the user's repo path. + - Do NOT invoke playbook-generator, do NOT call MCP tools, do NOT re-fetch. This should take seconds, not minutes. +2. Use Run tool: `git add ` (from repo root, e.g. `git add playbooks/remediation/cve-remediation.yml`) +3. **HITL Checkpoint** (REQUIRED): Display summary per reference file. Wait for "yes" or "proceed" +4. If confirmed: `git commit -m "Add/update remediation playbook for CVE-YYYY-NNNNN"` +5. `git push origin main` (or branch from project's scm_branch if available) + +**Note**: Git must be configured. Use Run tool for git commands. + +**Do NOT proceed to Phase 3 (Dry-Run) until user confirms sync complete.** + +### Phase 2: Git Flow (MANDATORY before Phase 3) + +**BLOCKING**: You MUST NOT proceed to Phase 3 (Dry-Run) until Git Flow is complete. + +**When**: Scenario 1 (same path, update content) or Scenario 2 (different path). See Phase 1 Step 1.3. + +**Checkpoint**: Before Phase 3, confirm: +- [ ] Playbook written to repo at target path +- [ ] Git commit and push completed (with user confirmation) +- [ ] User confirmed "sync complete" after AAP project sync + +**If any unchecked**: STOP. Do Git Flow. Do NOT launch the job. + +### Phase 3: Dry-Run Execution (Recommended) + +**Prerequisite**: Phase 2 (Git Flow) MUST be complete. User must have confirmed "sync complete". + +**Goal**: Test playbook in check mode before actual execution to simulate changes. + +**Read [references/04-dry-run-display-templates.md](references/04-dry-run-display-templates.md)** for: Playbook Preview, Dry-Run Offer, Dry-Run Results Display, Proceed prompt. + +#### Step 3.1–3.2: Display Preview and Offer Dry-Run + +Show playbook structure per reference. Offer dry-run with options: yes / no / abort. **ONLY if user confirms**, proceed. + +#### Step 3.3: Launch Dry-Run Job + +**Pre-launch check** (BLOCKING): If Scenario 1 or 2 applied, you MUST have completed Git Flow and received "sync complete" from the user. If not, STOP—do not launch. Return to Phase 2 / Git Flow. + +**MCP Tool**: `job_templates_launch_retrieve` (from aap-mcp-job-management) + +**Parameters**: `id`, `requestBody` with `job_type: "check"`, `extra_vars`, `limit` + +**Key**: `job_type: "check"` - Runs Ansible in check mode (dry-run) + +#### Step 3.4: Monitor Dry-Run Progress + +Poll `jobs_retrieve` every 2 seconds. Use `jobs_job_events_list` for live task updates. + +#### Step 3.5: Display Dry-Run Results + +**MCP Tools**: `jobs_stdout_retrieve` (id, format: "txt"), `jobs_job_host_summaries_list` (id). Use display format from reference. + +#### Step 3.6: Proceed to Actual Execution? + +Ask per reference. Wait for "yes" or "execute". + +### Phase 4: Actual Execution + +**ONLY execute if user explicitly confirms** (either after dry-run or directly if they skipped dry-run). + +#### Step 4.1: Final Confirmation + +``` +⚠️ CRITICAL: Playbook Execution Confirmation Required + +This playbook will: +- Execute on: 3 production systems +- Update packages: httpd (2.4.53-7.el9 → 2.4.57-8.el9) +- Restart services: httpd +- Estimated downtime: ~10 seconds per system +- Requires reboot: No + +Job Template: CVE Remediation Template (ID: 10) +AAP URL: https://aap.example.com/jobs/ + +❓ Execute this playbook now? + +Options: +- "yes" or "execute" - Proceed with execution +- "abort" - Cancel execution + +Please respond with your choice. +``` + +Wait for explicit "yes" or "execute" response. + +#### Step 4.2: Launch Production Job + +**Pre-launch check** (BLOCKING): Same as Phase 3—if Scenario 1 or 2 applied, Git Flow must be complete and user must have confirmed "sync complete". Do NOT launch without it. + +**MCP Tool**: `job_templates_launch_retrieve` (from aap-mcp-job-management) + +**Parameters**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run", + "extra_vars": { + "target_cve": "CVE-2025-49794", + "remediation_mode": "automated", + "verify_after": true + }, + "limit": "prod-web-01,prod-web-02,prod-web-03" + } +} +``` + +**Key Parameter**: `job_type: "run"` - Runs Ansible in execution mode (actual changes) + +**Expected Output**: +```json +{ + "job": 1235, + "status": "pending", + "url": "/api/controller/v2/jobs/1235/" +} +``` + +#### Step 4.3: Monitor Execution Progress + +**Polling Strategy**: +1. Call `jobs_retrieve(id=job_id)` every 2 seconds +2. Get task events with `jobs_job_events_list(id=job_id)` for progress updates +3. Display real-time task completion status +4. Continue until status is "successful", "failed", or "error" + +**Progress Display**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running +Elapsed: 1m 23s +AAP URL: https://aap.example.com/#/jobs/playbook/1235 + +Recent Events: +- ✓ Gathering Facts (completed - all hosts) +- ✓ Check Disk Space (completed - all hosts) +- ✓ Backup Configuration (completed - all hosts) +- ⏳ Update Package: httpd (running - prod-web-01, prod-web-02) + └─ prod-web-01: Installing httpd-2.4.57-8.el9... + └─ prod-web-02: Installing httpd-2.4.57-8.el9... +- ⏸ Restart Service: httpd (pending) +``` + +**Update every 2 seconds** until completion. + +### Phase 5: Execution Report + +**Goal**: Generate comprehensive report with job details, per-host results, and full output. + +**Read [references/01-execution-report-templates.md](references/01-execution-report-templates.md)** for JSON examples, comprehensive report template, and Success/Partial Success/Failure output templates. + +#### Step 5.1–5.4: Gather Data + +**MCP Tools** (all from aap-mcp-job-management): +- `jobs_retrieve` (id) - Job details +- `jobs_job_host_summaries_list` (id) - Per-host stats +- `jobs_job_events_list` (id) - Task timeline +- `jobs_stdout_retrieve` (id, format: "txt") - Full console output + +#### Step 5.5: Generate Report + +Format all gathered data per reference. Use Success / Partial Success / Failure template based on job status. + +#### Step 5.6: Validate Job Log for CVE Handling (MANDATORY) + +**Goal**: Confirm from the job stdout that the playbook actually addressed the target CVE(s). + +**Input**: Target CVE ID(s) from invocation (e.g. CVE-2025-49794). Job stdout from `jobs_stdout_retrieve` (already gathered in Step 5.4). + +**Parse stdout for**: +- Target CVE ID(s) in output (vars, task names, audit logs, playbook metadata) +- Package update tasks for affected packages (dnf/yum install/update, package module) +- Remediation-related task names (e.g. "Update package", "Restart service", "remediation") + +**Report** (add to execution report): +- **✓ Job log confirms CVE-XXXX-YYYY was addressed** — CVE ID or package updates found in stdout +- **⚠️ Job log did not show clear evidence of CVE handling** — No CVE ID or package updates found; recommend manual verification or `/remediation-verifier` + +**Batch**: For multiple CVEs, validate each. Report per-CVE confirmation or warning. + +### Phase 6: Error Handling + +**If job status is "failed" or "error"**, provide detailed troubleshooting. + +**Read [references/02-error-handling-guide.md](references/02-error-handling-guide.md)** for: Error categories, error report template, troubleshooting steps, relaunch parameters. + +#### Step 6.1: Parse Error Output + +**MCP Tool**: `jobs_stdout_retrieve`. Analyze output for error categories per reference. + +#### Step 6.2: Generate Error Report + +Use error report template from reference. Include per-host results, failed task details, troubleshooting steps, relaunch options. + +#### Step 6.3: Offer Relaunch + +If user chooses relaunch: **MCP Tool** `jobs_relaunch_retrieve` with `hosts: "failed"`, `job_type: "run"` per reference. + +## Reference Files + +| File | Use When | +|------|----------| +| [01-execution-report-templates.md](references/01-execution-report-templates.md) | Phase 5 reports, Success/Partial/Failure output | +| [02-error-handling-guide.md](references/02-error-handling-guide.md) | Phase 6 error reports, relaunch | +| [03-workflow-examples.md](references/03-workflow-examples.md) | Demo full workflow, failure handling, skip dry-run | +| [04-dry-run-display-templates.md](references/04-dry-run-display-templates.md) | Phase 3 preview, offer, results, proceed prompt | +| [05-git-flow-prompts.md](references/05-git-flow-prompts.md) | Scenario 1/2 prompts, Git Flow HITL, after-push | + +## Dependencies + +### Required MCP Servers +- `aap-mcp-job-management` - AAP job management and execution +- `aap-mcp-inventory-management` - AAP inventory management + +### Required MCP Tools +- `job_templates_list` (from aap-mcp-job-management) - List templates +- `job_templates_retrieve` (from aap-mcp-job-management) - Get template details +- `projects_list` (from aap-mcp-job-management) - Get project name and scm_url for Git Flow +- `job_templates_launch_retrieve` (from aap-mcp-job-management) - Launch jobs +- `jobs_retrieve` (from aap-mcp-job-management) - Get job status +- `jobs_stdout_retrieve` (from aap-mcp-job-management) - Get console output +- `jobs_job_events_list` (from aap-mcp-job-management) - Get task events +- `jobs_job_host_summaries_list` (from aap-mcp-job-management) - Get host statistics +- `inventories_list` (from aap-mcp-inventory-management) - List inventories +- `hosts_list` (from aap-mcp-inventory-management) - List hosts + +### Related Skills +- `mcp-aap-validator` - **PREREQUISITE** - Validates AAP MCP servers (invoke in Phase 0) +- `job-template-remediation-validator` - **REQUIRED** - Invoke for each candidate template in Phase 1 Step 1.2 to verify remediation requirements +- `job-template-creator` - Creates/guides AAP job template setup +- `playbook-generator` - Generates playbooks for execution +- `remediation-verifier` - Verifies success after execution + +### Reference Documentation +- [references/](references/) - Step-numbered reference files (01–05) for templates and examples +- [AAP Job Execution Guide](../../docs/ansible/aap-job-execution.md) - AAP job execution best practices +- [Playbook Integration with AAP](../../docs/ansible/playbook-integration-aap.md) - Playbook-to-AAP workflow + +## Critical: Human-in-the-Loop Requirements + +This skill executes code on production systems. **Explicit user confirmation is REQUIRED** at multiple stages. + +**Before Git commit/push** (Scenario 1 Override, Scenario 2): +1. **Display change summary**: File path, diff or file size +2. **Ask for confirmation**: "Ready to commit and push these changes? Reply 'yes' or 'proceed' to continue, or 'abort' to cancel." +3. **Wait for explicit "yes" or "proceed"**: Do not commit/push without confirmation + +**Before Dry-Run Execution** (if user chooses dry-run): +1. **Display Playbook Preview**: Show tasks and explain changes +2. **Ask for Dry-Run Confirmation**: + ``` + ❓ Run dry-run to simulate changes? + + Options: + - "yes" - Run dry-run (recommended) + - "no" - Skip to actual execution + - "abort" - Cancel + + Please respond with your choice. + ``` +3. **Wait for Explicit Response**: Do not proceed without confirmation + +**Before Actual Execution** (REQUIRED): +1. **Display Execution Summary**: Show systems, changes, downtime estimate +2. **Ask for Final Confirmation**: + ``` + ⚠️ CRITICAL: Execute playbook on production systems? + + This will make real changes to N systems. + + Options: + - "yes" or "execute" - Proceed + - "abort" - Cancel + + Please respond with your choice. + ``` +3. **Wait for Explicit "yes" or "execute"**: Do not proceed without confirmation + +**Never assume approval** - always wait for explicit user confirmation before executing playbooks. + +## Best Practices + +1. **Write path must be absolute** - When Git Flow writes the playbook to the user's repo, use `/playbooks/remediation/`. The path MUST start with `/`. Relative paths cause "Error writing file". +2. **Always validate AAP prerequisites** - Invoke mcp-aap-validator in Phase 0 +3. **Validate each template** - Invoke job-template-remediation-validator for each candidate before selection +4. **Never skip Git Flow** - If template playbook path ≠ generated playbook path (Scenario 2) or content must be updated (Scenario 1), you MUST complete Git Flow and receive "sync complete" before Phase 3. Do NOT launch without it. +5. **Recommend dry-run** - Offer check mode before production execution +6. **Filter compatible templates** - Check inventory, project, and credentials match +7. **Monitor in real-time** - Display task progress during execution +8. **Comprehensive reporting** - Include per-host stats, task timeline, full output +9. **Error categorization** - Parse errors and provide specific troubleshooting +10. **Relaunch capability** - Offer to retry failed hosts +11. **Link to AAP** - Provide direct URL to job in AAP Web UI +12. **Suggest verification** - Always recommend remediation-verifier after success +13. **Document job details** - Save job ID and template info for audit trail + +## Integration with Other Skills + +- **playbook-generator**: Generates playbooks that this skill executes +- **job-template-creator**: Creates AAP job templates when needed +- **remediation-verifier**: Verifies success after this skill completes execution +- **`/remediation` skill**: Orchestrates full workflow including playbook execution + +**Orchestration Example** (from `/remediation` skill): +1. Agent invokes playbook-generator skill → Creates playbook YAML +2. playbook-generator asks for confirmation → User approves playbook content +3. Agent invokes playbook-executor skill (this skill) → Execution workflow +4. Skill validates templates via job-template-remediation-validator → Filters valid candidates +5. Skill checks path match → If different path, offers Git Flow (HITL: commit/push, sync AAP) +6. Skill waits for "sync complete" before proceeding (if Git Flow was used) +7. Skill offers dry-run → User runs check mode +8. Skill asks for execution confirmation → User approves +9. Skill executes and monitors → Reports completion +10. Agent invokes remediation-verifier skill → Confirms CVE resolved + +**Note**: Both playbook-generator and playbook-executor require separate confirmations for different purposes: +- playbook-generator: Confirms playbook content is acceptable +- playbook-executor: Confirms execution on production systems is approved + +This two-step approval ensures user control over both what to run and when to run it. diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/01-execution-report-templates.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/01-execution-report-templates.md new file mode 100644 index 00000000..a6773c5f --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/01-execution-report-templates.md @@ -0,0 +1,168 @@ +# Step 01: Execution Report Templates + +Read this reference when generating Phase 5 execution reports or output templates. + +## Phase 5: Job Details (JSON Examples) + +### jobs_retrieve Expected Output + +```json +{ + "id": 1235, + "name": "CVE Remediation Template", + "status": "successful", + "started": "2026-02-24T15:35:02Z", + "finished": "2026-02-24T15:40:25Z", + "elapsed": 323.45, + "job_template": 10, + "inventory": 1, + "limit": "prod-web-01,prod-web-02,prod-web-03", + "playbook": "playbooks/remediation/remediation-CVE-2025-49794.yml" +} +``` + +### jobs_job_host_summaries_list Expected Output + +```json +{ + "results": [ + { + "host_name": "prod-web-01", + "ok": 8, + "changed": 3, + "failed": 0, + "unreachable": 0 + }, + { + "host_name": "prod-web-02", + "ok": 8, + "changed": 3, + "failed": 0, + "unreachable": 0 + }, + { + "host_name": "prod-web-03", + "ok": 5, + "changed": 0, + "failed": 1, + "unreachable": 0 + } + ] +} +``` + +## Comprehensive Report Template + +```markdown +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 5m 23s +**Started**: 2026-02-24 15:35:02 UTC +**Completed**: 2026-02-24 15:40:25 UTC +**Job Template**: CVE Remediation Template +**Playbook**: playbooks/remediation/remediation-CVE-2025-49794.yml +**AAP URL**: [View in AAP](https://aap.example.com/#/jobs/playbook/1235) + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| prod-web-01 | 8 | 3 | 0 | 0 | ✅ Success | +| prod-web-02 | 8 | 3 | 0 | 0 | ✅ Success | +| prod-web-03 | 8 | 3 | 0 | 0 | ✅ Success | + +**Summary**: 3 of 3 hosts successfully remediated + +## Task Timeline +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) + - prod-web-01: 2.4.53-7.el9 → 2.4.57-8.el9 + - prod-web-02: 2.4.53-7.el9 → 2.4.57-8.el9 + - prod-web-03: 2.4.53-7.el9 → 2.4.57-8.el9 +5. ✅ Restart httpd service (15s) +6. ✅ Verify service status (2s) +7. ✅ Update audit log (1s) + +## Full Console Output +
+Click to expand (187 lines) + +[Full stdout from jobs_stdout_retrieve] + +
+ +## Job Log CVE Validation (Step 5.6) +✓ Job log confirms CVE-XXXX-YYYY was addressed + +*(Or: ⚠️ Job log did not show clear evidence of CVE handling—verify manually or use remediation-verifier)* + +## Next Steps +1. ✅ All systems successfully remediated +2. ☐ Verify remediation with remediation-verifier skill +3. ☐ Update vulnerability tracking system +4. ☐ Schedule follow-up verification in 24-48 hours + +--- + +**Recommendation**: Run remediation-verifier skill to confirm CVE status has been updated in Red Hat Lightspeed. +``` + +## Output Templates + +### Success Template + +```markdown +✅ Playbook Execution Successful + +Job ID: 1235 +Duration: 5m 23s +Systems Remediated: 3 of 3 + +View full report above for details. + +Next Steps: +- Run remediation-verifier skill to confirm CVE resolution +- Update vulnerability tracking system +- Monitor systems for 24-48 hours + +AAP URL: https://aap.example.com/#/jobs/playbook/1235 +``` + +### Partial Success Template + +```markdown +⚠️ Playbook Execution Completed with Failures + +Job ID: 1235 +Duration: 2m 45s +Systems Remediated: 2 of 3 +Failed Systems: prod-web-03 + +See error details above for troubleshooting steps. + +Options: +- Relaunch for failed hosts +- Manual remediation +- Skip failed hosts + +AAP URL: https://aap.example.com/#/jobs/playbook/1235 +``` + +### Failure Template + +```markdown +❌ Playbook Execution Failed + +Job ID: 1235 +Duration: 1m 15s +Systems Remediated: 0 of 3 + +Critical errors prevented execution. +See error details above for troubleshooting. + +AAP URL: https://aap.example.com/#/jobs/playbook/1235 +``` diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/02-error-handling-guide.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/02-error-handling-guide.md new file mode 100644 index 00000000..90492f00 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/02-error-handling-guide.md @@ -0,0 +1,108 @@ +# Step 02: Error Handling Guide + +Read this reference when generating Phase 6 error reports or troubleshooting. + +## Error Categories + +**Parse error output** from `jobs_stdout_retrieve` for these common patterns: + +1. **Connection Failures**: SSH timeout, host unreachable, authentication failed +2. **Permission Errors**: sudo required, insufficient privileges, SELinux denials +3. **Package Manager Issues**: repo unavailable, package not found, dependency conflicts +4. **Service Failures**: service not found, restart failed, timeout +5. **Disk Space**: insufficient space for updates +6. **General Failures**: playbook syntax errors, task failures + +## Error Report Template + +```markdown +# Playbook Execution Failed + +## Job Summary +**Job ID**: 1235 +**Status**: ❌ Failed +**Duration**: 2m 45s +**Started**: 2026-02-24 15:35:02 UTC +**Failed At**: 2026-02-24 15:37:47 UTC +**Job Template**: CVE Remediation Template +**AAP URL**: [View in AAP](https://aap.example.com/#/jobs/playbook/1235) + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| prod-web-01 | 8 | 3 | 0 | 0 | ✅ Success | +| prod-web-02 | 8 | 3 | 0 | 0 | ✅ Success | +| prod-web-03 | 5 | 0 | 1 | 0 | ❌ Failed | + +**Summary**: 2 of 3 hosts succeeded, 1 failed + +## Failed Tasks Details + +### Host: prod-web-03 + +**Task**: Restart httpd service +**Error**: "Failed to restart httpd.service: Unit httpd.service not found." + +**Error Category**: Service Failure + +**Root Cause**: The httpd service is not installed or not recognized by systemd. + +**Troubleshooting Steps**: +1. Check if httpd is installed: + ```bash + ssh prod-web-03 'rpm -q httpd' + ``` +2. If not installed, the package update may have failed: + ```bash + ssh prod-web-03 'dnf info httpd' + ``` +3. Check systemd service status: + ```bash + ssh prod-web-03 'systemctl status httpd' + ``` +4. Review package manager logs: + ```bash + ssh prod-web-03 'tail -50 /var/log/dnf.log' + ``` + +**Recommended Action**: +- Verify httpd package installation on prod-web-03 +- Check if package update completed successfully +- Manually install httpd if needed: `dnf install httpd` +- Relaunch job for failed host only + +## Console Output (Last 50 Lines) +
+Click to expand error context + +[Relevant error output from jobs_stdout_retrieve] + +
+ +## Relaunch Options + +Would you like to: +1. **Relaunch for failed hosts only** - Run job again with limit="prod-web-03" +2. **Fix issues manually and relaunch** - Resolve problems first, then relaunch +3. **View full job output** - See complete execution logs +4. **Abort** - Stop remediation workflow + +Please choose an option (1-4): +``` + +## Relaunch Parameters + +**MCP Tool**: `jobs_relaunch_retrieve` (from aap-mcp-job-management) + +**Parameters**: +```json +{ + "id": "1235", + "requestBody": { + "hosts": "failed", + "job_type": "run" + } +} +``` + +This relaunches the job for only the failed hosts. diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/03-workflow-examples.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/03-workflow-examples.md new file mode 100644 index 00000000..f5caa0d1 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/03-workflow-examples.md @@ -0,0 +1,119 @@ +# Step 03: Workflow Examples + +Read this reference when demonstrating end-to-end workflow patterns. + +## Example 1: Full Workflow with Dry-Run + +**User Request**: "Execute the CVE-2025-49794 remediation playbook" + +**Skill Response**: + +1. **Validate AAP Prerequisites**: + - Invoke mcp-aap-validator skill → PASSED + +2. **List Job Templates**: + - Call `job_templates_list()` → Found 2 templates + - Filter compatible templates → 1 matches requirements + +3. **User Selects Template**: + ``` + Found 1 compatible job template: + 1. "CVE Remediation Template" (ID: 10) + + Select template (1) or "create" for new: 1 + ``` + +4. **Playbook Preparation**: + ``` + Guide user to add playbook to Git: + - Commands provided + - User syncs AAP project + - Verification: Playbook available ✓ + ``` + +5. **Offer Dry-Run**: + ``` + Run dry-run first? yes + ``` + +6. **Execute Dry-Run**: + - Launch with `job_type="check"` + - Monitor progress → COMPLETED + - Display dry-run results: + ``` + Would change 3 tasks on 3 hosts + No errors detected + ``` + +7. **Proceed to Actual Execution**: + ``` + Proceed with actual execution? yes + ``` + +8. **Execute Playbook**: + - Launch with `job_type="run"` + - Monitor progress → COMPLETED + - Generate report: + ``` + ✅ 3 of 3 hosts successfully remediated + Duration: 5m 23s + ``` + +9. **Suggest Verification**: + ``` + Recommend: Run remediation-verifier skill to confirm CVE resolution + ``` + +## Example 2: Handle Execution Failure + +**User Request**: "Execute remediation playbook" + +**Skill Response**: + +1-7. [Same as Example 1 through execution] + +8. **Execute Playbook**: + - Launch job → FAILED on 1 of 3 hosts + - Generate error report: + ``` + ❌ prod-web-03 failed + Error: httpd service not found + Troubleshooting steps provided + ``` + +9. **Offer Relaunch**: + ``` + Relaunch for failed host only? yes + ``` + +10. **Relaunch Job**: + - Call `jobs_relaunch_retrieve` with hosts="failed" + - Monitor → COMPLETED + - Final report: + ``` + ✅ All 3 hosts successfully remediated (1 after retry) + ``` + +## Example 3: Skip Dry-Run + +**User Request**: "Execute playbook directly, skip dry-run" + +**Skill Response**: + +1-4. [Same as Example 1 through template selection] + +5. **Offer Dry-Run**: + ``` + Run dry-run first? no + ``` + +6. **Final Confirmation**: + ``` + ⚠️ Execute on production without dry-run? + This will make changes immediately. + Confirm: yes + ``` + +7. **Execute Playbook**: + - Launch with `job_type="run"` + - Monitor and report as in Example 1 diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/04-dry-run-display-templates.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/04-dry-run-display-templates.md new file mode 100644 index 00000000..d9b6f0dc --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/04-dry-run-display-templates.md @@ -0,0 +1,93 @@ +# Step 04: Dry-Run Display Templates + +Read this reference when displaying Phase 3 dry-run content. + +## Playbook Preview + +```markdown +# Playbook Preview + +**Playbook**: remediation-CVE-2025-49794.yml +**Target Systems**: 5 systems + +## Tasks Overview: +1. **Gather Facts** - Collect system information +2. **Check Disk Space** - Ensure sufficient space for updates (>500MB) +3. **Backup Configuration** - Snapshot critical configs +4. **Update Package: httpd** - Upgrade to version 2.4.57-8.el9 +5. **Restart Service: httpd** - Apply changes +6. **Verify Service Status** - Confirm httpd is running +7. **Update Audit Log** - Record remediation event + +**Estimated Duration**: 3-5 minutes per system +**Requires Reboot**: No +**Downtime**: Brief (~10 seconds during service restart) +``` + +## Dry-Run Offer + +``` +⚠️ Recommended: Run dry-run first + +Dry-run mode (--check) simulates changes without applying them. +This helps identify: +- Package availability issues +- Permission problems +- Configuration conflicts +- Unexpected side effects + +❓ Run dry-run before actual execution? +- "yes" - Run dry-run first (recommended) +- "no" - Skip to actual execution +- "abort" - Cancel execution + +Please respond with your choice. +``` + +## Dry-Run Results Display + +```markdown +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s +**Completed**: 2024-01-20 15:32:17 UTC + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| prod-web-01 | 3 | 8 | 0 | ✓ Ready | +| prod-web-02 | 3 | 8 | 0 | ✓ Ready | +| prod-web-03 | 3 | 8 | 0 | ✓ Ready | + +## Changes That Would Be Made: +1. **httpd package** - Would update from 2.4.53-7.el9 to 2.4.57-8.el9 +2. **httpd service** - Would restart +3. **audit log** - Would add remediation entry + +## Dry-Run Output: +
+Click to expand full output + +[Full stdout from jobs_stdout_retrieve] + +
+ +✓ No errors detected in dry-run +✓ All systems passed pre-flight checks +``` + +## Proceed to Actual Execution Prompt + +``` +❓ Dry-run completed successfully. Proceed with actual execution? + +Options: +- "yes" or "execute" - Proceed with actual remediation +- "review" - Show dry-run output again +- "abort" - Cancel execution + +Please respond with your choice. +``` diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/05-git-flow-prompts.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/05-git-flow-prompts.md new file mode 100644 index 00000000..41945d0e --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/05-git-flow-prompts.md @@ -0,0 +1,97 @@ +# Step 05: Git Flow Prompts + +Read this reference when executing Git Flow (Scenario 1 Override or Scenario 2). + +## Scenario 1 Prompt (Same path) + +The template already points to our playbook path. The project may need the latest content. + +``` +Found template [name] (ID: X) with matching playbook path. The project may need to be updated with the latest playbook. + +Options: +(A) Override: I'll add/update the playbook in the project via git. You sync the AAP project, then confirm. +(B) Manual: You add the playbook and sync. Confirm when done. + +❓ Choose (A) or (B): +``` + +- **If A**: Execute Git Flow (see Git Flow section below). Wait for user: "Sync complete" or "done". +- **If B**: Wait for user confirmation. + +## Scenario 2 Prompt (Different path) + +**CRITICAL**: The template points to a DIFFERENT playbook than our generated playbook. You MUST NOT launch the job without Git Flow. AAP executes from synced project content—there is no "override at launch". The playbook MUST be in the repo before any job launch. + +``` +Found template [name] (ID: X) pointing to [template.playbook]. Our generated playbook is [our_playbook_path]. + +⚠️ The template's playbook path does NOT match. We must update the playbook in the project before execution. + +Options: +- "yes" or "proceed" - I'll add our playbook to the project via git (you'll confirm commit/push, then sync AAP) +- "no" - Create a new template via `/job-template-creator` skill + +❓ Proceed with playbook update (git flow)? +``` + +- **If yes**: Execute Git Flow. **Do NOT proceed to Phase 3 until Git Flow completes.** +- **If no**: Fall through to Scenario 3 (job-template-creator). + +## Repo Path Question + +``` +What is the local path to the Git repository for project [Project Name] (scm_url)? +``` + +Use `projects_list` to get project name and `scm_url`; display to help user identify the repo. + +**Path format**: Ask for the **absolute path** (e.g. `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project`). When writing the playbook, the Write tool path MUST be `/playbooks/remediation/` — the full absolute path. Do NOT use a relative path like `test-aap-project/playbooks/...`; that causes "Error writing file". + +## Git Flow: Write Step (FAST) + +**CRITICAL**: The playbook is ALREADY generated. During Git Flow you must WRITE the existing content to disk—nothing more. + +- **DO**: Single file write of the playbook content already in context (from playbook-generator or remediation) +- **DO NOT**: Invoke playbook-generator again, call create_vulnerability_playbook, re-fetch from MCP, or validate/transform the content +- **Expected duration**: Seconds. If it takes minutes, you are doing unnecessary work. + +### Write Path (ABSOLUTE REQUIRED) + +**⚠️ WRITE PATH MUST START WITH `/`** — The Write tool path MUST be an absolute path. Relative paths cause "Error writing file" because the repo is often outside the workspace. + +**Formula**: `write_path = user_provided_path + "/" + target_path` + +- `user_provided_path` = exactly what the user typed (e.g. `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project`) +- `target_path` = e.g. `playbooks/remediation/cve-remediation.yml` + +**Correct**: `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project/playbooks/remediation/cve-remediation.yml` + +**WRONG** (will fail): +- `test-aap-project/playbooks/remediation/cve-remediation.yml` +- `playbooks/remediation/cve-remediation.yml` + +**Before calling Write**: Verify the path starts with `/`. If it does not, prepend the user's repo path. + +## Git Flow HITL Checkpoint + +**REQUIRED** before commit/push: + +``` +Ready to commit and push these changes? +- File: [target_path] +- CVE: [cve_id] +- This will update the playbook in the AAP project. + +Reply 'yes' or 'proceed' to continue, or 'abort' to cancel. +``` + +**Wait for user confirmation.** If "yes" or "proceed": `git commit -m "Add/update remediation playbook for CVE-YYYY-NNNNN"` then `git push origin main`. + +## After Push Message + +``` +I've pushed the playbook. Sync the AAP project: Automation Execution > Projects > [Project] > Sync. Reply 'sync complete' when done. +``` + +**Do NOT proceed to Phase 3 (Dry-Run) until user confirms sync complete.** diff --git a/evaluation/with_skills/rh-sre__job-template-creator/instruction.md b/evaluation/with_skills/rh-sre__job-template-creator/instruction.md new file mode 100644 index 00000000..77c24f2b --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/instruction.md @@ -0,0 +1,17 @@ +# Job Template Creator Task + +You are a Red Hat SRE. A remediation playbook has been written for a critical CVE, and you need to set up an Ansible Automation Platform job template so the team can run it against affected systems. + +## Scenario +The security team delivered a remediation playbook for CVE-2026-1234. You need to create a job template in AAP that the operations team can use to run this playbook against production hosts. + +## Requirements +- Check which projects and inventories are available in AAP +- Determine the correct project, inventory, and credentials for the remediation playbook +- Document the job template configuration: name, playbook path, inventory, project, credentials, and execution settings (privilege escalation, variable prompts, limit prompts) +- Explain any decisions about template settings (e.g., why `become` is enabled, whether to prompt for variables at launch) +- If template creation requires manual steps (e.g., via the AAP Web UI), document those steps clearly + +Document your methodology, plan, and configuration in `/root/report.md`. + +Use MCP tools to query AAP. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-sre__job-template-creator/solution/solve.sh b/evaluation/with_skills/rh-sre__job-template-creator/solution/solve.sh new file mode 100644 index 00000000..ec9c5b02 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/solution/solve.sh @@ -0,0 +1,19 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Job Template Creation + +## Template Fields +- Inventory: production-systems +- Project: remediation-playbooks +- Playbook: playbooks/remediation/cve-2024-12345.yml +- Credentials: machine-credential +- become_enabled: true + +## Prompt on Launch +- Job Type (REQUIRED for dry-run + run) +- Variables +- Limit + +## Note +No job_templates_create API in AAP MCP. Create via Web UI. Execute mcp-aap-validator before operations. +REPORT_EOF diff --git a/evaluation/with_skills/rh-sre__job-template-creator/task.toml b/evaluation/with_skills/rh-sre__job-template-creator/task.toml new file mode 100644 index 00000000..bc2620fa --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__job-template-creator" +name = "rh-sre AAP Job Template Creation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "job-template-creator", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-sre__job-template-creator/tests/llm_judge.py b/evaluation/with_skills/rh-sre__job-template-creator/tests/llm_judge.py new file mode 100644 index 00000000..54c93ce1 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "no_create_tool", "file": "/root/report.md", "question": "Does the report acknowledge that AAP MCP has no create/update tools and template creation must be done via Web UI?", "reference": "A skilled report notes the MCP limitation and directs to Web UI. An unskilled report attempts to create templates via API."}, + {"id": "playbook_path_and_git", "file": "/root/report.md", "question": "Does the report require the playbook to be in a Git repo with proper path convention before template creation?", "reference": "A skilled report follows playbooks/remediation/ path convention. An unskilled report skips Git integration."}, + {"id": "launch_configuration", "file": "/root/report.md", "question": "Does the report configure prompt-on-launch for job type and privilege escalation?", "reference": "A skilled report enables prompt-on-launch and become_enabled. An unskilled report skips these configuration details."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-sre__job-template-creator/tests/test.sh b/evaluation/with_skills/rh-sre__job-template-creator/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-sre__job-template-creator/tests/test_outputs.py b/evaluation/with_skills/rh-sre__job-template-creator/tests/test_outputs.py new file mode 100644 index 00000000..53140085 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-creator/tests/test_outputs.py @@ -0,0 +1,98 @@ +""" +Tests for rh-sre__job-template-creator per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['job template', 'template', 'ansible']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_git_before_template(self): + """Skill: Playbook must be in Git repo before template creation; AAP syncs from project.""" + c = read_report().lower() + has_git = any(t in c for t in ["git", "commit", "push", "repository", "sync"]) + has_project = any(t in c for t in ["project", "scm", "sync"]) + assert has_git or has_project, ( + "should add playbook to Git before template (skill: Phase 1)" + ) + + def test_manual_creation_required(self): + """Skill teaches template creation requires manual steps (e.g., Web UI) + because the automation API is read-only for templates.""" + c = read_report().lower() + assert any(t in c for t in [ + "web ui", "manual", "read-only", "cannot create", + "no create", "gui", "interface", + ]), "should acknowledge template creation requires manual steps" + + def test_playbook_path_convention(self): + """Skill teaches following a consistent directory structure or location + convention for remediation playbooks.""" + c = read_report().lower() + assert any(t in c for t in [ + "playbook path", "remediation playbook", "playbook location", + "playbook directory", "playbook structure", + ]), "should follow a playbook path convention for remediation" + + def test_privilege_escalation_required(self): + """Skill: become_enabled required for remediation (package updates).""" + c = read_report().lower() + assert any(t in c for t in ["privilege", "become", "sudo", "escalat", "root"]), ( + "should require privilege escalation (skill: required for package updates)" + ) + + def test_launch_prompts(self): + """Skill: Prompt on Launch for Job Type, Variables, Limit.""" + c = read_report().lower() + assert any(t in c for t in ["launch", "prompt", "variable", "limit", "job type"]), ( + "should configure prompt on launch (skill: Phase 4)" + ) + + def test_configurable_variables(self): + """Docs teach configuring variables for CVE targeting, remediation mode, + and post-remediation verification. Without docs, agents skip variable design.""" + c = read_report().lower() + concepts = sum(1 for t in [ + "target_cve", "cve", "remediation_mode", "mode", + "verify_after", "verification", "extra_var", "extra var", + "variable", "parameter", + ] if t in c) + assert concepts >= 3, ( + "should define configurable variables for CVE targeting, " + "remediation mode, and verification" + ) + + def test_version_control_sync(self): + """Skill teaches AAP projects sync playbooks from version control. + Without skill, agents describe playbook management without + version-control-backed project sync.""" + c = read_report().lower() + assert any(t in c for t in [ + "scm", "source control", "version control", + "repository sync", "git-backed", "git sync", + ]), "should reference version control sync for AAP project playbooks" diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/Dockerfile b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/Dockerfile new file mode 100644 index 00000000..d5c9e7b7 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/Dockerfile @@ -0,0 +1,56 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + + + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + }, \ + "aap-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-aap-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..1092dd1d --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- pass rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/mcp-servers/mock-aap-mcp.py b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/mcp-servers/mock-aap-mcp.py new file mode 100644 index 00000000..d8ae4fd5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/mcp-servers/mock-aap-mcp.py @@ -0,0 +1,1048 @@ +#!/usr/bin/env python3 +""" +Mock AAP (Ansible Automation Platform) MCP Server + +Simulates the AAP MCP gateway for per-skill evaluation tasks. Implements +the full set of tools used by rh-sre skills: + - job_templates_list / job_templates_retrieve + - projects_list + - job_templates_launch_retrieve + - jobs_retrieve / jobs_stdout_retrieve + - jobs_job_events_list / jobs_job_host_summaries_list + - jobs_relaunch_retrieve + - inventories_list / hosts_list + +Data mirrors a realistic AAP deployment: + - 6 job templates (3 remediation, 1 compliance, 1 patching, 1 reporting) + - 3 projects (remediation, compliance, reporting) + - 3 inventories (production 30 hosts, staging 15 hosts, all-managed 63 hosts) + - 12 recent jobs with varied statuses + +Follows the same mock-server pattern as mock-lightspeed-mcp.py. +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +mcp = FastMCP("aap-mcp") + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +def _ts(delta: timedelta) -> str: + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +# --------------------------------------------------------------------------- +# Mock data: Projects +# --------------------------------------------------------------------------- + +MOCK_PROJECTS = [ + { + "id": 6, + "type": "project", + "name": "Remediation Playbooks", + "description": "CVE and security remediation playbooks managed via Git", + "scm_type": "git", + "scm_url": "https://github.com/org/remediation-playbooks.git", + "scm_branch": "main", + "scm_revision": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + "status": "successful", + "last_job_run": _ts(timedelta(hours=2)), + "last_update_failed": False, + "created": _ts(timedelta(days=90)), + "modified": _ts(timedelta(hours=2)), + }, + { + "id": 7, + "type": "project", + "name": "Compliance Checks", + "description": "STIG and CIS compliance scanning playbooks", + "scm_type": "git", + "scm_url": "https://github.com/org/compliance-playbooks.git", + "scm_branch": "main", + "scm_revision": "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3", + "status": "successful", + "last_job_run": _ts(timedelta(days=1)), + "last_update_failed": False, + "created": _ts(timedelta(days=120)), + "modified": _ts(timedelta(days=1)), + }, + { + "id": 8, + "type": "project", + "name": "Fleet Reporting", + "description": "System inventory and health reporting playbooks", + "scm_type": "git", + "scm_url": "https://github.com/org/fleet-reports.git", + "scm_branch": "main", + "scm_revision": "c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "status": "successful", + "last_job_run": _ts(timedelta(days=3)), + "last_update_failed": False, + "created": _ts(timedelta(days=180)), + "modified": _ts(timedelta(days=3)), + }, +] + +# --------------------------------------------------------------------------- +# Mock data: Inventories & Hosts +# --------------------------------------------------------------------------- + +MOCK_INVENTORIES = [ + { + "id": 1, + "type": "inventory", + "name": "Production Systems", + "description": "All production RHEL systems across data centers", + "total_hosts": 30, + "has_active_failures": False, + "hosts_with_active_failures": 0, + "total_groups": 5, + "groups_with_active_failures": 0, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=365)), + "modified": _ts(timedelta(days=1)), + }, + { + "id": 2, + "type": "inventory", + "name": "Staging Systems", + "description": "Pre-production staging environment", + "total_hosts": 15, + "has_active_failures": False, + "hosts_with_active_failures": 0, + "total_groups": 3, + "groups_with_active_failures": 0, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=300)), + "modified": _ts(timedelta(days=7)), + }, + { + "id": 3, + "type": "inventory", + "name": "All Managed Systems", + "description": "Complete fleet: production, staging, development, QA, legacy", + "total_hosts": 63, + "has_active_failures": True, + "hosts_with_active_failures": 2, + "total_groups": 8, + "groups_with_active_failures": 1, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=365)), + "modified": _ts(timedelta(hours=6)), + }, +] + + +def _generate_hosts(inventory_id: int) -> list[dict]: + """Generate realistic hosts for an inventory.""" + hosts: list[dict] = [] + if inventory_id == 1: + roles = ["web", "db", "app", "lb", "monitoring", "cache"] + for i, role in enumerate(roles): + for j in range(5 if role in ("web", "app") else 4 if role == "db" else 3 if role == "monitoring" else 2): + hosts.append({ + "id": len(hosts) + 1, + "type": "host", + "name": f"{role}-{j+1:02d}.prod.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "production", "role": "{role}"}}', + }) + if len(hosts) >= 30: + break + if len(hosts) >= 30: + break + elif inventory_id == 2: + for i in range(15): + role = ["web", "db", "app"][i % 3] + hosts.append({ + "id": 100 + i, + "type": "host", + "name": f"{role}-{i+1:02d}.staging.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "staging", "role": "{role}"}}', + }) + elif inventory_id == 3: + for i in range(30): + hosts.append({ + "id": 200 + i, + "type": "host", + "name": f"host-{i+1:02d}.prod.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": i in (45, 58), + "variables": f'{{"rhel_version": "9.3", "environment": "production"}}', + }) + for i in range(15): + hosts.append({ + "id": 230 + i, + "type": "host", + "name": f"host-{i+1:02d}.staging.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "staging"}}', + }) + for i in range(10): + hosts.append({ + "id": 245 + i, + "type": "host", + "name": f"dev-{i+1:02d}.dev.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "8.9", "environment": "development"}}', + }) + for i in range(5): + hosts.append({ + "id": 255 + i, + "type": "host", + "name": f"qa-{i+1:02d}.qa.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.2", "environment": "qa"}}', + }) + for i in range(3): + hosts.append({ + "id": 260 + i, + "type": "host", + "name": f"legacy-{i+1:02d}.corp.example.com", + "inventory": inventory_id, + "enabled": i < 2, + "has_active_failures": i == 2, + "variables": f'{{"rhel_version": "7.9", "environment": "legacy"}}', + }) + return hosts + + +# --------------------------------------------------------------------------- +# Mock data: Job Templates +# --------------------------------------------------------------------------- + +MOCK_JOB_TEMPLATES = [ + { + "id": 10, + "type": "job_template", + "name": "CVE Remediation - Kernel Update", + "description": "Kernel update with boom snapshot for rollback safety", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": True, + "job_type": "check", + "verbosity": 1, + "timeout": 3600, + "forks": 5, + "status": "successful", + "last_job_run": _ts(timedelta(hours=4)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1001, "status": "successful", "finished": _ts(timedelta(hours=4))}, + }, + "created": _ts(timedelta(days=60)), + "modified": _ts(timedelta(days=2)), + }, + { + "id": 11, + "type": "job_template", + "name": "CVE Remediation - Package Update", + "description": "General package update for CVE remediation with needs-restarting check", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-package-update.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": False, + "job_type": "check", + "verbosity": 1, + "timeout": 1800, + "forks": 10, + "status": "successful", + "last_job_run": _ts(timedelta(hours=12)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1005, "status": "successful", "finished": _ts(timedelta(hours=12))}, + }, + "created": _ts(timedelta(days=45)), + "modified": _ts(timedelta(days=5)), + }, + { + "id": 12, + "type": "job_template", + "name": "CVE Remediation - Generic", + "description": "Generic CVE remediation template for ad-hoc patches", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-remediation.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": True, + "job_type": "check", + "verbosity": 1, + "timeout": 3600, + "forks": 5, + "status": "never updated", + "last_job_run": None, + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + }, + "created": _ts(timedelta(days=30)), + "modified": _ts(timedelta(days=30)), + }, + { + "id": 20, + "type": "job_template", + "name": "Compliance Check - STIG", + "description": "Run STIG compliance scan across fleet", + "inventory": 3, + "project": 7, + "playbook": "playbooks/compliance/check-all.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": False, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 7200, + "forks": 20, + "status": "successful", + "last_job_run": _ts(timedelta(days=1)), + "summary_fields": { + "project": {"id": 7, "name": "Compliance Checks", "status": "successful"}, + "inventory": {"id": 3, "name": "All Managed Systems", "total_hosts": 63}, + "credentials": [ + {"id": 2, "name": "compliance-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1010, "status": "successful", "finished": _ts(timedelta(days=1))}, + }, + "created": _ts(timedelta(days=180)), + "modified": _ts(timedelta(days=14)), + }, + { + "id": 25, + "type": "job_template", + "name": "Emergency Patching", + "description": "Emergency patch application — NO become enabled (misconfigured)", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/emergency-patch.yml", + "become_enabled": False, + "ask_job_type_on_launch": False, + "ask_variables_on_launch": False, + "ask_limit_on_launch": False, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 600, + "forks": 25, + "status": "failed", + "last_job_run": _ts(timedelta(days=7)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1020, "status": "failed", "finished": _ts(timedelta(days=7))}, + }, + "created": _ts(timedelta(days=200)), + "modified": _ts(timedelta(days=200)), + }, + { + "id": 30, + "type": "job_template", + "name": "Fleet Health Report", + "description": "Generate fleet health and inventory report", + "inventory": 3, + "project": 8, + "playbook": "playbooks/reporting/fleet-health.yml", + "become_enabled": False, + "ask_job_type_on_launch": False, + "ask_variables_on_launch": True, + "ask_limit_on_launch": False, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 1800, + "forks": 30, + "status": "successful", + "last_job_run": _ts(timedelta(hours=6)), + "summary_fields": { + "project": {"id": 8, "name": "Fleet Reporting", "status": "successful"}, + "inventory": {"id": 3, "name": "All Managed Systems", "total_hosts": 63}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1025, "status": "successful", "finished": _ts(timedelta(hours=6))}, + }, + "created": _ts(timedelta(days=120)), + "modified": _ts(timedelta(days=14)), + }, +] + +# --------------------------------------------------------------------------- +# Mock data: Jobs (recent runs) +# --------------------------------------------------------------------------- + +PROD_HOSTS = [ + "web-01.prod.example.com", + "web-02.prod.example.com", + "db-01.prod.example.com", + "db-02.prod.example.com", + "app-01.prod.example.com", + "app-02.prod.example.com", +] + +MOCK_JOBS = [ + { + "id": 1001, + "type": "job", + "name": "CVE Remediation - Kernel Update", + "job_type": "check", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=4, minutes=30)), + "finished": _ts(timedelta(hours=4)), + "elapsed": 1800.0, + "job_template": 10, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "limit": "web-01.prod.example.com,web-02.prod.example.com,db-01.prod.example.com", + "extra_vars": '{"target_cve": "CVE-2024-12345", "remediation_mode": "automated", "verify_after": true}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 10, "name": "CVE Remediation - Kernel Update"}, + }, + }, + { + "id": 1002, + "type": "job", + "name": "CVE Remediation - Kernel Update", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=3, minutes=45)), + "finished": _ts(timedelta(hours=3)), + "elapsed": 2700.0, + "job_template": 10, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "limit": "web-01.prod.example.com,web-02.prod.example.com,db-01.prod.example.com", + "extra_vars": '{"target_cve": "CVE-2024-12345", "remediation_mode": "automated", "verify_after": true}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 10, "name": "CVE Remediation - Kernel Update"}, + }, + }, + { + "id": 1005, + "type": "job", + "name": "CVE Remediation - Package Update", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=12, minutes=20)), + "finished": _ts(timedelta(hours=12)), + "elapsed": 1200.0, + "job_template": 11, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-package-update.yml", + "limit": "", + "extra_vars": '{"target_cve": "CVE-2024-54321"}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 11, "name": "CVE Remediation - Package Update"}, + }, + }, + { + "id": 1010, + "type": "job", + "name": "Compliance Check - STIG", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(days=1, hours=2)), + "finished": _ts(timedelta(days=1)), + "elapsed": 7200.0, + "job_template": 20, + "inventory": 3, + "project": 7, + "playbook": "playbooks/compliance/check-all.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "scheduled", + "summary_fields": { + "job_template": {"id": 20, "name": "Compliance Check - STIG"}, + }, + }, + { + "id": 1020, + "type": "job", + "name": "Emergency Patching", + "job_type": "run", + "status": "failed", + "failed": True, + "started": _ts(timedelta(days=7, hours=1)), + "finished": _ts(timedelta(days=7)), + "elapsed": 3600.0, + "job_template": 25, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/emergency-patch.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 25, "name": "Emergency Patching"}, + }, + }, + { + "id": 1025, + "type": "job", + "name": "Fleet Health Report", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=6, minutes=30)), + "finished": _ts(timedelta(hours=6)), + "elapsed": 1800.0, + "job_template": 30, + "inventory": 3, + "project": 8, + "playbook": "playbooks/reporting/fleet-health.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "scheduled", + "summary_fields": { + "job_template": {"id": 30, "name": "Fleet Health Report"}, + }, + }, +] + +_next_job_id = 2000 + + +# --------------------------------------------------------------------------- +# Mock stdout generators +# --------------------------------------------------------------------------- + +def _generate_stdout(job: dict) -> str: + """Generate realistic Ansible playbook stdout for a job.""" + playbook_name = job.get("name", "Unknown") + job_type = job.get("job_type", "run") + status = job.get("status", "successful") + limit = job.get("limit", "") + hosts = limit.split(",") if limit else PROD_HOSTS[:3] + hosts = [h.strip() for h in hosts if h.strip()] + extra_vars = job.get("extra_vars", "{}") + mode = " (CHECK MODE)" if job_type == "check" else "" + + lines = [] + lines.append(f"PLAY [{playbook_name}] *****") + lines.append("") + + lines.append(f"TASK [Gathering Facts{mode}] *****") + for h in hosts: + lines.append(f"ok: [{h}]") + lines.append("") + + if "kernel" in playbook_name.lower(): + lines.append(f"TASK [Create boom snapshot for rollback{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}] => {{\"msg\": \"boom create --title pre-remediation-CVE-2024-12345\"}}") + lines.append("") + + lines.append(f"TASK [Check disk space for kernel update{mode}] *****") + for h in hosts: + lines.append(f"ok: [{h}] => {{\"msg\": \"Disk space OK: 45% used\"}}") + lines.append("") + + lines.append(f"TASK [Update kernel package{mode}] *****") + for h in hosts: + result = "changed" if status == "successful" else "fatal" + if result == "changed": + lines.append(f'changed: [{h}] => {{"msg": "kernel-5.14.0-362.24.1.el9_3 -> kernel-5.14.0-362.24.2.el9_3"}}') + else: + lines.append(f'fatal: [{h}]: FAILED! => {{"msg": "Permission denied", "rc": 1}}') + lines.append("") + + lines.append(f"TASK [Check if reboot is needed (needs-restarting -r){mode}] *****") + for h in hosts: + lines.append(f'changed: [{h}] => {{"rc": 1, "msg": "Reboot is required to fully utilize updates."}}') + lines.append("") + + elif "package" in playbook_name.lower(): + lines.append(f"TASK [Update target packages for CVE remediation{mode}] *****") + for h in hosts: + lines.append(f'changed: [{h}] => {{"msg": "httpd-2.4.53-7.el9 -> httpd-2.4.57-8.el9"}}') + lines.append("") + + lines.append(f"TASK [Restart affected services{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}]") + lines.append("") + + lines.append(f"TASK [Verify service health{mode}] *****") + for h in hosts: + lines.append(f'ok: [{h}] => {{"msg": "Service httpd is running"}}') + lines.append("") + + elif "emergency" in playbook_name.lower() and status == "failed": + lines.append(f"TASK [Apply emergency patch{mode}] *****") + for h in hosts: + lines.append(f'fatal: [{h}]: FAILED! => {{"msg": "Missing sudo password (become_enabled not set)", "rc": 1}}') + lines.append("") + lines.append("NO MORE HOSTS LEFT *****") + lines.append("") + + else: + lines.append(f"TASK [Execute playbook tasks{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}]") + lines.append("") + + lines.append("PLAY RECAP *****") + for h in hosts: + if status == "successful": + ok_count = random.randint(3, 6) + changed_count = random.randint(1, 3) + lines.append(f"{h:<45} : ok={ok_count} changed={changed_count} unreachable=0 failed=0 skipped=0 rescued=0 ignored=0") + elif status == "failed": + lines.append(f"{h:<45} : ok=1 changed=0 unreachable=0 failed=1 skipped=0 rescued=0 ignored=0") + lines.append("") + + return "\n".join(lines) + + +def _generate_events(job: dict) -> list[dict]: + """Generate realistic Ansible task events for a job.""" + hosts = (job.get("limit", "").split(",") if job.get("limit") else PROD_HOSTS[:3]) + hosts = [h.strip() for h in hosts if h.strip()] + events: list[dict] = [] + eid = 1 + + task_names = ["Gathering Facts"] + if "kernel" in job.get("name", "").lower(): + task_names += [ + "Create boom snapshot for rollback", + "Check disk space for kernel update", + "Update kernel package", + "Check if reboot is needed (needs-restarting -r)", + ] + elif "package" in job.get("name", "").lower(): + task_names += [ + "Update target packages for CVE remediation", + "Restart affected services", + "Verify service health", + ] + else: + task_names += ["Execute playbook tasks"] + + for task_name in task_names: + for host in hosts: + is_failed = job.get("status") == "failed" and task_name != "Gathering Facts" + events.append({ + "id": eid, + "type": "job_event", + "event": "runner_on_ok" if not is_failed else "runner_on_failed", + "task": task_name, + "host": host, + "host_name": host, + "play": job.get("name", ""), + "changed": task_name != "Gathering Facts" and not is_failed, + "failed": is_failed, + "event_data": { + "task": task_name, + "host": host, + "res": { + "changed": task_name != "Gathering Facts" and not is_failed, + "msg": "Task completed" if not is_failed else "Permission denied", + }, + }, + "created": _ts(timedelta(hours=4, minutes=30 - eid)), + }) + eid += 1 + + return events + + +def _generate_host_summaries(job: dict) -> list[dict]: + """Generate per-host summaries for a job.""" + hosts = (job.get("limit", "").split(",") if job.get("limit") else PROD_HOSTS[:3]) + hosts = [h.strip() for h in hosts if h.strip()] + summaries: list[dict] = [] + + for i, host in enumerate(hosts): + is_failed = job.get("status") == "failed" + summaries.append({ + "id": i + 1, + "type": "job_host_summary", + "host": i + 1, + "host_name": host, + "ok": 1 if is_failed else random.randint(3, 6), + "changed": 0 if is_failed else random.randint(1, 3), + "dark": 0, + "failures": 1 if is_failed else 0, + "skipped": 0, + "processed": 1, + "failed": is_failed, + }) + + return summaries + + +# --------------------------------------------------------------------------- +# MCP Tools: Job Management +# --------------------------------------------------------------------------- + +@mcp.tool() +def job_templates_list( + page_size: int = 10, + search: Optional[str] = None, +) -> dict: + """List available job templates in AAP. + + Args: + page_size: Number of results per page (default 10, max 200). + search: Optional search string to filter templates by name. + """ + results = MOCK_JOB_TEMPLATES + if search: + s = search.lower() + results = [t for t in results if s in t["name"].lower() or s in t.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def job_templates_retrieve(id: str) -> dict: + """Retrieve detailed information about a specific job template. + + Args: + id: Job template ID (as string). + """ + tid = int(id) + template = next((t for t in MOCK_JOB_TEMPLATES if t["id"] == tid), None) + if not template: + return {"detail": f"Not found. Job template {id} does not exist."} + return template + + +@mcp.tool() +def projects_list( + page_size: int = 50, + search: Optional[str] = None, +) -> dict: + """List available projects in AAP. + + Args: + page_size: Number of results per page. + search: Optional search string to filter projects by name. + """ + results = MOCK_PROJECTS + if search: + s = search.lower() + results = [p for p in results if s in p["name"].lower() or s in p.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def job_templates_launch_retrieve( + id: str, + requestBody: Optional[dict] = None, +) -> dict: + """Launch a job from a job template. + + Args: + id: Job template ID to launch. + requestBody: Optional launch parameters including job_type ('run' or 'check'), + extra_vars (dict), and limit (comma-separated host list). + """ + global _next_job_id + tid = int(id) + template = next((t for t in MOCK_JOB_TEMPLATES if t["id"] == tid), None) + if not template: + return {"detail": f"Not found. Job template {id} does not exist."} + + body = requestBody or {} + job_type = body.get("job_type", template.get("job_type", "run")) + + if not template.get("ask_job_type_on_launch") and job_type != template.get("job_type"): + return { + "error": f"Cannot override job_type: ask_job_type_on_launch is disabled on template {id}", + } + + job_id = _next_job_id + _next_job_id += 1 + + new_job = { + "id": job_id, + "type": "job", + "name": template["name"], + "job_type": job_type, + "status": "pending", + "failed": False, + "started": _ts(timedelta(seconds=0)), + "finished": None, + "elapsed": 0.0, + "job_template": tid, + "inventory": template["inventory"], + "project": template["project"], + "playbook": template["playbook"], + "limit": body.get("limit", ""), + "extra_vars": str(body.get("extra_vars", {})), + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": tid, "name": template["name"]}, + }, + } + MOCK_JOBS.append(new_job) + + # Simulate job completion after launch + new_job["status"] = "successful" + new_job["finished"] = _ts(timedelta(seconds=-300)) + new_job["elapsed"] = 300.0 + + return { + "job": job_id, + "status": "pending", + "type": "job", + "url": f"/api/controller/v2/jobs/{job_id}/", + "related": { + "stdout": f"/api/controller/v2/jobs/{job_id}/stdout/", + "job_events": f"/api/controller/v2/jobs/{job_id}/job_events/", + "job_host_summaries": f"/api/controller/v2/jobs/{job_id}/job_host_summaries/", + }, + } + + +@mcp.tool() +def jobs_retrieve(id: int) -> dict: + """Get the status and details of a job run. + + Args: + id: Job ID to retrieve. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + return job + + +@mcp.tool() +def jobs_list(page_size: int = 10) -> dict: + """List recent job runs. + + Args: + page_size: Number of results to return. + """ + results = sorted(MOCK_JOBS, key=lambda j: j.get("started", ""), reverse=True) + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def jobs_stdout_retrieve(id: int, format: str = "txt") -> dict: + """Get the stdout (console output) from a job run. + + Args: + id: Job ID. + format: Output format ('txt' or 'json'). Default 'txt'. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + return { + "content": _generate_stdout(job), + "range": {"start": 0, "end": 1}, + } + + +@mcp.tool() +def jobs_job_events_list(id: int, page_size: int = 50) -> dict: + """Get task-level events for a job run. + + Args: + id: Job ID. + page_size: Number of events to return. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + events = _generate_events(job) + return { + "count": len(events), + "next": None, + "previous": None, + "results": events[:page_size], + } + + +@mcp.tool() +def jobs_job_host_summaries_list(id: int) -> dict: + """Get per-host execution summaries for a job run. + + Args: + id: Job ID. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + summaries = _generate_host_summaries(job) + return { + "count": len(summaries), + "next": None, + "previous": None, + "results": summaries, + } + + +@mcp.tool() +def jobs_relaunch_retrieve( + id: int, + hosts: str = "all", + job_type: str = "run", +) -> dict: + """Relaunch a previously completed or failed job. + + Args: + id: Original job ID to relaunch. + hosts: Which hosts to target ('all' or 'failed'). + job_type: Job type for relaunch ('run' or 'check'). + """ + global _next_job_id + original = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not original: + return {"detail": f"Not found. Job {id} does not exist."} + + new_id = _next_job_id + _next_job_id += 1 + + new_job = { + **original, + "id": new_id, + "job_type": job_type, + "status": "successful", + "failed": False, + "started": _ts(timedelta(seconds=0)), + "finished": _ts(timedelta(seconds=-300)), + "elapsed": 300.0, + "launch_type": "relaunch", + } + MOCK_JOBS.append(new_job) + + return { + "job": new_id, + "status": "pending", + "type": "job", + "url": f"/api/controller/v2/jobs/{new_id}/", + } + + +# --------------------------------------------------------------------------- +# MCP Tools: Inventory Management +# --------------------------------------------------------------------------- + +@mcp.tool() +def inventories_list( + page_size: int = 10, + search: Optional[str] = None, +) -> dict: + """List available inventories in AAP. + + Args: + page_size: Number of results per page. + search: Optional search string to filter inventories. + """ + results = MOCK_INVENTORIES + if search: + s = search.lower() + results = [inv for inv in results if s in inv["name"].lower() or s in inv.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def hosts_list( + inventory_id: Optional[int] = None, + page_size: int = 50, + search: Optional[str] = None, +) -> dict: + """List hosts in an inventory. + + Args: + inventory_id: Filter by inventory ID. If not provided, lists hosts from all inventories. + page_size: Number of results per page. + search: Optional search string to filter hosts by name. + """ + inv_id = inventory_id or 1 + hosts = _generate_hosts(inv_id) + if search: + s = search.lower() + hosts = [h for h in hosts if s in h["name"].lower()] + return { + "count": len(hosts), + "next": None if len(hosts) <= page_size else f"/api/controller/v2/hosts/?page=2", + "previous": None, + "results": hosts[:page_size], + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..fe5d549c --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/SKILL.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/SKILL.md new file mode 100644 index 00000000..510776ca --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/SKILL.md @@ -0,0 +1,321 @@ +--- +name: job-template-creator +description: | + Create AAP (Ansible Automation Platform) job templates for executing playbooks. Use when users request: + - "Create a job template for this playbook" + - "Set up a template to run remediation playbooks" + - "Configure AAP to execute this playbook" + - "Add a new job template for CVE remediation" + + This skill guides through adding playbooks to Git projects and creating job templates via AAP Web UI. +model: inherit +color: blue +--- + +# AAP Job Template Creator Skill + +This skill helps SREs create AAP job templates for executing Ansible playbooks, particularly for CVE remediation workflows. + +## Prerequisites + +**Required AAP Components**: +- AAP (Ansible Automation Platform) instance with API access +- Projects configured with playbooks +- Inventories with target hosts +- Credentials for authentication + +**Required MCP Servers**: `aap-mcp-job-management` ([setup guide](https://docs.redhat.com/)) + +**Currently Available MCP Tools** (read-only): +- `job_templates_list` - List existing templates +- `job_templates_retrieve` - Get template details +- `projects_list` - List available projects +- `inventories_list` - List available inventories + +**Missing MCP Tools** (needed for creation): +- ⚠️ `job_templates_create` - **NOT CURRENTLY AVAILABLE** +- ⚠️ `job_templates_update` - **NOT CURRENTLY AVAILABLE** + +**Required Environment Variables**: +- `AAP_MCP_SERVER` - Base URL for the MCP endpoint of the AAP server (must point to the AAP MCP gateway) +- `AAP_API_TOKEN` - AAP API authentication token + +### Prerequisite Validation + +**CRITICAL**: Before executing operations, execute the `/mcp-aap-validator` skill to verify AAP MCP server availability. + +**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-aap-validator/SKILL.md#validation-freshness-policy). + +**How to invoke**: Execute the `/mcp-aap-validator` skill + +**Handle validation result**: +- **If validation PASSED**: Continue with job template creation workflow +- **If validation PARTIAL**: Warn user and ask to proceed +- **If validation FAILED**: Stop execution, provide setup instructions from validator + +## Current Limitation: No Create Tools Available + +⚠️ **IMPORTANT**: The current AAP MCP implementation does **NOT** include tools to create job templates programmatically. The available MCP tools are read-only (list, retrieve, launch). + +**Current Approach**: +- Job templates must be created through the **AAP Web UI** +- This skill provides step-by-step instructions for Web UI creation +- Future MCP tool additions will enable programmatic template creation + +This skill documents both the **current manual workflow** and the **intended automated workflow** for when creation tools become available. + +## When to Use This Skill + +**Use this skill when you need**: +- Create a new job template for a remediation playbook +- Configure AAP to execute dynamically generated playbooks +- Set up templates for CVE remediation workflows +- Automate job template creation as part of remediation setup + +**Do NOT use this skill when**: +- Job templates already exist (use `/playbook-executor` skill instead) +- Only need to execute existing templates (use `job_templates_launch_retrieve`) +- Need to modify existing templates (requires AAP Web UI currently) + +## Invocation from playbook-executor + +When invoked from the [playbook-executor](../playbook-executor/SKILL.md) skill (Scenario 3 - No suitable template), this skill receives playbook content in context. The playbook-executor invokes with an instruction such as: + +``` +Create a job template for this remediation playbook. Playbook: [content]. Filename: [filename]. Path: [our_playbook_path]. CVE: [cve_id]. Target systems: [list]. +``` + +**When playbook content is provided**: +- Use the provided content for Phase 1 (Prepare Playbook in Git) instead of asking the user to supply it +- Write the playbook to the specified path in the user's Git repository (ask for repo path if not provided) +- Follow the Git flow: add, commit (with checkpoint for confirmation), push +- Then guide template creation via AAP Web UI (Phase 4) +- **Output**: Include the created template ID and name in the final report so playbook-executor can retrieve and validate it + +**Phase 0 - Check context**: If playbook content is provided by the invoking skill, execute the git flow (write, add, commit with confirmation checkpoint, push) before guiding template creation. Otherwise, use the existing manual flow where the user supplies the playbook. + +## Workflow + +### Phase 0: Validate AAP MCP Prerequisites + +**Action**: Execute the `/mcp-aap-validator` skill + +**Note**: Can skip if validation was performed earlier in this session and succeeded. See [Validation Freshness Policy](../mcp-aap-validator/SKILL.md#validation-freshness-policy). + +**How to invoke**: Execute the `/mcp-aap-validator` skill + +**Handle validation result**: +- **If validation PASSED**: Continue to Phase 1 +- **If validation PARTIAL**: Warn user and ask to proceed +- **If validation FAILED**: Stop execution, user must set up AAP MCP servers + +### Phase 1: Prepare Playbook in Git Project + +**Goal**: Add playbook to a Git repository AAP can access. + +**Read [references/01-git-setup.md](references/01-git-setup.md)** for Option A (existing repo) and Option B (new repo). + +**Verification**: Playbook committed, pushed, AAP synced, playbook path noted. + +### Phase 2: Gather Required Information + +Before creating a job template, collect: + +1. **Playbook Information**: + - Playbook name/path (e.g., `remediation-CVE-2025-49794.yml`) + - Project where playbook is stored + - Required variables/parameters + +2. **Target Information**: + - Inventory containing target hosts + - Host groups or specific hosts to target + - Any host limits or filters + +3. **Credentials**: + - SSH credentials for host access + - Vault passwords (if playbook uses Ansible Vault) + - Cloud credentials (if targeting cloud resources) + +4. **Execution Settings**: + - Job type (run/check) + - Verbosity level + - Concurrent execution limits + - Timeout settings + +### Phase 3: Verify Prerequisites + +**Step 1: List Available Projects** + +**MCP Tool**: `projects_list` (from aap-mcp-job-management) + +**Parameters**: +- `page_size`: 50 (retrieve up to 50 projects) +- `search`: "remediation" (optional - filter by keyword) + +**Expected Output**: +```json +{ + "count": 1, + "results": [ + { + "id": 6, + "name": "Remediation Playbooks", + "scm_type": "git", + "scm_url": "https://github.com/org/playbooks.git", + "status": "successful" + } + ] +} +``` + +**Action**: Identify the project ID where your playbook is stored. + +**Step 2: List Available Inventories** + +**MCP Tool**: `inventories_list` (from aap-mcp-inventory-management) + +**Parameters**: +- `page_size`: 50 +- `search`: "production" (optional - filter by keyword) + +**Expected Output**: +```json +{ + "count": 1, + "results": [ + { + "id": 1, + "name": "Production Inventory", + "total_hosts": 150, + "has_active_failures": false + } + ] +} +``` + +**Action**: Identify the inventory ID containing your target hosts. + +**Step 3: Verify Credentials** + +**Note**: The current AAP MCP doesn't expose credential listing tools. You'll need credential IDs from AAP Web UI or administrator. + +### Phase 4: Create Job Template via AAP Web UI + +⚠️ **CURRENT LIMITATION**: AAP MCP has no create tools. Template creation must be done via AAP Web UI. + +**Read [references/02-web-ui-form.md](references/02-web-ui-form.md)** for form fields and steps. + +**Required**: Name, Inventory, Project, Playbook, Credentials. Enable Privilege Escalation. Prompt on Launch: Job Type (REQUIRED), Variables, Limit. + +### Phase 5: Verify Template Creation + +**MCP Tool**: `job_templates_list` (from aap-mcp-job-management) + +**Parameters**: +- `search`: "CVE-2025-49794" (search for your template) +- `page_size`: 10 + +**Expected Output**: +```json +{ + "results": [ + { + "id": 42, + "name": "Remediate CVE-2025-49794", + "playbook": "remediation-CVE-2025-49794.yml", + "project": 6, + "inventory": 1, + "status": "never updated" + } + ] +} +``` + +**Success Criteria**: +- ✓ Template appears in search results +- ✓ Playbook path matches your playbook +- ✓ Project and inventory IDs are correct +- ✓ Template status is valid + +### Phase 6: Test Template Execution (Optional) + +**MCP Tool**: `job_templates_launch_retrieve` (from aap-mcp-job-management) + +**Parameters**: +- `id`: "42" (template ID from Phase 5) + +**Expected Output**: +```json +{ + "job": 1234, + "status": "pending", + "url": "/api/controller/v2/jobs/1234/" +} +``` + +**Follow-up**: Use `playbook-executor` skill to track job execution. + +## Output and Examples + +**Read [references/03-output-template.md](references/03-output-template.md)** for report format. +**Read [references/04-examples.md](references/04-examples.md)** for CVE remediation and dynamic variable examples. + +## Dependencies + +### Required MCP Servers +- `aap-mcp-job-management` - AAP job management API access + +### Required MCP Tools (Current) +- `job_templates_list` - List existing templates (verification) +- `job_templates_retrieve` - Get template details (verification) +- `projects_list` - List available projects (prerequisite) +- `inventories_list` (from aap-mcp-inventory-management) - List inventories (prerequisite) + +### Missing MCP Tools (Needed for Full Automation) +- `job_templates_create` - Create new job templates +- `job_templates_update` - Modify existing templates +- `credentials_list` - List available credentials + +### Related Skills +- `mcp-aap-validator` - **PREREQUISITE** - Validates AAP MCP server before creation (invoke in Phase 0 if not validated in session) +- `job-template-remediation-validator` - Validates created template meets remediation requirements +- `playbook-executor` - Execute templates after creation +- `playbook-generator` - Generate remediation playbooks for templates +- `system-context` - Identify target systems for inventory selection + +### Reference Documentation +- [AAP 2.6 Job Templates Documentation](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates) +- [AAP 2.6 Creating Projects](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects) + +## Best Practices + +1. **Use descriptive template names** - Include CVE ID or purpose: "Remediate CVE-2025-49794" +2. **Enable variable prompts for flexibility** - Check "Variables" in the "Prompt on Launch" section for dynamic values +3. **Set appropriate timeouts** - CVE remediation can take time; set generous timeouts +4. **Use privilege escalation** - Most remediation requires sudo/root access +5. **Document template purpose** - Use description field to explain usage +6. **Version playbooks** - Keep playbooks in Git for change tracking +7. **Test templates first** - Use check mode or test inventory before production +8. **Set concurrent limits** - Prevent overwhelming infrastructure with simultaneous jobs +9. **Enable notifications** - Configure email/webhook alerts for job completion +10. **Regular template audits** - Review and update templates as playbooks evolve + +## Human-in-the-Loop Requirements + +This skill requires user confirmation for: + +1. **Git Operations** (adding playbook to repository): + - Display: "I'll help you add the playbook to your Git repository" + - Ask: "Proceed with Git operations (clone, commit, push)?" + - Wait for confirmation + +2. **Manual Template Creation** (AAP Web UI): + - Display: "Template creation requires using the AAP Web UI" + - Ask: "I'll provide step-by-step instructions. Ready to proceed?" + - Wait for confirmation + +3. **Test Execution** (optional verification): + - Ask: "Should I test the template by launching a job?" + - Wait for confirmation before launching + +**Never assume approval** - always wait for explicit user confirmation. diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/01-git-setup.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/01-git-setup.md new file mode 100644 index 00000000..1a367a2e --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/01-git-setup.md @@ -0,0 +1,25 @@ +# Git Setup for Playbooks + +Read when guiding user through Phase 1 (Prepare Playbook in Git). + +## Option A: Add to Existing Project + +1. Ask: repo URL, local path, or "I don't have one" +2. Clone or `cd` to repo +3. `mkdir -p playbooks/remediation`; copy playbook; `git add`; `git commit`; `git push` +4. Sync AAP project (Automation Execution → Projects → Sync) +5. Note playbook path: `playbooks/remediation/remediation-CVE-YYYY-NNNNN.yml` + +## Option B: Create New Repository + +1. `mkdir ansible-remediation-playbooks`; `git init`; `mkdir -p playbooks/remediation` +2. Copy playbook; create README, .gitignore; `git add .`; `git commit` +3. Create remote repo; `git remote add origin `; `git push -u origin main` +4. Add project in AAP Web UI (Automation Execution → Projects → Add) +5. Note playbook path + +## Verification Checklist + +- Playbook committed and pushed +- AAP project synced +- Playbook path noted for template creation diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/02-web-ui-form.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/02-web-ui-form.md new file mode 100644 index 00000000..690d63ec --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/02-web-ui-form.md @@ -0,0 +1,24 @@ +# AAP Web UI Job Template Form + +Read when guiding Phase 4 (Create Template via Web UI). AAP MCP has no create tools—use Web UI. + +## Form Fields + +**Required**: Name, Inventory, Project, Playbook, Credentials (Machine/SSH) +**Job Type**: Run (or Check for dry-run) +**Options**: Enable Privilege Escalation: Yes +**Prompt on Launch** (check): Job Type (REQUIRED), Variables, Limit + +**Extra Variables** (optional): +```yaml +target_cve: "CVE-YYYY-NNNNN" +remediation_mode: "automated" +verify_after: true +``` + +## Steps + +1. Automation Execution → Templates → Add → Job Template +2. Fill form; Save +3. Note template ID from URL or details +4. Verify via `job_templates_list(search="CVE-ID")` diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/03-output-template.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/03-output-template.md new file mode 100644 index 00000000..496d2c45 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/03-output-template.md @@ -0,0 +1,20 @@ +# Job Template Creation Output + +Read when completing template creation. + +## Report Format + +```markdown +# AAP Job Template Created + +**Name**: Remediate CVE-YYYY-NNNNN +**ID**: [template_id] +**Project**: [name] (ID: [id]) +**Playbook**: playbooks/remediation/remediation-CVE-YYYY-NNNNN.yml +**Inventory**: [name] (ID: [id]) + +## Next Steps +1. Execute via AAP Web UI or job_templates_launch_retrieve +2. Monitor via jobs_retrieve, jobs_stdout_retrieve +3. Verify via remediation-verifier skill +``` diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/04-examples.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/04-examples.md new file mode 100644 index 00000000..d19c66c6 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/04-examples.md @@ -0,0 +1,19 @@ +# Job Template Creator Examples + +Read when handling specific request types. + +## Example 1: CVE Remediation Template + +**Request**: "Create a job template for CVE-2025-49794 playbook" +- Phase 1: Git setup (see 01-git-setup.md)—add playbook, commit, push, sync AAP +- Phase 2: Gather playbook path, project, inventory +- Phase 3: projects_list, inventories_list +- Phase 4: Web UI instructions (see 02-web-ui-form.md) +- Phase 5: job_templates_list to verify + +## Example 2: Dynamic CVE Template + +**Request**: "Template with variable CVE ID" +- Enable "Prompt on Launch" → Variables +- Extra vars: cve_id, remediation_mode, verify_after +- Override at launch for different CVEs diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-remediation-validator/SKILL.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-remediation-validator/SKILL.md new file mode 100644 index 00000000..c86141d4 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-remediation-validator/SKILL.md @@ -0,0 +1,414 @@ +--- +name: job-template-remediation-validator +description: | + Verify an AAP job template meets requirements for executing CVE remediation playbooks. + + Use when: + - "Does this job template support remediation playbooks?" + - "Validate job template X for CVE remediation" + - "Check if template is ready for playbook-executor" + - Before playbook-executor selects a template + + NOT for: AAP MCP connectivity (use `/mcp-aap-validator`), creating templates (use `/job-template-creator`). +model: inherit +color: blue +--- + +# AAP Job Template Remediation Validator + +This skill verifies that an AAP (Ansible Automation Platform) job template meets the requirements for executing CVE remediation playbooks as defined by the remediation skill and playbook-executor workflow. + +## Prerequisites + +**Required MCP Servers**: `aap-mcp-job-management`, `aap-mcp-inventory-management` ([setup guide](https://docs.redhat.com/)) + +**Required MCP Tools**: +- `job_templates_list` (from aap-mcp-job-management) - List job templates +- `job_templates_retrieve` (from aap-mcp-job-management) - Get template details +- `projects_list` (from aap-mcp-job-management) - Verify project exists and status +- `inventories_list` (from aap-mcp-inventory-management) - Verify inventory exists + +**Required Environment Variables**: +- `AAP_MCP_SERVER` - Base URL for the MCP endpoint of the AAP server (must point to the AAP MCP gateway) +- `AAP_API_TOKEN` - AAP API authentication token + +### Prerequisite Validation + +**CRITICAL**: Before executing, execute the `/mcp-aap-validator` skill to verify AAP MCP server availability. + +**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-aap-validator/SKILL.md#validation-freshness-policy). + +**How to invoke**: Execute the `/mcp-aap-validator` skill + +**Handle validation result**: +- **If validation PASSED**: Continue with template validation +- **If validation PARTIAL**: Warn user and ask to proceed +- **If validation FAILED**: Stop execution, provide setup instructions from validator + +**Human Notification on Failure**: +If prerequisites are not met: +- ❌ "Cannot proceed: AAP MCP servers are not available" +- 📋 "Setup required: Configure AAP_MCP_SERVER and AAP_API_TOKEN environment variables" +- ❓ "How would you like to proceed? (setup now / skip / abort)" +- ⏸️ Wait for user decision + +## When to Use This Skill + +**Use this skill when**: +- Verifying a job template before playbook execution +- Checking if a template meets remediation requirements +- Auditing existing templates for remediation readiness +- Troubleshooting "template not compatible" in playbook-executor + +**Do NOT use when**: +- Validating AAP MCP connectivity → Use `/mcp-aap-validator` skill +- Creating new job templates → Use `/job-template-creator` skill +- Executing playbooks → Use `/playbook-executor` skill + +## Remediation Template Requirements + +This skill validates against the requirements documented in [playbook-executor](../playbook-executor/SKILL.md) and [job-template-creator](../job-template-creator/SKILL.md). + +### Required (Must Pass) + +| Requirement | Description | Validation | +|-------------|-------------|------------| +| **Inventory** | Template has inventory configured | `inventory` field present and non-null | +| **Project** | Template has project configured | `project` field present and non-null | +| **Playbook** | Template has playbook path | `playbook` field present, non-empty | +| **Credentials** | Machine credential (SSH) configured | `summary_fields.credentials` or `credentials` has at least one credential | +| **Privilege Escalation** | Required for package updates | `become_enabled` is true | +| **Ask Job Type on Launch** | Required for dry-run and run modes | `ask_job_type_on_launch` is true | + +**Why Ask Job Type on Launch**: playbook-executor uses the same template for dry-run (`job_type: "check"`) and actual execution (`job_type: "run"`). Without `ask_job_type_on_launch: true`, the template is locked to one mode and you would need separate templates for check vs run. + +**Example**: Template with `job_type: "check"` (default) and `ask_job_type_on_launch: true` allows launching as check for dry-run or run for execution. + +### Recommended (Warnings if Missing) + +| Requirement | Description | Validation | +|-------------|-------------|------------| +| **Ask Variables on Launch** | Enables dynamic CVE targeting | `ask_variables_on_launch` is true | +| **Ask Limit on Launch** | Enables host targeting at launch | `ask_limit_on_launch` is true | +| **Ask Inventory on Launch** | Enables inventory override at launch | `ask_inventory_on_launch` is true | + +### Optional Context Checks + +| Check | Description | +|-------|-------------| +| **Project Status** | Project exists and is synced (status "successful") | +| **Inventory Exists** | Inventory exists in AAP | +| **Playbook Path** | Path suggests remediation playbook (e.g., contains "remediation") | +| **Playbook Path Matching** | When used by playbook-executor (Scenario 3), the template's playbook path is trusted to match the playbook just created via job-template-creator | + +## Workflow + +### Phase 0: Validate AAP MCP Prerequisites + +**Action**: Execute the `/mcp-aap-validator` skill + +**Note**: Can skip if validation was performed earlier in this session and succeeded. + +**Handle validation result**: +- **If validation PASSED**: Continue to Phase 1 +- **If validation PARTIAL**: Warn user and ask to proceed +- **If validation FAILED**: Stop execution + +### Phase 1: Obtain Job Template + +**Goal**: Get the job template to validate. User may provide template ID or name. + +#### Option A: User Provides Template ID + +If user specifies a template ID (e.g., "42" or "template 42"): + +**MCP Tool**: `job_templates_retrieve` (from aap-mcp-job-management) + +**Parameters**: +- `id`: Template ID as string (e.g., `"42"`) + +**Expected Output**: Full job template object with fields: `id`, `name`, `inventory`, `project`, `playbook`, `become_enabled`, `ask_variables_on_launch`, `ask_limit_on_launch`, `summary_fields` (may include `credentials`), `credentials` (array of credential IDs) + +**Error Handling**: +- If 404 or template not found: Report "Template ID X not found. Verify the ID exists in AAP." +- If connection error: Report per mcp-aap-validator troubleshooting + +#### Option B: User Provides Template Name or No ID + +If user says "validate my remediation template" or provides a name: + +**MCP Tool**: `job_templates_list` (from aap-mcp-job-management) + +**Parameters**: +- `page_size`: 50 +- `search`: User-provided name or "remediation" (optional) + +**Action**: List templates, let user select by number or ID. If exactly one match, use it. If multiple, present list and ask user to choose. + +### Phase 2: Validate Required Fields + +**Goal**: Check each required field against the template response. + +**Input**: Template object from `job_templates_retrieve` + +**Validation Logic**: + +``` +required_checks = [] +required_checks.append(("Inventory", template.get("inventory") is not None and template.get("inventory") != "")) +required_checks.append(("Project", template.get("project") is not None and template.get("project") != "")) +required_checks.append(("Playbook", template.get("playbook") is not None and len(str(template.get("playbook", "")).strip()) > 0)) +required_checks.append(("Privilege Escalation", template.get("become_enabled") == True)) + +# Credentials: AAP API may return credentials in summary_fields.credentials or credentials array +creds = template.get("summary_fields", {}).get("credentials") or template.get("credentials") or [] +has_creds = (isinstance(creds, list) and len(creds) > 0) or (isinstance(creds, dict) and creds) +required_checks.append(("Credentials", has_creds)) +required_checks.append(("Ask Job Type on Launch", template.get("ask_job_type_on_launch") == True)) +``` + +**Note**: If the AAP MCP response structure differs, adapt the field paths. Common AAP API response structures: +- `inventory`: number (ID) +- `project`: number (ID) +- `playbook`: string (path) +- `become_enabled`: boolean +- `credentials`: array of credential IDs, or `summary_fields.credentials` array of objects with `id`, `name` + +### Phase 3: Validate Recommended Fields + +**Validation Logic**: + +``` +recommended_checks = [] +recommended_checks.append(("Ask Variables on Launch", template.get("ask_variables_on_launch") == True)) +recommended_checks.append(("Ask Limit on Launch", template.get("ask_limit_on_launch") == True)) +recommended_checks.append(("Ask Inventory on Launch", template.get("ask_inventory_on_launch") == True)) +``` + +### Phase 4: Optional Context Verification + +**Goal**: Verify referenced project and inventory exist and are usable. + +**Step 4.1: Verify Project Exists and Status** + +**MCP Tool**: `projects_list` (from aap-mcp-job-management) + +**Parameters**: +- `page_size`: 100 +- `search`: Optional - filter by project ID if API supports it + +**Action**: Search results for `id == template["project"]`. If found, check `status`: +- `"successful"`: ✓ Project synced, playbooks available +- `"failed"` or `"error"`: ⚠ Project sync failed - playbooks may be stale +- `"pending"` or `"running"`: ⚠ Project syncing - wait before use + +**Step 4.2: Verify Inventory Exists** + +**MCP Tool**: `inventories_list` (from aap-mcp-inventory-management) + +**Parameters**: +- `page_size`: 100 + +**Action**: Search results for `id == template["inventory"]`. If found: ✓ Inventory exists. If not found: ⚠ Inventory ID not found (may be permission issue). + +### Phase 5: Generate Validation Report + +**Output Format**: + +```markdown +# Job Template Remediation Validation Report + +**Template**: {name} (ID: {id}) +**Validated**: {timestamp} + +## Required Checks +| Requirement | Status | Details | +|-------------|--------|---------| +| Inventory | ✓/✗ | {inventory_id} - {inventory_name or "configured"} | +| Project | ✓/✗ | {project_id} - {project_name or "configured"} | +| Playbook | ✓/✗ | {playbook_path} | +| Credentials | ✓/✗ | {count} credential(s) configured | +| Privilege Escalation | ✓/✗ | become_enabled: {value} | +| Ask Job Type on Launch | ✓/✗ | Required for dry-run + run modes | + +## Recommended Checks +| Requirement | Status | Details | +|-------------|--------|---------| +| Ask Variables on Launch | ✓/⚠ | {value} | +| Ask Limit on Launch | ✓/⚠ | {value} | +| Ask Inventory on Launch | ✓/⚠ | {value} | + +## Context Verification +| Check | Status | Details | +|-------|--------|---------| +| Project Exists | ✓/⚠/✗ | {status} | +| Inventory Exists | ✓/⚠/✗ | {details} | + +## Overall Result +{✓ PASSED / ⚠ PASSED WITH WARNINGS / ✗ FAILED} + +{If PASSED}: Template is ready for remediation playbook execution. +{If WARNINGS}: Template works but consider enabling ask_variables_on_launch and ask_limit_on_launch for flexibility. +{If FAILED}: Fix required checks before using with playbook-executor. See job-template-creator for setup guidance. If Ask Job Type on Launch fails: Enable "Prompt on Launch" for Job Type in AAP Web UI → Templates → [Template] → Edit → Options. +``` + +### Pass/Fail Determination + +- **PASSED**: All 6 required checks pass +- **PASSED WITH WARNINGS**: All required pass, one or more recommended fail +- **FAILED**: One or more required checks fail + +## Dependencies + +### Required MCP Servers +- `aap-mcp-job-management` - AAP job template and execution management +- `aap-mcp-inventory-management` - AAP inventory management + +### Required MCP Tools +- `job_templates_list` (from aap-mcp-job-management) - List templates +- `job_templates_retrieve` (from aap-mcp-job-management) - Get template details +- `projects_list` (from aap-mcp-job-management) - Verify project +- `inventories_list` (from aap-mcp-inventory-management) - Verify inventory + +### Related Skills +- `mcp-aap-validator` - **PREREQUISITE** - Validates AAP MCP before this skill +- `playbook-executor` - **PRIMARY USER** - Uses compatible templates for execution +- `job-template-creator` - Creates templates that this skill validates + +### Reference Documentation +- [playbook-executor/SKILL.md](../playbook-executor/SKILL.md) - Template compatibility requirements (Phase 1 Step 1.2, Scenario 3 validation) +- [job-template-creator/SKILL.md](../job-template-creator/SKILL.md) - Template configuration for remediation +- [AAP Job Templates](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates) + +## Example Usage + +### Example 1: Validate Template by ID + +**User Request**: "Validate job template 42 for remediation" + +**Skill Response**: + +1. Invoke mcp-aap-validator → PASSED +2. Call `job_templates_retrieve(id="42")` → Get template +3. Run validation checks +4. Call `projects_list`, `inventories_list` for context +5. Output report: + +``` +# Job Template Remediation Validation Report + +**Template**: CVE Remediation Template (ID: 42) +**Validated**: 2026-02-24 + +## Required Checks +| Requirement | Status | Details | +|-------------|--------|---------| +| Inventory | ✓ | 1 - Production Inventory | +| Project | ✓ | 6 - Remediation Playbooks | +| Playbook | ✓ | playbooks/remediation/remediation-template.yml | +| Credentials | ✓ | 1 credential(s) configured | +| Privilege Escalation | ✓ | become_enabled: true | + +## Recommended Checks +| Requirement | Status | Details | +|-------------|--------|---------| +| Ask Variables on Launch | ✓ | true | +| Ask Limit on Launch | ✓ | true | +| Ask Inventory on Launch | ✓ | true | + +## Overall Result +✓ PASSED + +Template is ready for remediation playbook execution. +``` + +### Example 2: Template Fails - Missing Ask Job Type on Launch + +**User Request**: "Validate template 20 for remediation" + +**Skill Response**: + +``` +# Job Template Remediation Validation Report + +**Template**: CVE Remediation (ID: 20) +**Validated**: 2026-02-24 + +## Required Checks +| Requirement | Status | Details | +|-------------|--------|---------| +| Inventory | ✓ | 1 - Production Inventory | +| Project | ✓ | 6 - Remediation Playbooks | +| Playbook | ✓ | playbooks/remediation/remediation-template.yml | +| Credentials | ✓ | 1 credential(s) configured | +| Privilege Escalation | ✓ | become_enabled: true | +| Ask Job Type on Launch | ✗ | ask_job_type_on_launch: false | + +## Overall Result +✗ FAILED + +Fix required: Enable "Prompt on Launch" for Job Type. +playbook-executor needs the same template for dry-run (job_type: "check") and actual execution (job_type: "run"). Without this, you would need separate templates for each mode. + +To fix: AAP Web UI → Templates → [Template] → Edit → Options → ✓ Prompt on Launch → Job Type +``` + +### Example 3: Template Fails - Missing Privilege Escalation + +**User Request**: "Check if template 15 works for remediation" + +**Skill Response**: + +``` +# Job Template Remediation Validation Report + +**Template**: General Playbook Runner (ID: 15) +**Validated**: 2026-02-24 + +## Required Checks +| Requirement | Status | Details | +|-------------|--------|---------| +| Inventory | ✓ | 1 - Production Inventory | +| Project | ✓ | 6 - Remediation Playbooks | +| Playbook | ✓ | playbooks/example.yml | +| Credentials | ✓ | 1 credential(s) configured | +| Privilege Escalation | ✗ | become_enabled: false | + +## Overall Result +✗ FAILED + +Fix required: Enable privilege escalation (become) on this template. +Remediation playbooks require sudo/root for package updates and system changes. + +To fix: AAP Web UI → Templates → [Template] → Edit → Options → ✓ Enable Privilege Escalation +``` + +### Example 4: Invoked by Playbook-Executor + +**Context**: playbook-executor filters templates and may invoke this skill to validate user-selected template before execution. + +**Workflow**: +``` +[playbook-executor] → User selects template ID 10 +[playbook-executor] → Invoke job-template-remediation-validator with template 10 +[job-template-remediation-validator] → Returns PASSED +[playbook-executor] → Proceeds with execution +``` + +## Critical: Human-in-the-Loop Requirements + +This skill performs **read-only validation** only. It does not modify AAP resources or execute playbooks. + +**When user input is needed**: +- **Template selection**: If multiple templates match a search, present the list and ask user to select by number or ID before proceeding +- **Template not found**: If template ID invalid, report error and ask user for correct ID or "list" to see available templates + +**No confirmation required** for validation execution - the skill only reads and reports. + +## Best Practices + +1. **Validate before execution** - Run this skill before playbook-executor when using a new or unfamiliar template +2. **Enable recommended options** - ask_variables_on_launch and ask_limit_on_launch improve flexibility +3. **Project sync** - Ensure project status is "successful" before execution +4. **Credential types** - Template should have Machine (SSH) credential; Vault optional for encrypted playbooks +5. **Naming convention** - Use descriptive names like "Remediate CVE-YYYY-NNNNN" for auditability diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/mcp-aap-validator/SKILL.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/mcp-aap-validator/SKILL.md new file mode 100644 index 00000000..a1c4f708 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/mcp-aap-validator/SKILL.md @@ -0,0 +1,66 @@ +--- +name: mcp-aap-validator +description: | + Validate AAP (Ansible Automation Platform) MCP server connectivity. Use when the user asks to "validate AAP MCP", "check AAP connection", or when other skills need to verify AAP MCP availability before job management or inventory operations. +model: haiku +color: yellow +--- + +# MCP AAP Validator + +Validates connectivity to AAP MCP servers by running lightweight tool calls. + +## When to Use This Skill + +Use when validating AAP MCP before job template operations, troubleshooting connection issues, or when other skills (e.g. playbook-executor) need to verify availability. Do NOT use for creating templates—use job-template-creator. + +## Workflow + +1. **Test connectivity**: Call these tools to verify each server responds: + - `job_templates_list` (page_size: 10) from aap-mcp-job-management + - `inventories_list` (page_size: 10) from aap-mcp-inventory-management +2. **If any fails**: Provide a comprehensive message with possible root causes (see below). +3. **Report**: Output a table with validated servers and outcome (emojis). + +## Failure Message (Root Causes) + +When a tool call fails, include: + +``` +❌ AAP MCP connection failed + +**Possible root causes:** +- **Credentials**: AAP_MCP_SERVER or AAP_API_TOKEN not set or invalid +- **401 Unauthorized**: Token expired or invalid → regenerate in AAP Web UI +- **403 Forbidden**: Token lacks RBAC permissions (need Job Templates, Inventories) +- **404 Not Found**: Wrong AAP_MCP_SERVER URL (must point to MCP gateway, not main AAP UI) +- **Connection timeout**: Server unreachable, firewall, or network issue +- **SSL/TLS error**: Certificate verification problem + +**Troubleshooting:** +1. Verify env vars: AAP_MCP_SERVER, AAP_API_TOKEN (never echo values) +2. Get token: AAP Web UI → Users → [Your User] → Tokens → Create +3. Ensure AAP_MCP_SERVER points to MCP gateway endpoint +4. Restart host after config changes +``` + +## Report Format + +Always end with a table: + +| Server | Outcome | +|--------|---------| +| aap-mcp-job-management | ✅ PASSED | +| aap-mcp-inventory-management | ✅ PASSED | + +Use ✅ for success, ❌ for failure, ⚠️ for partial (e.g. one server OK, one failed). + +## Dependencies + +### Required MCP Tools +- `job_templates_list` (from aap-mcp-job-management) - Connectivity test +- `inventories_list` (from aap-mcp-inventory-management) - Connectivity test + +### Required MCP Servers +- `aap-mcp-job-management` - AAP job template and execution +- `aap-mcp-inventory-management` - AAP inventory management diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/SKILL.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/SKILL.md new file mode 100644 index 00000000..a29c9443 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/SKILL.md @@ -0,0 +1,499 @@ +--- +name: playbook-executor +description: | + **CRITICAL**: Use for Ansible playbook execution via AAP. DO NOT call AAP MCP tools directly. + + Execute remediation playbooks with job management, dry-run, and reporting. Use after playbook-generator. + + **Git Flow**: If template playbook path ≠ generated playbook, perform Git Flow (commit, push, sync) BEFORE launch. +--- + +# AAP Playbook Executor Skill + +This skill executes Ansible remediation playbooks through AAP (Ansible Automation Platform) with full job management capabilities. + +**Integration with Remediation Skill**: The `/remediation` skill orchestrates this skill as part of its Step 5 (Execute Playbook) workflow. For standalone playbook execution, you can invoke this skill directly. + +## Prerequisites + +**Required MCP Servers**: `aap-mcp-job-management`, `aap-mcp-inventory-management` ([setup guide](https://docs.redhat.com/)) + +**Required MCP Tools**: +- `job_templates_list` (from aap-mcp-job-management) - List job templates +- `job_templates_retrieve` (from aap-mcp-job-management) - Get template details +- `projects_list` (from aap-mcp-job-management) - Get project name and scm_url for Git Flow +- `job_templates_launch_retrieve` (from aap-mcp-job-management) - Launch jobs +- `jobs_retrieve` (from aap-mcp-job-management) - Get job status +- `jobs_stdout_retrieve` (from aap-mcp-job-management) - Get console output +- `jobs_job_events_list` (from aap-mcp-job-management) - Get task events +- `jobs_job_host_summaries_list` (from aap-mcp-job-management) - Get host statistics +- `inventories_list` (from aap-mcp-inventory-management) - List inventories +- `hosts_list` (from aap-mcp-inventory-management) - List inventory hosts + +**Required Environment Variables**: +- `AAP_MCP_SERVER` - Base URL for the MCP endpoint of the AAP server (must point to the AAP MCP gateway) +- `AAP_API_TOKEN` - AAP API authentication token + +### Prerequisite Validation + +**CRITICAL**: Before executing operations, execute the `/mcp-aap-validator` skill to verify AAP MCP server availability. + +**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-aap-validator/SKILL.md#validation-freshness-policy). + +**How to invoke**: Execute the `/mcp-aap-validator` skill + +**Handle validation result**: +- **If validation PASSED**: Continue with playbook execution workflow +- **If validation PARTIAL**: Warn user and ask to proceed +- **If validation FAILED**: Stop execution, provide setup instructions from validator + +**Human Notification on Failure**: +If prerequisites are not met: +- ❌ "Cannot proceed: AAP MCP servers are not available" +- 📋 "Setup required: Configure AAP_MCP_SERVER and AAP_API_TOKEN environment variables" +- ❓ "How would you like to proceed? (setup now / skip / abort)" +- ⏸️ Wait for user decision + +## When to Use This Skill + +**Use this skill directly when you need**: +- Execute a previously generated Ansible playbook via AAP +- Track the status of a running AAP job +- Monitor playbook job completion +- Run dry-run (check mode) before production execution +- Verify playbook execution succeeded + +**Use the `/remediation` skill when you need**: +- Full remediation workflow including playbook execution +- Integrated CVE analysis → playbook generation → execution → verification +- End-to-end remediation orchestration + +**How they work together**: The `/remediation` skill invokes this skill after generating a remediation playbook, managing the full workflow from analysis to verification. + +## Workflow + +**Git Flow is MANDATORY**: When the job template's playbook path differs from the generated playbook (or content must be updated), you MUST perform Git Flow (write, commit, push, sync) and receive "sync complete" from the user BEFORE launching any job. Do NOT skip this—launching without it executes the wrong playbook. + +### Phase 0: Validate AAP MCP Prerequisites + +**Action**: Execute the `/mcp-aap-validator` skill + +**Note**: Can skip if validation was performed earlier in this session and succeeded. + +**How to invoke**: Execute the `/mcp-aap-validator` skill + +**Handle validation result**: +- **If validation PASSED**: Continue to Phase 1 +- **If validation PARTIAL**: Warn user and ask to proceed +- **If validation FAILED**: Stop execution, user must set up AAP MCP servers + +### Phase 1: Job Template Selection and Playbook Preparation + +**Goal**: Identify an AAP job template suitable for executing the remediation playbook. **Git Flow is MANDATORY** before Phase 3 when the template points to a different playbook or when content must be updated. + +**Input**: Playbook content and metadata from playbook-generator (filename, CVE ID, target systems). The playbook YAML is already in context—do NOT regenerate it during Git Flow. Playbook path is derived from metadata: `playbooks/remediation/` (e.g., `playbooks/remediation/remediation-CVE-2025-49794.yml`). + +**BLOCKING**: You MUST NOT launch any job (dry-run or production) until the playbook is in the Git repo and the user has confirmed "sync complete". AAP executes from the synced project—there is no "override at launch". Launching without Git Flow executes the WRONG playbook. + +#### Step 1.1: Derive Playbook Path + +From playbook metadata (filename from playbook-generator): +- Use convention `playbooks/remediation/` +- Support both `remediation-CVE-*.yml` and `remediation-CVE-*-playbook.yml` patterns. +- Example: CVE-2026-26103 → `playbooks/remediation/remediation-CVE-2026-26103.yml` + +#### Step 1.2: List Templates and Validate Each Candidate + +**MCP Tool**: `job_templates_list` (from aap-mcp-job-management) + +**Parameters**: +- `page_size`: 50 (retrieve up to 50 templates) +- `search`: "" (search for all templates) + +**REQUIRED**: For each template in results: +1. Call `job_templates_retrieve(id)` to get full details +2. **Invoke the `/job-template-remediation-validator` skill** with the template ID to verify it meets remediation requirements (inventory, project, playbook, credentials, become_enabled) +3. Only include templates that PASS validation in the lists below + +Build two lists: +- **exact_match**: `template.playbook` equals `our_playbook_path` (normalize slashes; match if equal or basenames match) +- **compatible_other**: Passes job-template-remediation-validator but **different playbook path** (template points to e.g. `cve-remediation.yml` while we have `remediation-CVE-2026-26103.yml`) + +**Path normalization**: Normalize slashes, handle `playbooks/remediation/` prefix. Match if `template.playbook` equals `our_playbook_path` or if basenames match. **Different filenames = different path = Scenario 2.** + +#### Step 1.3: Scenario Selection (MANDATORY - Do Not Skip) + +**Scenario 1 - Same playbook path** (exact_match not empty): + +The template already points to our playbook path. The project may need the latest content. **Read [references/05-git-flow-prompts.md](references/05-git-flow-prompts.md)** for Scenario 1 prompt, options (A/B), and Git Flow steps. + +- **If A**: Execute Git Flow (see Git Flow section below). **BLOCK Phase 3** until user confirms "sync complete" or "done". +- **If B**: Wait for user confirmation. **BLOCK Phase 3** until user confirms. + +**Scenario 2 - Different playbook path** (compatible_other not empty, exact_match empty): + +**CRITICAL**: The template points to a DIFFERENT playbook than our generated playbook. You MUST NOT launch the job without Git Flow—AAP executes from synced content; there is no override at launch. **Read [references/05-git-flow-prompts.md](references/05-git-flow-prompts.md)** for Scenario 2 prompt and Git Flow steps. + +- **If yes**: Execute Git Flow. **BLOCK Phase 3** until Git Flow completes and user confirms "sync complete". +- **If no**: Fall through to Scenario 3. + +**Anti-pattern**: Do NOT say "I'll override with our playbook" and then launch—that is impossible. The playbook MUST be in the repo before launch. + +**Scenario 3 - No suitable template** (exact_match and compatible_other both empty, or user chose "no" in Scenario 2): + +Execute the `/job-template-creator` skill with instruction: +``` +"Create a job template for this remediation playbook. Playbook: [content]. Filename: [filename]. Path: [our_playbook_path]. CVE: [cve_id]. Target systems: [list]." +``` + +The job-template-creator skill guides the user through: (1) Adding playbook to Git repository, (2) Syncing AAP project, (3) Creating job template via AAP Web UI with correct path, inventory, credentials, privilege escalation. + +After `/job-template-creator` completes, retrieve the template ID (from skill output or user confirmation). Execute `/job-template-remediation-validator` to validate the newly created template. If passed, proceed to Phase 3 (Dry-Run). If failed, report issues and ask user to fix in AAP Web UI. + +**Multiple matches**: If multiple exact matches, present list and ask user to choose by number. If multiple different-path matches, prefer by project name containing "remediation" or "CVE", else first. + +**Phase 1 Checkpoint** (BLOCKING - must pass before Phase 3): +- **Git Flow required**: If Scenario 1 or 2, you MUST complete Git Flow and receive "sync complete" from the user before proceeding. Do NOT skip. +- **No override**: There is no way to "override" the playbook at launch. AAP runs whatever is in the synced project. +- **Never launch** if the playbook has not been committed, pushed, and synced + +#### Git Flow (for Scenario 1 Override and Scenario 2) - MANDATORY HITL + +**When**: Scenario 1 (same path, update content) or Scenario 2 (different path, replace playbook). **Do not skip**—execution with wrong playbook content will remediate the wrong CVE. + +**Target path**: +- Scenario 1: `our_playbook_path` (e.g. `playbooks/remediation/remediation-CVE-2026-26103.yml`) +- Scenario 2: `template.playbook` (e.g. `playbooks/remediation/cve-remediation.yml`)—we replace the template's playbook with our generated content + +**Prerequisite**: Ask user for the local path to the Git repository. Use `projects_list` for project name and `scm_url`. **Read [references/05-git-flow-prompts.md](references/05-git-flow-prompts.md)** for repo path question, HITL checkpoint text, and after-push message. + +**Steps** (execute in order; HITL at checkpoint): +1. **Write playbook to file** (FAST—do NOT regenerate): + - The playbook content is ALREADY in context from playbook-generator (or remediation skill). Use it directly. + - **⚠️ ABSOLUTE PATH REQUIRED**: The Write path MUST start with `/`. Use: `/`. Example: `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project/playbooks/remediation/cve-remediation.yml` + - **WRONG** (causes "Error writing file"): `test-aap-project/playbooks/...` or `playbooks/remediation/...` — these are relative and fail when repo is outside workspace. + - **Before Write**: Confirm path starts with `/`. If not, prepend the user's repo path. + - Do NOT invoke playbook-generator, do NOT call MCP tools, do NOT re-fetch. This should take seconds, not minutes. +2. Use Run tool: `git add ` (from repo root, e.g. `git add playbooks/remediation/cve-remediation.yml`) +3. **HITL Checkpoint** (REQUIRED): Display summary per reference file. Wait for "yes" or "proceed" +4. If confirmed: `git commit -m "Add/update remediation playbook for CVE-YYYY-NNNNN"` +5. `git push origin main` (or branch from project's scm_branch if available) + +**Note**: Git must be configured. Use Run tool for git commands. + +**Do NOT proceed to Phase 3 (Dry-Run) until user confirms sync complete.** + +### Phase 2: Git Flow (MANDATORY before Phase 3) + +**BLOCKING**: You MUST NOT proceed to Phase 3 (Dry-Run) until Git Flow is complete. + +**When**: Scenario 1 (same path, update content) or Scenario 2 (different path). See Phase 1 Step 1.3. + +**Checkpoint**: Before Phase 3, confirm: +- [ ] Playbook written to repo at target path +- [ ] Git commit and push completed (with user confirmation) +- [ ] User confirmed "sync complete" after AAP project sync + +**If any unchecked**: STOP. Do Git Flow. Do NOT launch the job. + +### Phase 3: Dry-Run Execution (Recommended) + +**Prerequisite**: Phase 2 (Git Flow) MUST be complete. User must have confirmed "sync complete". + +**Goal**: Test playbook in check mode before actual execution to simulate changes. + +**Read [references/04-dry-run-display-templates.md](references/04-dry-run-display-templates.md)** for: Playbook Preview, Dry-Run Offer, Dry-Run Results Display, Proceed prompt. + +#### Step 3.1–3.2: Display Preview and Offer Dry-Run + +Show playbook structure per reference. Offer dry-run with options: yes / no / abort. **ONLY if user confirms**, proceed. + +#### Step 3.3: Launch Dry-Run Job + +**Pre-launch check** (BLOCKING): If Scenario 1 or 2 applied, you MUST have completed Git Flow and received "sync complete" from the user. If not, STOP—do not launch. Return to Phase 2 / Git Flow. + +**MCP Tool**: `job_templates_launch_retrieve` (from aap-mcp-job-management) + +**Parameters**: `id`, `requestBody` with `job_type: "check"`, `extra_vars`, `limit` + +**Key**: `job_type: "check"` - Runs Ansible in check mode (dry-run) + +#### Step 3.4: Monitor Dry-Run Progress + +Poll `jobs_retrieve` every 2 seconds. Use `jobs_job_events_list` for live task updates. + +#### Step 3.5: Display Dry-Run Results + +**MCP Tools**: `jobs_stdout_retrieve` (id, format: "txt"), `jobs_job_host_summaries_list` (id). Use display format from reference. + +#### Step 3.6: Proceed to Actual Execution? + +Ask per reference. Wait for "yes" or "execute". + +### Phase 4: Actual Execution + +**ONLY execute if user explicitly confirms** (either after dry-run or directly if they skipped dry-run). + +#### Step 4.1: Final Confirmation + +``` +⚠️ CRITICAL: Playbook Execution Confirmation Required + +This playbook will: +- Execute on: 3 production systems +- Update packages: httpd (2.4.53-7.el9 → 2.4.57-8.el9) +- Restart services: httpd +- Estimated downtime: ~10 seconds per system +- Requires reboot: No + +Job Template: CVE Remediation Template (ID: 10) +AAP URL: https://aap.example.com/jobs/ + +❓ Execute this playbook now? + +Options: +- "yes" or "execute" - Proceed with execution +- "abort" - Cancel execution + +Please respond with your choice. +``` + +Wait for explicit "yes" or "execute" response. + +#### Step 4.2: Launch Production Job + +**Pre-launch check** (BLOCKING): Same as Phase 3—if Scenario 1 or 2 applied, Git Flow must be complete and user must have confirmed "sync complete". Do NOT launch without it. + +**MCP Tool**: `job_templates_launch_retrieve` (from aap-mcp-job-management) + +**Parameters**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run", + "extra_vars": { + "target_cve": "CVE-2025-49794", + "remediation_mode": "automated", + "verify_after": true + }, + "limit": "prod-web-01,prod-web-02,prod-web-03" + } +} +``` + +**Key Parameter**: `job_type: "run"` - Runs Ansible in execution mode (actual changes) + +**Expected Output**: +```json +{ + "job": 1235, + "status": "pending", + "url": "/api/controller/v2/jobs/1235/" +} +``` + +#### Step 4.3: Monitor Execution Progress + +**Polling Strategy**: +1. Call `jobs_retrieve(id=job_id)` every 2 seconds +2. Get task events with `jobs_job_events_list(id=job_id)` for progress updates +3. Display real-time task completion status +4. Continue until status is "successful", "failed", or "error" + +**Progress Display**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running +Elapsed: 1m 23s +AAP URL: https://aap.example.com/#/jobs/playbook/1235 + +Recent Events: +- ✓ Gathering Facts (completed - all hosts) +- ✓ Check Disk Space (completed - all hosts) +- ✓ Backup Configuration (completed - all hosts) +- ⏳ Update Package: httpd (running - prod-web-01, prod-web-02) + └─ prod-web-01: Installing httpd-2.4.57-8.el9... + └─ prod-web-02: Installing httpd-2.4.57-8.el9... +- ⏸ Restart Service: httpd (pending) +``` + +**Update every 2 seconds** until completion. + +### Phase 5: Execution Report + +**Goal**: Generate comprehensive report with job details, per-host results, and full output. + +**Read [references/01-execution-report-templates.md](references/01-execution-report-templates.md)** for JSON examples, comprehensive report template, and Success/Partial Success/Failure output templates. + +#### Step 5.1–5.4: Gather Data + +**MCP Tools** (all from aap-mcp-job-management): +- `jobs_retrieve` (id) - Job details +- `jobs_job_host_summaries_list` (id) - Per-host stats +- `jobs_job_events_list` (id) - Task timeline +- `jobs_stdout_retrieve` (id, format: "txt") - Full console output + +#### Step 5.5: Generate Report + +Format all gathered data per reference. Use Success / Partial Success / Failure template based on job status. + +#### Step 5.6: Validate Job Log for CVE Handling (MANDATORY) + +**Goal**: Confirm from the job stdout that the playbook actually addressed the target CVE(s). + +**Input**: Target CVE ID(s) from invocation (e.g. CVE-2025-49794). Job stdout from `jobs_stdout_retrieve` (already gathered in Step 5.4). + +**Parse stdout for**: +- Target CVE ID(s) in output (vars, task names, audit logs, playbook metadata) +- Package update tasks for affected packages (dnf/yum install/update, package module) +- Remediation-related task names (e.g. "Update package", "Restart service", "remediation") + +**Report** (add to execution report): +- **✓ Job log confirms CVE-XXXX-YYYY was addressed** — CVE ID or package updates found in stdout +- **⚠️ Job log did not show clear evidence of CVE handling** — No CVE ID or package updates found; recommend manual verification or `/remediation-verifier` + +**Batch**: For multiple CVEs, validate each. Report per-CVE confirmation or warning. + +### Phase 6: Error Handling + +**If job status is "failed" or "error"**, provide detailed troubleshooting. + +**Read [references/02-error-handling-guide.md](references/02-error-handling-guide.md)** for: Error categories, error report template, troubleshooting steps, relaunch parameters. + +#### Step 6.1: Parse Error Output + +**MCP Tool**: `jobs_stdout_retrieve`. Analyze output for error categories per reference. + +#### Step 6.2: Generate Error Report + +Use error report template from reference. Include per-host results, failed task details, troubleshooting steps, relaunch options. + +#### Step 6.3: Offer Relaunch + +If user chooses relaunch: **MCP Tool** `jobs_relaunch_retrieve` with `hosts: "failed"`, `job_type: "run"` per reference. + +## Reference Files + +| File | Use When | +|------|----------| +| [01-execution-report-templates.md](references/01-execution-report-templates.md) | Phase 5 reports, Success/Partial/Failure output | +| [02-error-handling-guide.md](references/02-error-handling-guide.md) | Phase 6 error reports, relaunch | +| [03-workflow-examples.md](references/03-workflow-examples.md) | Demo full workflow, failure handling, skip dry-run | +| [04-dry-run-display-templates.md](references/04-dry-run-display-templates.md) | Phase 3 preview, offer, results, proceed prompt | +| [05-git-flow-prompts.md](references/05-git-flow-prompts.md) | Scenario 1/2 prompts, Git Flow HITL, after-push | + +## Dependencies + +### Required MCP Servers +- `aap-mcp-job-management` - AAP job management and execution +- `aap-mcp-inventory-management` - AAP inventory management + +### Required MCP Tools +- `job_templates_list` (from aap-mcp-job-management) - List templates +- `job_templates_retrieve` (from aap-mcp-job-management) - Get template details +- `projects_list` (from aap-mcp-job-management) - Get project name and scm_url for Git Flow +- `job_templates_launch_retrieve` (from aap-mcp-job-management) - Launch jobs +- `jobs_retrieve` (from aap-mcp-job-management) - Get job status +- `jobs_stdout_retrieve` (from aap-mcp-job-management) - Get console output +- `jobs_job_events_list` (from aap-mcp-job-management) - Get task events +- `jobs_job_host_summaries_list` (from aap-mcp-job-management) - Get host statistics +- `inventories_list` (from aap-mcp-inventory-management) - List inventories +- `hosts_list` (from aap-mcp-inventory-management) - List hosts + +### Related Skills +- `mcp-aap-validator` - **PREREQUISITE** - Validates AAP MCP servers (invoke in Phase 0) +- `job-template-remediation-validator` - **REQUIRED** - Invoke for each candidate template in Phase 1 Step 1.2 to verify remediation requirements +- `job-template-creator` - Creates/guides AAP job template setup +- `playbook-generator` - Generates playbooks for execution +- `remediation-verifier` - Verifies success after execution + +### Reference Documentation +- [references/](references/) - Step-numbered reference files (01–05) for templates and examples +- [AAP Job Execution Guide](../../docs/ansible/aap-job-execution.md) - AAP job execution best practices +- [Playbook Integration with AAP](../../docs/ansible/playbook-integration-aap.md) - Playbook-to-AAP workflow + +## Critical: Human-in-the-Loop Requirements + +This skill executes code on production systems. **Explicit user confirmation is REQUIRED** at multiple stages. + +**Before Git commit/push** (Scenario 1 Override, Scenario 2): +1. **Display change summary**: File path, diff or file size +2. **Ask for confirmation**: "Ready to commit and push these changes? Reply 'yes' or 'proceed' to continue, or 'abort' to cancel." +3. **Wait for explicit "yes" or "proceed"**: Do not commit/push without confirmation + +**Before Dry-Run Execution** (if user chooses dry-run): +1. **Display Playbook Preview**: Show tasks and explain changes +2. **Ask for Dry-Run Confirmation**: + ``` + ❓ Run dry-run to simulate changes? + + Options: + - "yes" - Run dry-run (recommended) + - "no" - Skip to actual execution + - "abort" - Cancel + + Please respond with your choice. + ``` +3. **Wait for Explicit Response**: Do not proceed without confirmation + +**Before Actual Execution** (REQUIRED): +1. **Display Execution Summary**: Show systems, changes, downtime estimate +2. **Ask for Final Confirmation**: + ``` + ⚠️ CRITICAL: Execute playbook on production systems? + + This will make real changes to N systems. + + Options: + - "yes" or "execute" - Proceed + - "abort" - Cancel + + Please respond with your choice. + ``` +3. **Wait for Explicit "yes" or "execute"**: Do not proceed without confirmation + +**Never assume approval** - always wait for explicit user confirmation before executing playbooks. + +## Best Practices + +1. **Write path must be absolute** - When Git Flow writes the playbook to the user's repo, use `/playbooks/remediation/`. The path MUST start with `/`. Relative paths cause "Error writing file". +2. **Always validate AAP prerequisites** - Invoke mcp-aap-validator in Phase 0 +3. **Validate each template** - Invoke job-template-remediation-validator for each candidate before selection +4. **Never skip Git Flow** - If template playbook path ≠ generated playbook path (Scenario 2) or content must be updated (Scenario 1), you MUST complete Git Flow and receive "sync complete" before Phase 3. Do NOT launch without it. +5. **Recommend dry-run** - Offer check mode before production execution +6. **Filter compatible templates** - Check inventory, project, and credentials match +7. **Monitor in real-time** - Display task progress during execution +8. **Comprehensive reporting** - Include per-host stats, task timeline, full output +9. **Error categorization** - Parse errors and provide specific troubleshooting +10. **Relaunch capability** - Offer to retry failed hosts +11. **Link to AAP** - Provide direct URL to job in AAP Web UI +12. **Suggest verification** - Always recommend remediation-verifier after success +13. **Document job details** - Save job ID and template info for audit trail + +## Integration with Other Skills + +- **playbook-generator**: Generates playbooks that this skill executes +- **job-template-creator**: Creates AAP job templates when needed +- **remediation-verifier**: Verifies success after this skill completes execution +- **`/remediation` skill**: Orchestrates full workflow including playbook execution + +**Orchestration Example** (from `/remediation` skill): +1. Agent invokes playbook-generator skill → Creates playbook YAML +2. playbook-generator asks for confirmation → User approves playbook content +3. Agent invokes playbook-executor skill (this skill) → Execution workflow +4. Skill validates templates via job-template-remediation-validator → Filters valid candidates +5. Skill checks path match → If different path, offers Git Flow (HITL: commit/push, sync AAP) +6. Skill waits for "sync complete" before proceeding (if Git Flow was used) +7. Skill offers dry-run → User runs check mode +8. Skill asks for execution confirmation → User approves +9. Skill executes and monitors → Reports completion +10. Agent invokes remediation-verifier skill → Confirms CVE resolved + +**Note**: Both playbook-generator and playbook-executor require separate confirmations for different purposes: +- playbook-generator: Confirms playbook content is acceptable +- playbook-executor: Confirms execution on production systems is approved + +This two-step approval ensures user control over both what to run and when to run it. diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/01-execution-report-templates.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/01-execution-report-templates.md new file mode 100644 index 00000000..a6773c5f --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/01-execution-report-templates.md @@ -0,0 +1,168 @@ +# Step 01: Execution Report Templates + +Read this reference when generating Phase 5 execution reports or output templates. + +## Phase 5: Job Details (JSON Examples) + +### jobs_retrieve Expected Output + +```json +{ + "id": 1235, + "name": "CVE Remediation Template", + "status": "successful", + "started": "2026-02-24T15:35:02Z", + "finished": "2026-02-24T15:40:25Z", + "elapsed": 323.45, + "job_template": 10, + "inventory": 1, + "limit": "prod-web-01,prod-web-02,prod-web-03", + "playbook": "playbooks/remediation/remediation-CVE-2025-49794.yml" +} +``` + +### jobs_job_host_summaries_list Expected Output + +```json +{ + "results": [ + { + "host_name": "prod-web-01", + "ok": 8, + "changed": 3, + "failed": 0, + "unreachable": 0 + }, + { + "host_name": "prod-web-02", + "ok": 8, + "changed": 3, + "failed": 0, + "unreachable": 0 + }, + { + "host_name": "prod-web-03", + "ok": 5, + "changed": 0, + "failed": 1, + "unreachable": 0 + } + ] +} +``` + +## Comprehensive Report Template + +```markdown +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 5m 23s +**Started**: 2026-02-24 15:35:02 UTC +**Completed**: 2026-02-24 15:40:25 UTC +**Job Template**: CVE Remediation Template +**Playbook**: playbooks/remediation/remediation-CVE-2025-49794.yml +**AAP URL**: [View in AAP](https://aap.example.com/#/jobs/playbook/1235) + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| prod-web-01 | 8 | 3 | 0 | 0 | ✅ Success | +| prod-web-02 | 8 | 3 | 0 | 0 | ✅ Success | +| prod-web-03 | 8 | 3 | 0 | 0 | ✅ Success | + +**Summary**: 3 of 3 hosts successfully remediated + +## Task Timeline +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) + - prod-web-01: 2.4.53-7.el9 → 2.4.57-8.el9 + - prod-web-02: 2.4.53-7.el9 → 2.4.57-8.el9 + - prod-web-03: 2.4.53-7.el9 → 2.4.57-8.el9 +5. ✅ Restart httpd service (15s) +6. ✅ Verify service status (2s) +7. ✅ Update audit log (1s) + +## Full Console Output +
+Click to expand (187 lines) + +[Full stdout from jobs_stdout_retrieve] + +
+ +## Job Log CVE Validation (Step 5.6) +✓ Job log confirms CVE-XXXX-YYYY was addressed + +*(Or: ⚠️ Job log did not show clear evidence of CVE handling—verify manually or use remediation-verifier)* + +## Next Steps +1. ✅ All systems successfully remediated +2. ☐ Verify remediation with remediation-verifier skill +3. ☐ Update vulnerability tracking system +4. ☐ Schedule follow-up verification in 24-48 hours + +--- + +**Recommendation**: Run remediation-verifier skill to confirm CVE status has been updated in Red Hat Lightspeed. +``` + +## Output Templates + +### Success Template + +```markdown +✅ Playbook Execution Successful + +Job ID: 1235 +Duration: 5m 23s +Systems Remediated: 3 of 3 + +View full report above for details. + +Next Steps: +- Run remediation-verifier skill to confirm CVE resolution +- Update vulnerability tracking system +- Monitor systems for 24-48 hours + +AAP URL: https://aap.example.com/#/jobs/playbook/1235 +``` + +### Partial Success Template + +```markdown +⚠️ Playbook Execution Completed with Failures + +Job ID: 1235 +Duration: 2m 45s +Systems Remediated: 2 of 3 +Failed Systems: prod-web-03 + +See error details above for troubleshooting steps. + +Options: +- Relaunch for failed hosts +- Manual remediation +- Skip failed hosts + +AAP URL: https://aap.example.com/#/jobs/playbook/1235 +``` + +### Failure Template + +```markdown +❌ Playbook Execution Failed + +Job ID: 1235 +Duration: 1m 15s +Systems Remediated: 0 of 3 + +Critical errors prevented execution. +See error details above for troubleshooting. + +AAP URL: https://aap.example.com/#/jobs/playbook/1235 +``` diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/02-error-handling-guide.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/02-error-handling-guide.md new file mode 100644 index 00000000..90492f00 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/02-error-handling-guide.md @@ -0,0 +1,108 @@ +# Step 02: Error Handling Guide + +Read this reference when generating Phase 6 error reports or troubleshooting. + +## Error Categories + +**Parse error output** from `jobs_stdout_retrieve` for these common patterns: + +1. **Connection Failures**: SSH timeout, host unreachable, authentication failed +2. **Permission Errors**: sudo required, insufficient privileges, SELinux denials +3. **Package Manager Issues**: repo unavailable, package not found, dependency conflicts +4. **Service Failures**: service not found, restart failed, timeout +5. **Disk Space**: insufficient space for updates +6. **General Failures**: playbook syntax errors, task failures + +## Error Report Template + +```markdown +# Playbook Execution Failed + +## Job Summary +**Job ID**: 1235 +**Status**: ❌ Failed +**Duration**: 2m 45s +**Started**: 2026-02-24 15:35:02 UTC +**Failed At**: 2026-02-24 15:37:47 UTC +**Job Template**: CVE Remediation Template +**AAP URL**: [View in AAP](https://aap.example.com/#/jobs/playbook/1235) + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| prod-web-01 | 8 | 3 | 0 | 0 | ✅ Success | +| prod-web-02 | 8 | 3 | 0 | 0 | ✅ Success | +| prod-web-03 | 5 | 0 | 1 | 0 | ❌ Failed | + +**Summary**: 2 of 3 hosts succeeded, 1 failed + +## Failed Tasks Details + +### Host: prod-web-03 + +**Task**: Restart httpd service +**Error**: "Failed to restart httpd.service: Unit httpd.service not found." + +**Error Category**: Service Failure + +**Root Cause**: The httpd service is not installed or not recognized by systemd. + +**Troubleshooting Steps**: +1. Check if httpd is installed: + ```bash + ssh prod-web-03 'rpm -q httpd' + ``` +2. If not installed, the package update may have failed: + ```bash + ssh prod-web-03 'dnf info httpd' + ``` +3. Check systemd service status: + ```bash + ssh prod-web-03 'systemctl status httpd' + ``` +4. Review package manager logs: + ```bash + ssh prod-web-03 'tail -50 /var/log/dnf.log' + ``` + +**Recommended Action**: +- Verify httpd package installation on prod-web-03 +- Check if package update completed successfully +- Manually install httpd if needed: `dnf install httpd` +- Relaunch job for failed host only + +## Console Output (Last 50 Lines) +
+Click to expand error context + +[Relevant error output from jobs_stdout_retrieve] + +
+ +## Relaunch Options + +Would you like to: +1. **Relaunch for failed hosts only** - Run job again with limit="prod-web-03" +2. **Fix issues manually and relaunch** - Resolve problems first, then relaunch +3. **View full job output** - See complete execution logs +4. **Abort** - Stop remediation workflow + +Please choose an option (1-4): +``` + +## Relaunch Parameters + +**MCP Tool**: `jobs_relaunch_retrieve` (from aap-mcp-job-management) + +**Parameters**: +```json +{ + "id": "1235", + "requestBody": { + "hosts": "failed", + "job_type": "run" + } +} +``` + +This relaunches the job for only the failed hosts. diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/03-workflow-examples.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/03-workflow-examples.md new file mode 100644 index 00000000..f5caa0d1 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/03-workflow-examples.md @@ -0,0 +1,119 @@ +# Step 03: Workflow Examples + +Read this reference when demonstrating end-to-end workflow patterns. + +## Example 1: Full Workflow with Dry-Run + +**User Request**: "Execute the CVE-2025-49794 remediation playbook" + +**Skill Response**: + +1. **Validate AAP Prerequisites**: + - Invoke mcp-aap-validator skill → PASSED + +2. **List Job Templates**: + - Call `job_templates_list()` → Found 2 templates + - Filter compatible templates → 1 matches requirements + +3. **User Selects Template**: + ``` + Found 1 compatible job template: + 1. "CVE Remediation Template" (ID: 10) + + Select template (1) or "create" for new: 1 + ``` + +4. **Playbook Preparation**: + ``` + Guide user to add playbook to Git: + - Commands provided + - User syncs AAP project + - Verification: Playbook available ✓ + ``` + +5. **Offer Dry-Run**: + ``` + Run dry-run first? yes + ``` + +6. **Execute Dry-Run**: + - Launch with `job_type="check"` + - Monitor progress → COMPLETED + - Display dry-run results: + ``` + Would change 3 tasks on 3 hosts + No errors detected + ``` + +7. **Proceed to Actual Execution**: + ``` + Proceed with actual execution? yes + ``` + +8. **Execute Playbook**: + - Launch with `job_type="run"` + - Monitor progress → COMPLETED + - Generate report: + ``` + ✅ 3 of 3 hosts successfully remediated + Duration: 5m 23s + ``` + +9. **Suggest Verification**: + ``` + Recommend: Run remediation-verifier skill to confirm CVE resolution + ``` + +## Example 2: Handle Execution Failure + +**User Request**: "Execute remediation playbook" + +**Skill Response**: + +1-7. [Same as Example 1 through execution] + +8. **Execute Playbook**: + - Launch job → FAILED on 1 of 3 hosts + - Generate error report: + ``` + ❌ prod-web-03 failed + Error: httpd service not found + Troubleshooting steps provided + ``` + +9. **Offer Relaunch**: + ``` + Relaunch for failed host only? yes + ``` + +10. **Relaunch Job**: + - Call `jobs_relaunch_retrieve` with hosts="failed" + - Monitor → COMPLETED + - Final report: + ``` + ✅ All 3 hosts successfully remediated (1 after retry) + ``` + +## Example 3: Skip Dry-Run + +**User Request**: "Execute playbook directly, skip dry-run" + +**Skill Response**: + +1-4. [Same as Example 1 through template selection] + +5. **Offer Dry-Run**: + ``` + Run dry-run first? no + ``` + +6. **Final Confirmation**: + ``` + ⚠️ Execute on production without dry-run? + This will make changes immediately. + Confirm: yes + ``` + +7. **Execute Playbook**: + - Launch with `job_type="run"` + - Monitor and report as in Example 1 diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/04-dry-run-display-templates.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/04-dry-run-display-templates.md new file mode 100644 index 00000000..d9b6f0dc --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/04-dry-run-display-templates.md @@ -0,0 +1,93 @@ +# Step 04: Dry-Run Display Templates + +Read this reference when displaying Phase 3 dry-run content. + +## Playbook Preview + +```markdown +# Playbook Preview + +**Playbook**: remediation-CVE-2025-49794.yml +**Target Systems**: 5 systems + +## Tasks Overview: +1. **Gather Facts** - Collect system information +2. **Check Disk Space** - Ensure sufficient space for updates (>500MB) +3. **Backup Configuration** - Snapshot critical configs +4. **Update Package: httpd** - Upgrade to version 2.4.57-8.el9 +5. **Restart Service: httpd** - Apply changes +6. **Verify Service Status** - Confirm httpd is running +7. **Update Audit Log** - Record remediation event + +**Estimated Duration**: 3-5 minutes per system +**Requires Reboot**: No +**Downtime**: Brief (~10 seconds during service restart) +``` + +## Dry-Run Offer + +``` +⚠️ Recommended: Run dry-run first + +Dry-run mode (--check) simulates changes without applying them. +This helps identify: +- Package availability issues +- Permission problems +- Configuration conflicts +- Unexpected side effects + +❓ Run dry-run before actual execution? +- "yes" - Run dry-run first (recommended) +- "no" - Skip to actual execution +- "abort" - Cancel execution + +Please respond with your choice. +``` + +## Dry-Run Results Display + +```markdown +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s +**Completed**: 2024-01-20 15:32:17 UTC + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| prod-web-01 | 3 | 8 | 0 | ✓ Ready | +| prod-web-02 | 3 | 8 | 0 | ✓ Ready | +| prod-web-03 | 3 | 8 | 0 | ✓ Ready | + +## Changes That Would Be Made: +1. **httpd package** - Would update from 2.4.53-7.el9 to 2.4.57-8.el9 +2. **httpd service** - Would restart +3. **audit log** - Would add remediation entry + +## Dry-Run Output: +
+Click to expand full output + +[Full stdout from jobs_stdout_retrieve] + +
+ +✓ No errors detected in dry-run +✓ All systems passed pre-flight checks +``` + +## Proceed to Actual Execution Prompt + +``` +❓ Dry-run completed successfully. Proceed with actual execution? + +Options: +- "yes" or "execute" - Proceed with actual remediation +- "review" - Show dry-run output again +- "abort" - Cancel execution + +Please respond with your choice. +``` diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/05-git-flow-prompts.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/05-git-flow-prompts.md new file mode 100644 index 00000000..41945d0e --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/05-git-flow-prompts.md @@ -0,0 +1,97 @@ +# Step 05: Git Flow Prompts + +Read this reference when executing Git Flow (Scenario 1 Override or Scenario 2). + +## Scenario 1 Prompt (Same path) + +The template already points to our playbook path. The project may need the latest content. + +``` +Found template [name] (ID: X) with matching playbook path. The project may need to be updated with the latest playbook. + +Options: +(A) Override: I'll add/update the playbook in the project via git. You sync the AAP project, then confirm. +(B) Manual: You add the playbook and sync. Confirm when done. + +❓ Choose (A) or (B): +``` + +- **If A**: Execute Git Flow (see Git Flow section below). Wait for user: "Sync complete" or "done". +- **If B**: Wait for user confirmation. + +## Scenario 2 Prompt (Different path) + +**CRITICAL**: The template points to a DIFFERENT playbook than our generated playbook. You MUST NOT launch the job without Git Flow. AAP executes from synced project content—there is no "override at launch". The playbook MUST be in the repo before any job launch. + +``` +Found template [name] (ID: X) pointing to [template.playbook]. Our generated playbook is [our_playbook_path]. + +⚠️ The template's playbook path does NOT match. We must update the playbook in the project before execution. + +Options: +- "yes" or "proceed" - I'll add our playbook to the project via git (you'll confirm commit/push, then sync AAP) +- "no" - Create a new template via `/job-template-creator` skill + +❓ Proceed with playbook update (git flow)? +``` + +- **If yes**: Execute Git Flow. **Do NOT proceed to Phase 3 until Git Flow completes.** +- **If no**: Fall through to Scenario 3 (job-template-creator). + +## Repo Path Question + +``` +What is the local path to the Git repository for project [Project Name] (scm_url)? +``` + +Use `projects_list` to get project name and `scm_url`; display to help user identify the repo. + +**Path format**: Ask for the **absolute path** (e.g. `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project`). When writing the playbook, the Write tool path MUST be `/playbooks/remediation/` — the full absolute path. Do NOT use a relative path like `test-aap-project/playbooks/...`; that causes "Error writing file". + +## Git Flow: Write Step (FAST) + +**CRITICAL**: The playbook is ALREADY generated. During Git Flow you must WRITE the existing content to disk—nothing more. + +- **DO**: Single file write of the playbook content already in context (from playbook-generator or remediation) +- **DO NOT**: Invoke playbook-generator again, call create_vulnerability_playbook, re-fetch from MCP, or validate/transform the content +- **Expected duration**: Seconds. If it takes minutes, you are doing unnecessary work. + +### Write Path (ABSOLUTE REQUIRED) + +**⚠️ WRITE PATH MUST START WITH `/`** — The Write tool path MUST be an absolute path. Relative paths cause "Error writing file" because the repo is often outside the workspace. + +**Formula**: `write_path = user_provided_path + "/" + target_path` + +- `user_provided_path` = exactly what the user typed (e.g. `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project`) +- `target_path` = e.g. `playbooks/remediation/cve-remediation.yml` + +**Correct**: `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project/playbooks/remediation/cve-remediation.yml` + +**WRONG** (will fail): +- `test-aap-project/playbooks/remediation/cve-remediation.yml` +- `playbooks/remediation/cve-remediation.yml` + +**Before calling Write**: Verify the path starts with `/`. If it does not, prepend the user's repo path. + +## Git Flow HITL Checkpoint + +**REQUIRED** before commit/push: + +``` +Ready to commit and push these changes? +- File: [target_path] +- CVE: [cve_id] +- This will update the playbook in the AAP project. + +Reply 'yes' or 'proceed' to continue, or 'abort' to cancel. +``` + +**Wait for user confirmation.** If "yes" or "proceed": `git commit -m "Add/update remediation playbook for CVE-YYYY-NNNNN"` then `git push origin main`. + +## After Push Message + +``` +I've pushed the playbook. Sync the AAP project: Automation Execution > Projects > [Project] > Sync. Reply 'sync complete' when done. +``` + +**Do NOT proceed to Phase 3 (Dry-Run) until user confirms sync complete.** diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/instruction.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/instruction.md new file mode 100644 index 00000000..55b78ca1 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/instruction.md @@ -0,0 +1,18 @@ +# Job Template Validation Task + +You are a Red Hat SRE. Before running a CVE remediation playbook through AAP, you need to verify that the job template is correctly configured and safe to execute. + +## Scenario +The team wants to use an existing AAP job template to remediate a critical vulnerability. Before giving the green light, you need to confirm the template meets all requirements for a safe remediation run. + +## Requirements +- Retrieve the job template configuration from AAP +- Verify required fields are set: inventory, project, playbook, credentials, and privilege escalation +- Check recommended settings: whether the template prompts for variables, limit, and inventory at launch +- Verify the referenced project and inventory actually exist in AAP +- Produce a pass/warn/fail assessment for each configuration item +- Summarize whether the template is ready for production remediation use + +Document your methodology, validation results, and assessment in `/root/report.md`. + +Use MCP tools to query AAP. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/solution/solve.sh b/evaluation/with_skills/rh-sre__job-template-remediation-validator/solution/solve.sh new file mode 100644 index 00000000..6e9ff39d --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/solution/solve.sh @@ -0,0 +1,21 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Job Template Validation + +## Required Checks +| Field | Expected | Status | +|-------|----------|--------| +| ask_job_type_on_launch | true | ✅ | +| become_enabled | true | ✅ | +| credentials | present | ✅ | +| inventory | present | ✅ | +| project | present | ✅ | +| playbook | present | ✅ | + +## Recommended +- ask_variables_on_launch: true +- ask_limit_on_launch: true + +## Overall +✓ PASSED - Template ready for remediation playbook execution. +REPORT_EOF diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/task.toml b/evaluation/with_skills/rh-sre__job-template-remediation-validator/task.toml new file mode 100644 index 00000000..2b6428ba --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__job-template-remediation-validator" +name = "rh-sre Job Template Validation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "job-template-remediation-validator", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/tests/llm_judge.py b/evaluation/with_skills/rh-sre__job-template-remediation-validator/tests/llm_judge.py new file mode 100644 index 00000000..106f21c9 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "ask_job_type_required", "file": "/root/report.md", "question": "Does the report require ask_job_type_on_launch: true for dual check/run mode support?", "reference": "A skilled report requires this for dry-run vs run flexibility. An unskilled report doesn't validate this field."}, + {"id": "become_and_credentials", "file": "/root/report.md", "question": "Does the report validate both become_enabled and credentials (checking summary_fields.credentials or credentials array)?", "reference": "A skilled report checks both credential locations. An unskilled report checks only one."}, + {"id": "required_vs_recommended", "file": "/root/report.md", "question": "Does the report distinguish required fields (inventory, project, playbook, credentials, become, ask_job_type) from recommended (ask_variables, ask_limit)?", "reference": "A skilled report categorizes validation checks by priority. An unskilled report treats all checks equally."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/tests/test.sh b/evaluation/with_skills/rh-sre__job-template-remediation-validator/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/tests/test_outputs.py b/evaluation/with_skills/rh-sre__job-template-remediation-validator/tests/test_outputs.py new file mode 100644 index 00000000..b39c5886 --- /dev/null +++ b/evaluation/with_skills/rh-sre__job-template-remediation-validator/tests/test_outputs.py @@ -0,0 +1,63 @@ +""" +Tests for rh-sre__job-template-remediation-validator per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['valid', 'job template', 'check']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_ask_job_type_on_launch(self): + """Skill teaches ask_job_type_on_launch: true is required for check vs run modes.""" + c = read_report().lower() + assert any(t in c for t in ["ask_job_type", "ask_job_type_on_launch"]), ( + "should require ask_job_type_on_launch (skill: for check vs run)" + ) + + def test_credentials_check_both_fields(self): + """Skill teaches credentials may be in summary_fields.credentials OR credentials array.""" + c = read_report().lower() + assert any(t in c for t in ["summary_fields", "credentials array", "both"]), ( + "should check credentials in summary_fields or credentials array (skill-specific)" + ) + + def test_become_enabled_required(self): + """Skill: become_enabled required for package updates.""" + c = read_report().lower() + assert any(t in c for t in ["become", "privilege", "escalat", "sudo"]), ( + "should require privilege escalation (skill: required for remediation)" + ) + + def test_required_vs_recommended(self): + """Skill: Distinguish required (inventory, project, playbook, credentials, become, ask_job_type) vs recommended (ask_variables, ask_limit).""" + c = read_report().lower() + has_required = any(t in c for t in ["required", "must", "inventory", "project", "playbook"]) + has_recommended = any(t in c for t in ["recommended", "warn", "variable", "limit"]) + assert has_required or has_recommended, ( + "should distinguish required vs recommended checks (skill: Phase 2 vs 3)" + ) diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/Dockerfile b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/Dockerfile new file mode 100644 index 00000000..d5c9e7b7 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/Dockerfile @@ -0,0 +1,56 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + + + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + }, \ + "aap-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-aap-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..1092dd1d --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- pass rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/mcp-servers/mock-aap-mcp.py b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/mcp-servers/mock-aap-mcp.py new file mode 100644 index 00000000..d8ae4fd5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/mcp-servers/mock-aap-mcp.py @@ -0,0 +1,1048 @@ +#!/usr/bin/env python3 +""" +Mock AAP (Ansible Automation Platform) MCP Server + +Simulates the AAP MCP gateway for per-skill evaluation tasks. Implements +the full set of tools used by rh-sre skills: + - job_templates_list / job_templates_retrieve + - projects_list + - job_templates_launch_retrieve + - jobs_retrieve / jobs_stdout_retrieve + - jobs_job_events_list / jobs_job_host_summaries_list + - jobs_relaunch_retrieve + - inventories_list / hosts_list + +Data mirrors a realistic AAP deployment: + - 6 job templates (3 remediation, 1 compliance, 1 patching, 1 reporting) + - 3 projects (remediation, compliance, reporting) + - 3 inventories (production 30 hosts, staging 15 hosts, all-managed 63 hosts) + - 12 recent jobs with varied statuses + +Follows the same mock-server pattern as mock-lightspeed-mcp.py. +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +mcp = FastMCP("aap-mcp") + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +def _ts(delta: timedelta) -> str: + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +# --------------------------------------------------------------------------- +# Mock data: Projects +# --------------------------------------------------------------------------- + +MOCK_PROJECTS = [ + { + "id": 6, + "type": "project", + "name": "Remediation Playbooks", + "description": "CVE and security remediation playbooks managed via Git", + "scm_type": "git", + "scm_url": "https://github.com/org/remediation-playbooks.git", + "scm_branch": "main", + "scm_revision": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + "status": "successful", + "last_job_run": _ts(timedelta(hours=2)), + "last_update_failed": False, + "created": _ts(timedelta(days=90)), + "modified": _ts(timedelta(hours=2)), + }, + { + "id": 7, + "type": "project", + "name": "Compliance Checks", + "description": "STIG and CIS compliance scanning playbooks", + "scm_type": "git", + "scm_url": "https://github.com/org/compliance-playbooks.git", + "scm_branch": "main", + "scm_revision": "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3", + "status": "successful", + "last_job_run": _ts(timedelta(days=1)), + "last_update_failed": False, + "created": _ts(timedelta(days=120)), + "modified": _ts(timedelta(days=1)), + }, + { + "id": 8, + "type": "project", + "name": "Fleet Reporting", + "description": "System inventory and health reporting playbooks", + "scm_type": "git", + "scm_url": "https://github.com/org/fleet-reports.git", + "scm_branch": "main", + "scm_revision": "c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "status": "successful", + "last_job_run": _ts(timedelta(days=3)), + "last_update_failed": False, + "created": _ts(timedelta(days=180)), + "modified": _ts(timedelta(days=3)), + }, +] + +# --------------------------------------------------------------------------- +# Mock data: Inventories & Hosts +# --------------------------------------------------------------------------- + +MOCK_INVENTORIES = [ + { + "id": 1, + "type": "inventory", + "name": "Production Systems", + "description": "All production RHEL systems across data centers", + "total_hosts": 30, + "has_active_failures": False, + "hosts_with_active_failures": 0, + "total_groups": 5, + "groups_with_active_failures": 0, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=365)), + "modified": _ts(timedelta(days=1)), + }, + { + "id": 2, + "type": "inventory", + "name": "Staging Systems", + "description": "Pre-production staging environment", + "total_hosts": 15, + "has_active_failures": False, + "hosts_with_active_failures": 0, + "total_groups": 3, + "groups_with_active_failures": 0, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=300)), + "modified": _ts(timedelta(days=7)), + }, + { + "id": 3, + "type": "inventory", + "name": "All Managed Systems", + "description": "Complete fleet: production, staging, development, QA, legacy", + "total_hosts": 63, + "has_active_failures": True, + "hosts_with_active_failures": 2, + "total_groups": 8, + "groups_with_active_failures": 1, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=365)), + "modified": _ts(timedelta(hours=6)), + }, +] + + +def _generate_hosts(inventory_id: int) -> list[dict]: + """Generate realistic hosts for an inventory.""" + hosts: list[dict] = [] + if inventory_id == 1: + roles = ["web", "db", "app", "lb", "monitoring", "cache"] + for i, role in enumerate(roles): + for j in range(5 if role in ("web", "app") else 4 if role == "db" else 3 if role == "monitoring" else 2): + hosts.append({ + "id": len(hosts) + 1, + "type": "host", + "name": f"{role}-{j+1:02d}.prod.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "production", "role": "{role}"}}', + }) + if len(hosts) >= 30: + break + if len(hosts) >= 30: + break + elif inventory_id == 2: + for i in range(15): + role = ["web", "db", "app"][i % 3] + hosts.append({ + "id": 100 + i, + "type": "host", + "name": f"{role}-{i+1:02d}.staging.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "staging", "role": "{role}"}}', + }) + elif inventory_id == 3: + for i in range(30): + hosts.append({ + "id": 200 + i, + "type": "host", + "name": f"host-{i+1:02d}.prod.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": i in (45, 58), + "variables": f'{{"rhel_version": "9.3", "environment": "production"}}', + }) + for i in range(15): + hosts.append({ + "id": 230 + i, + "type": "host", + "name": f"host-{i+1:02d}.staging.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "staging"}}', + }) + for i in range(10): + hosts.append({ + "id": 245 + i, + "type": "host", + "name": f"dev-{i+1:02d}.dev.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "8.9", "environment": "development"}}', + }) + for i in range(5): + hosts.append({ + "id": 255 + i, + "type": "host", + "name": f"qa-{i+1:02d}.qa.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.2", "environment": "qa"}}', + }) + for i in range(3): + hosts.append({ + "id": 260 + i, + "type": "host", + "name": f"legacy-{i+1:02d}.corp.example.com", + "inventory": inventory_id, + "enabled": i < 2, + "has_active_failures": i == 2, + "variables": f'{{"rhel_version": "7.9", "environment": "legacy"}}', + }) + return hosts + + +# --------------------------------------------------------------------------- +# Mock data: Job Templates +# --------------------------------------------------------------------------- + +MOCK_JOB_TEMPLATES = [ + { + "id": 10, + "type": "job_template", + "name": "CVE Remediation - Kernel Update", + "description": "Kernel update with boom snapshot for rollback safety", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": True, + "job_type": "check", + "verbosity": 1, + "timeout": 3600, + "forks": 5, + "status": "successful", + "last_job_run": _ts(timedelta(hours=4)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1001, "status": "successful", "finished": _ts(timedelta(hours=4))}, + }, + "created": _ts(timedelta(days=60)), + "modified": _ts(timedelta(days=2)), + }, + { + "id": 11, + "type": "job_template", + "name": "CVE Remediation - Package Update", + "description": "General package update for CVE remediation with needs-restarting check", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-package-update.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": False, + "job_type": "check", + "verbosity": 1, + "timeout": 1800, + "forks": 10, + "status": "successful", + "last_job_run": _ts(timedelta(hours=12)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1005, "status": "successful", "finished": _ts(timedelta(hours=12))}, + }, + "created": _ts(timedelta(days=45)), + "modified": _ts(timedelta(days=5)), + }, + { + "id": 12, + "type": "job_template", + "name": "CVE Remediation - Generic", + "description": "Generic CVE remediation template for ad-hoc patches", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-remediation.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": True, + "job_type": "check", + "verbosity": 1, + "timeout": 3600, + "forks": 5, + "status": "never updated", + "last_job_run": None, + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + }, + "created": _ts(timedelta(days=30)), + "modified": _ts(timedelta(days=30)), + }, + { + "id": 20, + "type": "job_template", + "name": "Compliance Check - STIG", + "description": "Run STIG compliance scan across fleet", + "inventory": 3, + "project": 7, + "playbook": "playbooks/compliance/check-all.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": False, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 7200, + "forks": 20, + "status": "successful", + "last_job_run": _ts(timedelta(days=1)), + "summary_fields": { + "project": {"id": 7, "name": "Compliance Checks", "status": "successful"}, + "inventory": {"id": 3, "name": "All Managed Systems", "total_hosts": 63}, + "credentials": [ + {"id": 2, "name": "compliance-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1010, "status": "successful", "finished": _ts(timedelta(days=1))}, + }, + "created": _ts(timedelta(days=180)), + "modified": _ts(timedelta(days=14)), + }, + { + "id": 25, + "type": "job_template", + "name": "Emergency Patching", + "description": "Emergency patch application — NO become enabled (misconfigured)", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/emergency-patch.yml", + "become_enabled": False, + "ask_job_type_on_launch": False, + "ask_variables_on_launch": False, + "ask_limit_on_launch": False, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 600, + "forks": 25, + "status": "failed", + "last_job_run": _ts(timedelta(days=7)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1020, "status": "failed", "finished": _ts(timedelta(days=7))}, + }, + "created": _ts(timedelta(days=200)), + "modified": _ts(timedelta(days=200)), + }, + { + "id": 30, + "type": "job_template", + "name": "Fleet Health Report", + "description": "Generate fleet health and inventory report", + "inventory": 3, + "project": 8, + "playbook": "playbooks/reporting/fleet-health.yml", + "become_enabled": False, + "ask_job_type_on_launch": False, + "ask_variables_on_launch": True, + "ask_limit_on_launch": False, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 1800, + "forks": 30, + "status": "successful", + "last_job_run": _ts(timedelta(hours=6)), + "summary_fields": { + "project": {"id": 8, "name": "Fleet Reporting", "status": "successful"}, + "inventory": {"id": 3, "name": "All Managed Systems", "total_hosts": 63}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1025, "status": "successful", "finished": _ts(timedelta(hours=6))}, + }, + "created": _ts(timedelta(days=120)), + "modified": _ts(timedelta(days=14)), + }, +] + +# --------------------------------------------------------------------------- +# Mock data: Jobs (recent runs) +# --------------------------------------------------------------------------- + +PROD_HOSTS = [ + "web-01.prod.example.com", + "web-02.prod.example.com", + "db-01.prod.example.com", + "db-02.prod.example.com", + "app-01.prod.example.com", + "app-02.prod.example.com", +] + +MOCK_JOBS = [ + { + "id": 1001, + "type": "job", + "name": "CVE Remediation - Kernel Update", + "job_type": "check", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=4, minutes=30)), + "finished": _ts(timedelta(hours=4)), + "elapsed": 1800.0, + "job_template": 10, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "limit": "web-01.prod.example.com,web-02.prod.example.com,db-01.prod.example.com", + "extra_vars": '{"target_cve": "CVE-2024-12345", "remediation_mode": "automated", "verify_after": true}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 10, "name": "CVE Remediation - Kernel Update"}, + }, + }, + { + "id": 1002, + "type": "job", + "name": "CVE Remediation - Kernel Update", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=3, minutes=45)), + "finished": _ts(timedelta(hours=3)), + "elapsed": 2700.0, + "job_template": 10, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "limit": "web-01.prod.example.com,web-02.prod.example.com,db-01.prod.example.com", + "extra_vars": '{"target_cve": "CVE-2024-12345", "remediation_mode": "automated", "verify_after": true}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 10, "name": "CVE Remediation - Kernel Update"}, + }, + }, + { + "id": 1005, + "type": "job", + "name": "CVE Remediation - Package Update", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=12, minutes=20)), + "finished": _ts(timedelta(hours=12)), + "elapsed": 1200.0, + "job_template": 11, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-package-update.yml", + "limit": "", + "extra_vars": '{"target_cve": "CVE-2024-54321"}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 11, "name": "CVE Remediation - Package Update"}, + }, + }, + { + "id": 1010, + "type": "job", + "name": "Compliance Check - STIG", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(days=1, hours=2)), + "finished": _ts(timedelta(days=1)), + "elapsed": 7200.0, + "job_template": 20, + "inventory": 3, + "project": 7, + "playbook": "playbooks/compliance/check-all.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "scheduled", + "summary_fields": { + "job_template": {"id": 20, "name": "Compliance Check - STIG"}, + }, + }, + { + "id": 1020, + "type": "job", + "name": "Emergency Patching", + "job_type": "run", + "status": "failed", + "failed": True, + "started": _ts(timedelta(days=7, hours=1)), + "finished": _ts(timedelta(days=7)), + "elapsed": 3600.0, + "job_template": 25, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/emergency-patch.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 25, "name": "Emergency Patching"}, + }, + }, + { + "id": 1025, + "type": "job", + "name": "Fleet Health Report", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=6, minutes=30)), + "finished": _ts(timedelta(hours=6)), + "elapsed": 1800.0, + "job_template": 30, + "inventory": 3, + "project": 8, + "playbook": "playbooks/reporting/fleet-health.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "scheduled", + "summary_fields": { + "job_template": {"id": 30, "name": "Fleet Health Report"}, + }, + }, +] + +_next_job_id = 2000 + + +# --------------------------------------------------------------------------- +# Mock stdout generators +# --------------------------------------------------------------------------- + +def _generate_stdout(job: dict) -> str: + """Generate realistic Ansible playbook stdout for a job.""" + playbook_name = job.get("name", "Unknown") + job_type = job.get("job_type", "run") + status = job.get("status", "successful") + limit = job.get("limit", "") + hosts = limit.split(",") if limit else PROD_HOSTS[:3] + hosts = [h.strip() for h in hosts if h.strip()] + extra_vars = job.get("extra_vars", "{}") + mode = " (CHECK MODE)" if job_type == "check" else "" + + lines = [] + lines.append(f"PLAY [{playbook_name}] *****") + lines.append("") + + lines.append(f"TASK [Gathering Facts{mode}] *****") + for h in hosts: + lines.append(f"ok: [{h}]") + lines.append("") + + if "kernel" in playbook_name.lower(): + lines.append(f"TASK [Create boom snapshot for rollback{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}] => {{\"msg\": \"boom create --title pre-remediation-CVE-2024-12345\"}}") + lines.append("") + + lines.append(f"TASK [Check disk space for kernel update{mode}] *****") + for h in hosts: + lines.append(f"ok: [{h}] => {{\"msg\": \"Disk space OK: 45% used\"}}") + lines.append("") + + lines.append(f"TASK [Update kernel package{mode}] *****") + for h in hosts: + result = "changed" if status == "successful" else "fatal" + if result == "changed": + lines.append(f'changed: [{h}] => {{"msg": "kernel-5.14.0-362.24.1.el9_3 -> kernel-5.14.0-362.24.2.el9_3"}}') + else: + lines.append(f'fatal: [{h}]: FAILED! => {{"msg": "Permission denied", "rc": 1}}') + lines.append("") + + lines.append(f"TASK [Check if reboot is needed (needs-restarting -r){mode}] *****") + for h in hosts: + lines.append(f'changed: [{h}] => {{"rc": 1, "msg": "Reboot is required to fully utilize updates."}}') + lines.append("") + + elif "package" in playbook_name.lower(): + lines.append(f"TASK [Update target packages for CVE remediation{mode}] *****") + for h in hosts: + lines.append(f'changed: [{h}] => {{"msg": "httpd-2.4.53-7.el9 -> httpd-2.4.57-8.el9"}}') + lines.append("") + + lines.append(f"TASK [Restart affected services{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}]") + lines.append("") + + lines.append(f"TASK [Verify service health{mode}] *****") + for h in hosts: + lines.append(f'ok: [{h}] => {{"msg": "Service httpd is running"}}') + lines.append("") + + elif "emergency" in playbook_name.lower() and status == "failed": + lines.append(f"TASK [Apply emergency patch{mode}] *****") + for h in hosts: + lines.append(f'fatal: [{h}]: FAILED! => {{"msg": "Missing sudo password (become_enabled not set)", "rc": 1}}') + lines.append("") + lines.append("NO MORE HOSTS LEFT *****") + lines.append("") + + else: + lines.append(f"TASK [Execute playbook tasks{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}]") + lines.append("") + + lines.append("PLAY RECAP *****") + for h in hosts: + if status == "successful": + ok_count = random.randint(3, 6) + changed_count = random.randint(1, 3) + lines.append(f"{h:<45} : ok={ok_count} changed={changed_count} unreachable=0 failed=0 skipped=0 rescued=0 ignored=0") + elif status == "failed": + lines.append(f"{h:<45} : ok=1 changed=0 unreachable=0 failed=1 skipped=0 rescued=0 ignored=0") + lines.append("") + + return "\n".join(lines) + + +def _generate_events(job: dict) -> list[dict]: + """Generate realistic Ansible task events for a job.""" + hosts = (job.get("limit", "").split(",") if job.get("limit") else PROD_HOSTS[:3]) + hosts = [h.strip() for h in hosts if h.strip()] + events: list[dict] = [] + eid = 1 + + task_names = ["Gathering Facts"] + if "kernel" in job.get("name", "").lower(): + task_names += [ + "Create boom snapshot for rollback", + "Check disk space for kernel update", + "Update kernel package", + "Check if reboot is needed (needs-restarting -r)", + ] + elif "package" in job.get("name", "").lower(): + task_names += [ + "Update target packages for CVE remediation", + "Restart affected services", + "Verify service health", + ] + else: + task_names += ["Execute playbook tasks"] + + for task_name in task_names: + for host in hosts: + is_failed = job.get("status") == "failed" and task_name != "Gathering Facts" + events.append({ + "id": eid, + "type": "job_event", + "event": "runner_on_ok" if not is_failed else "runner_on_failed", + "task": task_name, + "host": host, + "host_name": host, + "play": job.get("name", ""), + "changed": task_name != "Gathering Facts" and not is_failed, + "failed": is_failed, + "event_data": { + "task": task_name, + "host": host, + "res": { + "changed": task_name != "Gathering Facts" and not is_failed, + "msg": "Task completed" if not is_failed else "Permission denied", + }, + }, + "created": _ts(timedelta(hours=4, minutes=30 - eid)), + }) + eid += 1 + + return events + + +def _generate_host_summaries(job: dict) -> list[dict]: + """Generate per-host summaries for a job.""" + hosts = (job.get("limit", "").split(",") if job.get("limit") else PROD_HOSTS[:3]) + hosts = [h.strip() for h in hosts if h.strip()] + summaries: list[dict] = [] + + for i, host in enumerate(hosts): + is_failed = job.get("status") == "failed" + summaries.append({ + "id": i + 1, + "type": "job_host_summary", + "host": i + 1, + "host_name": host, + "ok": 1 if is_failed else random.randint(3, 6), + "changed": 0 if is_failed else random.randint(1, 3), + "dark": 0, + "failures": 1 if is_failed else 0, + "skipped": 0, + "processed": 1, + "failed": is_failed, + }) + + return summaries + + +# --------------------------------------------------------------------------- +# MCP Tools: Job Management +# --------------------------------------------------------------------------- + +@mcp.tool() +def job_templates_list( + page_size: int = 10, + search: Optional[str] = None, +) -> dict: + """List available job templates in AAP. + + Args: + page_size: Number of results per page (default 10, max 200). + search: Optional search string to filter templates by name. + """ + results = MOCK_JOB_TEMPLATES + if search: + s = search.lower() + results = [t for t in results if s in t["name"].lower() or s in t.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def job_templates_retrieve(id: str) -> dict: + """Retrieve detailed information about a specific job template. + + Args: + id: Job template ID (as string). + """ + tid = int(id) + template = next((t for t in MOCK_JOB_TEMPLATES if t["id"] == tid), None) + if not template: + return {"detail": f"Not found. Job template {id} does not exist."} + return template + + +@mcp.tool() +def projects_list( + page_size: int = 50, + search: Optional[str] = None, +) -> dict: + """List available projects in AAP. + + Args: + page_size: Number of results per page. + search: Optional search string to filter projects by name. + """ + results = MOCK_PROJECTS + if search: + s = search.lower() + results = [p for p in results if s in p["name"].lower() or s in p.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def job_templates_launch_retrieve( + id: str, + requestBody: Optional[dict] = None, +) -> dict: + """Launch a job from a job template. + + Args: + id: Job template ID to launch. + requestBody: Optional launch parameters including job_type ('run' or 'check'), + extra_vars (dict), and limit (comma-separated host list). + """ + global _next_job_id + tid = int(id) + template = next((t for t in MOCK_JOB_TEMPLATES if t["id"] == tid), None) + if not template: + return {"detail": f"Not found. Job template {id} does not exist."} + + body = requestBody or {} + job_type = body.get("job_type", template.get("job_type", "run")) + + if not template.get("ask_job_type_on_launch") and job_type != template.get("job_type"): + return { + "error": f"Cannot override job_type: ask_job_type_on_launch is disabled on template {id}", + } + + job_id = _next_job_id + _next_job_id += 1 + + new_job = { + "id": job_id, + "type": "job", + "name": template["name"], + "job_type": job_type, + "status": "pending", + "failed": False, + "started": _ts(timedelta(seconds=0)), + "finished": None, + "elapsed": 0.0, + "job_template": tid, + "inventory": template["inventory"], + "project": template["project"], + "playbook": template["playbook"], + "limit": body.get("limit", ""), + "extra_vars": str(body.get("extra_vars", {})), + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": tid, "name": template["name"]}, + }, + } + MOCK_JOBS.append(new_job) + + # Simulate job completion after launch + new_job["status"] = "successful" + new_job["finished"] = _ts(timedelta(seconds=-300)) + new_job["elapsed"] = 300.0 + + return { + "job": job_id, + "status": "pending", + "type": "job", + "url": f"/api/controller/v2/jobs/{job_id}/", + "related": { + "stdout": f"/api/controller/v2/jobs/{job_id}/stdout/", + "job_events": f"/api/controller/v2/jobs/{job_id}/job_events/", + "job_host_summaries": f"/api/controller/v2/jobs/{job_id}/job_host_summaries/", + }, + } + + +@mcp.tool() +def jobs_retrieve(id: int) -> dict: + """Get the status and details of a job run. + + Args: + id: Job ID to retrieve. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + return job + + +@mcp.tool() +def jobs_list(page_size: int = 10) -> dict: + """List recent job runs. + + Args: + page_size: Number of results to return. + """ + results = sorted(MOCK_JOBS, key=lambda j: j.get("started", ""), reverse=True) + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def jobs_stdout_retrieve(id: int, format: str = "txt") -> dict: + """Get the stdout (console output) from a job run. + + Args: + id: Job ID. + format: Output format ('txt' or 'json'). Default 'txt'. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + return { + "content": _generate_stdout(job), + "range": {"start": 0, "end": 1}, + } + + +@mcp.tool() +def jobs_job_events_list(id: int, page_size: int = 50) -> dict: + """Get task-level events for a job run. + + Args: + id: Job ID. + page_size: Number of events to return. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + events = _generate_events(job) + return { + "count": len(events), + "next": None, + "previous": None, + "results": events[:page_size], + } + + +@mcp.tool() +def jobs_job_host_summaries_list(id: int) -> dict: + """Get per-host execution summaries for a job run. + + Args: + id: Job ID. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + summaries = _generate_host_summaries(job) + return { + "count": len(summaries), + "next": None, + "previous": None, + "results": summaries, + } + + +@mcp.tool() +def jobs_relaunch_retrieve( + id: int, + hosts: str = "all", + job_type: str = "run", +) -> dict: + """Relaunch a previously completed or failed job. + + Args: + id: Original job ID to relaunch. + hosts: Which hosts to target ('all' or 'failed'). + job_type: Job type for relaunch ('run' or 'check'). + """ + global _next_job_id + original = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not original: + return {"detail": f"Not found. Job {id} does not exist."} + + new_id = _next_job_id + _next_job_id += 1 + + new_job = { + **original, + "id": new_id, + "job_type": job_type, + "status": "successful", + "failed": False, + "started": _ts(timedelta(seconds=0)), + "finished": _ts(timedelta(seconds=-300)), + "elapsed": 300.0, + "launch_type": "relaunch", + } + MOCK_JOBS.append(new_job) + + return { + "job": new_id, + "status": "pending", + "type": "job", + "url": f"/api/controller/v2/jobs/{new_id}/", + } + + +# --------------------------------------------------------------------------- +# MCP Tools: Inventory Management +# --------------------------------------------------------------------------- + +@mcp.tool() +def inventories_list( + page_size: int = 10, + search: Optional[str] = None, +) -> dict: + """List available inventories in AAP. + + Args: + page_size: Number of results per page. + search: Optional search string to filter inventories. + """ + results = MOCK_INVENTORIES + if search: + s = search.lower() + results = [inv for inv in results if s in inv["name"].lower() or s in inv.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def hosts_list( + inventory_id: Optional[int] = None, + page_size: int = 50, + search: Optional[str] = None, +) -> dict: + """List hosts in an inventory. + + Args: + inventory_id: Filter by inventory ID. If not provided, lists hosts from all inventories. + page_size: Number of results per page. + search: Optional search string to filter hosts by name. + """ + inv_id = inventory_id or 1 + hosts = _generate_hosts(inv_id) + if search: + s = search.lower() + hosts = [h for h in hosts if s in h["name"].lower()] + return { + "count": len(hosts), + "next": None if len(hosts) <= page_size else f"/api/controller/v2/hosts/?page=2", + "previous": None, + "results": hosts[:page_size], + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..fe5d549c --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/skills/mcp-aap-validator/SKILL.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/skills/mcp-aap-validator/SKILL.md new file mode 100644 index 00000000..a1c4f708 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/skills/mcp-aap-validator/SKILL.md @@ -0,0 +1,66 @@ +--- +name: mcp-aap-validator +description: | + Validate AAP (Ansible Automation Platform) MCP server connectivity. Use when the user asks to "validate AAP MCP", "check AAP connection", or when other skills need to verify AAP MCP availability before job management or inventory operations. +model: haiku +color: yellow +--- + +# MCP AAP Validator + +Validates connectivity to AAP MCP servers by running lightweight tool calls. + +## When to Use This Skill + +Use when validating AAP MCP before job template operations, troubleshooting connection issues, or when other skills (e.g. playbook-executor) need to verify availability. Do NOT use for creating templates—use job-template-creator. + +## Workflow + +1. **Test connectivity**: Call these tools to verify each server responds: + - `job_templates_list` (page_size: 10) from aap-mcp-job-management + - `inventories_list` (page_size: 10) from aap-mcp-inventory-management +2. **If any fails**: Provide a comprehensive message with possible root causes (see below). +3. **Report**: Output a table with validated servers and outcome (emojis). + +## Failure Message (Root Causes) + +When a tool call fails, include: + +``` +❌ AAP MCP connection failed + +**Possible root causes:** +- **Credentials**: AAP_MCP_SERVER or AAP_API_TOKEN not set or invalid +- **401 Unauthorized**: Token expired or invalid → regenerate in AAP Web UI +- **403 Forbidden**: Token lacks RBAC permissions (need Job Templates, Inventories) +- **404 Not Found**: Wrong AAP_MCP_SERVER URL (must point to MCP gateway, not main AAP UI) +- **Connection timeout**: Server unreachable, firewall, or network issue +- **SSL/TLS error**: Certificate verification problem + +**Troubleshooting:** +1. Verify env vars: AAP_MCP_SERVER, AAP_API_TOKEN (never echo values) +2. Get token: AAP Web UI → Users → [Your User] → Tokens → Create +3. Ensure AAP_MCP_SERVER points to MCP gateway endpoint +4. Restart host after config changes +``` + +## Report Format + +Always end with a table: + +| Server | Outcome | +|--------|---------| +| aap-mcp-job-management | ✅ PASSED | +| aap-mcp-inventory-management | ✅ PASSED | + +Use ✅ for success, ❌ for failure, ⚠️ for partial (e.g. one server OK, one failed). + +## Dependencies + +### Required MCP Tools +- `job_templates_list` (from aap-mcp-job-management) - Connectivity test +- `inventories_list` (from aap-mcp-inventory-management) - Connectivity test + +### Required MCP Servers +- `aap-mcp-job-management` - AAP job template and execution +- `aap-mcp-inventory-management` - AAP inventory management diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/instruction.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/instruction.md new file mode 100644 index 00000000..54d1a0e6 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/instruction.md @@ -0,0 +1,16 @@ +# AAP Connectivity Check Task + +You are a Red Hat SRE. Before starting a remediation workflow that depends on Ansible Automation Platform, you need to verify that the AAP integration is working correctly. + +## Scenario +You are about to run a remediation workflow that uses AAP to execute playbooks. First, you need to confirm that the AAP connection is healthy and that you can access the necessary resources. + +## Requirements +- Test connectivity to the AAP server by querying job templates and inventories +- Verify that the response is valid and contains expected data +- If any connection fails, document the error and provide troubleshooting guidance (credentials, network, SSL, permissions) +- Report the overall AAP readiness status: which capabilities are available and which are not + +Document your methodology, connectivity check results, and troubleshooting findings in `/root/report.md`. + +Use MCP tools to interact with AAP. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/solution/solve.sh b/evaluation/with_skills/rh-sre__mcp-aap-validator/solution/solve.sh new file mode 100644 index 00000000..88542def --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/solution/solve.sh @@ -0,0 +1,25 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# AAP MCP Validation + +## Test Calls +- `job_templates_list(page_size: 10)` from aap-mcp-job-management ✅ +- `inventories_list(page_size: 10)` from aap-mcp-inventory-management ✅ + +## Result +| Server | Outcome | +|--------|---------| +| aap-mcp-job-management | ✅ PASSED | +| aap-mcp-inventory-management | ✅ PASSED | + +## Diagnostics +| Code | Meaning | +|------|---------| +| 401 | Token expired or invalid → regenerate in AAP Web UI → Users → Tokens | +| 403 | Insufficient RBAC (need Job Templates, Inventories) | +| 404 | Wrong URL — AAP_MCP_SERVER must point to MCP gateway, not main AAP UI | + +## Environment +- AAP_MCP_SERVER: Set (must point to MCP gateway) +- AAP_API_TOKEN: Set +REPORT_EOF diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/task.toml b/evaluation/with_skills/rh-sre__mcp-aap-validator/task.toml new file mode 100644 index 00000000..aad389ea --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__mcp-aap-validator" +name = "rh-sre AAP MCP Validation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "mcp-aap-validator", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/tests/llm_judge.py b/evaluation/with_skills/rh-sre__mcp-aap-validator/tests/llm_judge.py new file mode 100644 index 00000000..474598a6 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "gateway_vs_ui_url", "file": "/root/report.md", "question": "Does the report note that AAP_MCP_SERVER must point to the MCP gateway endpoint, not the main AAP UI URL, and that 404 indicates wrong URL?", "reference": "A skilled report explains the gateway/UI URL distinction and maps 404 to wrong URL. An unskilled report doesn't distinguish these endpoints."}, + {"id": "both_servers_tested", "file": "/root/report.md", "question": "Does the report test both job_templates_list and inventories_list for AAP MCP validation?", "reference": "A skilled report validates both MCP servers. An unskilled report tests only one."}, + {"id": "structured_outcome", "file": "/root/report.md", "question": "Does the report present per-server validation outcomes (PASSED/FAILED/PARTIAL) in table format?", "reference": "A skilled report uses structured table with per-server status. An unskilled report uses unstructured text."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/tests/test.sh b/evaluation/with_skills/rh-sre__mcp-aap-validator/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/tests/test_outputs.py b/evaluation/with_skills/rh-sre__mcp-aap-validator/tests/test_outputs.py new file mode 100644 index 00000000..615713b5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-aap-validator/tests/test_outputs.py @@ -0,0 +1,66 @@ +""" +Tests for rh-sre__mcp-aap-validator per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['aap', 'mcp', 'valid', 'connect']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_both_servers_tested(self): + """Skill: Test BOTH job_templates_list (job-management) AND inventories_list (inventory-management).""" + c = read_report().lower() + has_job = any(t in c for t in ["job_template", "job template", "job-management"]) + has_inv = any(t in c for t in ["inventor", "inventory-management"]) + assert has_job or has_inv, ( + "should test both AAP MCP servers (skill: job-management + inventory-management)" + ) + + def test_mcp_gateway_not_ui(self): + """Skill teaches AAP_MCP_SERVER must point to MCP gateway endpoint, not main AAP UI URL.""" + c = read_report().lower() + assert ("gateway" in c and "mcp" in c) or "aap_mcp_server" in c, ( + "should note AAP_MCP_SERVER must point to MCP gateway, not UI (skill: wrong URL = 404)" + ) + + def test_404_wrong_url(self): + """Skill teaches HTTP 404 = wrong AAP_MCP_SERVER URL.""" + c = read_report().lower() + assert "404" in c and any(t in c for t in ["url", "wrong"]), ( + "should explain 404 indicates wrong URL (skill: troubleshooting)" + ) + + def test_table_format(self): + """Skill: Output table with Server | Outcome (PASSED/FAILED/PARTIAL).""" + content = read_report() + c = content.lower() + has_table = "|" in content + has_outcome = any(t in c for t in ["passed", "failed", "partial", "job_templates_list", "inventories_list"]) + assert has_table or has_outcome, ( + "should use table format with outcome (skill: Report Format)" + ) diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/Dockerfile b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/Dockerfile new file mode 100644 index 00000000..484ebb33 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/Dockerfile @@ -0,0 +1,52 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + + + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..1092dd1d --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- pass rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..fe5d549c --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/skills/mcp-lightspeed-validator/SKILL.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/skills/mcp-lightspeed-validator/SKILL.md new file mode 100644 index 00000000..e1f1528e --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/skills/mcp-lightspeed-validator/SKILL.md @@ -0,0 +1,61 @@ +--- +name: mcp-lightspeed-validator +description: | + Validate Red Hat Lightspeed MCP server connectivity. Use when the user asks to "validate Lightspeed MCP", "check Lightspeed connection", or when other skills need to verify lightspeed-mcp availability before CVE operations. +model: haiku +color: yellow +--- + +# MCP Lightspeed Validator + +Validates connectivity to the Red Hat Lightspeed MCP server by running a lightweight tool call. + +## When to Use This Skill + +Use when validating Lightspeed MCP before CVE operations, troubleshooting connection issues, or when other skills (e.g. remediation) need to verify availability. Do NOT use for actual CVE queries—use cve-impact or cve-validation. + +## Workflow + +1. **Test connectivity**: Call `vulnerability__get_cves` with **no parameters** (uses default limit=10). Do NOT pass `limit`—some MCP clients incorrectly serialize it as `limit_`, causing validation errors. +2. **If it fails**: Provide a comprehensive message with possible root causes (see below). +3. **Report**: Output a table with validated servers and outcome (emojis). + +## Failure Message (Root Causes) + +When the tool call fails, include: + +``` +❌ Lightspeed MCP connection failed + +**Possible root causes:** +- **Credentials**: LIGHTSPEED_CLIENT_ID or LIGHTSPEED_CLIENT_SECRET not set or invalid +- **Expired credentials**: Red Hat Console tokens may have expired +- **Server not running**: MCP server/container may be stopped +- **Network**: Firewall or proxy blocking console.redhat.com +- **Configuration**: .mcp.json misconfigured or server not registered + +**Troubleshooting:** +1. Verify env vars: LIGHTSPEED_CLIENT_ID, LIGHTSPEED_CLIENT_SECRET (never echo values) +2. Check credentials at: https://console.redhat.com/settings/integrations +3. Restart MCP server or host after config changes +4. Check container logs if using podman/docker +``` + +## Report Format + +Always end with a table: + +| Server | Outcome | +|--------|---------| +| lightspeed-mcp | ✅ PASSED | +| lightspeed-mcp | ❌ FAILED | + +Use ✅ for success, ❌ for failure, ⚠️ for partial (e.g. connected but error on tool). + +## Dependencies + +### Required MCP Tools +- `vulnerability__get_cves` or `get_cves` (from lightspeed-mcp) - Connectivity test + +### Required MCP Servers +- `lightspeed-mcp` - Red Hat Lightspeed vulnerability and inventory data diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/instruction.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/instruction.md new file mode 100644 index 00000000..37d450b8 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/instruction.md @@ -0,0 +1,16 @@ +# Lightspeed Connectivity Check Task + +You are a Red Hat SRE. Before querying CVE data or generating remediation playbooks, you need to verify that the Red Hat Insights/Lightspeed integration is working correctly. + +## Scenario +You are about to start a CVE investigation that depends on querying vulnerability data from Red Hat Insights. First, you need to confirm the Lightspeed connection is healthy and returning valid data. + +## Requirements +- Test connectivity to the Lightspeed service by querying CVE data +- Verify the response is valid and contains expected vulnerability information +- If the connection fails, document the error and provide troubleshooting guidance (expired tokens, credentials, network issues, server availability) +- Report the overall Lightspeed readiness status + +Document your methodology, connectivity check results, and troubleshooting findings in `/root/report.md`. + +Use MCP tools to interact with the Lightspeed service. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/solution/solve.sh b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/solution/solve.sh new file mode 100644 index 00000000..8336f1ee --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/solution/solve.sh @@ -0,0 +1,29 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Lightspeed MCP Validation + +## Test: Call vulnerability__get_cves with no parameters +- Do NOT pass `limit` parameter (serialization issue: `limit` → `limit_`) +- Default limit=10 is applied automatically + +## Result +| Server | Outcome | +|--------|---------| +| lightspeed-mcp | ✅ PASSED | + +## Failure Root Causes (when connection fails) +- **Credentials**: LIGHTSPEED_CLIENT_ID or LIGHTSPEED_CLIENT_SECRET not set or invalid +- **Expired credentials**: Red Hat Console tokens may have expired +- **Server not running**: MCP server/container may be stopped +- **Network**: Firewall or proxy blocking console.redhat.com +- **Configuration**: .mcp.json misconfigured or server not registered + +## Troubleshooting +1. Verify env vars: LIGHTSPEED_CLIENT_ID, LIGHTSPEED_CLIENT_SECRET (never echo values) +2. Check credentials at: https://console.redhat.com/settings/integrations +3. Restart MCP server or host after config changes + +## Environment +- LIGHTSPEED_CLIENT_ID: Set +- LIGHTSPEED_CLIENT_SECRET: Set +REPORT_EOF diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/task.toml b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/task.toml new file mode 100644 index 00000000..1e356701 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__mcp-lightspeed-validator" +name = "rh-sre Lightspeed MCP Validation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "mcp-lightspeed-validator", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/tests/llm_judge.py b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/tests/llm_judge.py new file mode 100644 index 00000000..905e9250 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "no_params_get_cves", "file": "/root/report.md", "question": "Does the report call get_cves with no parameters (due to limit_ serialization bug)?", "reference": "A skilled report avoids passing limit parameter. An unskilled report passes limit which may break the call."}, + {"id": "credential_handling", "file": "/root/report.md", "question": "Does the report reference LIGHTSPEED_CLIENT_ID/CLIENT_SECRET env vars and warn against echoing credentials?", "reference": "A skilled report identifies the correct env vars and warns about credential exposure. An unskilled report doesn't know the specific variable names."}, + {"id": "validation_structure", "file": "/root/report.md", "question": "Does the report present Lightspeed MCP validation in structured table format?", "reference": "A skilled report uses table with PASSED/FAILED outcome. An unskilled report uses unstructured text."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/tests/test.sh b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/tests/test_outputs.py b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/tests/test_outputs.py new file mode 100644 index 00000000..05e6bf9b --- /dev/null +++ b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/tests/test_outputs.py @@ -0,0 +1,64 @@ +""" +Tests for rh-sre__mcp-lightspeed-validator per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['lightspeed', 'mcp', 'valid']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_get_cves_no_params(self): + """Skill: Call vulnerability__get_cves with NO parameters (limit causes limit_ serialization bug).""" + c = read_report().lower() + assert any(t in c for t in ["no param", "without param", "limit_"]), ( + "should call get_cves without parameters (skill: passing limit breaks some clients)" + ) + + def test_lightspeed_credentials(self): + """Skill: LIGHTSPEED_CLIENT_ID + LIGHTSPEED_CLIENT_SECRET are the env vars.""" + c = read_report().lower() + assert any(t in c for t in ["lightspeed_client_id", "client_id", "client_secret"]), ( + "should reference Lightspeed credential env vars (skill: LIGHTSPEED_CLIENT_ID/SECRET)" + ) + + def test_never_echo_credentials(self): + """Skill: Never echo or log credential values.""" + c = read_report().lower() + has_security = any(t in c for t in ["never echo", "do not echo", "redact", "sensitive", "protect"]) + assert has_security or "credential" in c, ( + "should address credential handling (skill: never echo values)" + ) + + def test_table_format(self): + """Skill: Output table with Server | Outcome.""" + c = read_report().lower() + has_table = "|" in read_report() + has_outcome = any(t in c for t in ["passed", "failed", "get_cves", "lightspeed"]) + assert has_table or has_outcome, ( + "should use table format (skill: Report Format)" + ) diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/Dockerfile b/evaluation/with_skills/rh-sre__playbook-executor/environment/Dockerfile new file mode 100644 index 00000000..d5c9e7b7 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/Dockerfile @@ -0,0 +1,56 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + + + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + }, \ + "aap-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-aap-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..1092dd1d --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- pass rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/mcp-servers/mock-aap-mcp.py b/evaluation/with_skills/rh-sre__playbook-executor/environment/mcp-servers/mock-aap-mcp.py new file mode 100644 index 00000000..d8ae4fd5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/mcp-servers/mock-aap-mcp.py @@ -0,0 +1,1048 @@ +#!/usr/bin/env python3 +""" +Mock AAP (Ansible Automation Platform) MCP Server + +Simulates the AAP MCP gateway for per-skill evaluation tasks. Implements +the full set of tools used by rh-sre skills: + - job_templates_list / job_templates_retrieve + - projects_list + - job_templates_launch_retrieve + - jobs_retrieve / jobs_stdout_retrieve + - jobs_job_events_list / jobs_job_host_summaries_list + - jobs_relaunch_retrieve + - inventories_list / hosts_list + +Data mirrors a realistic AAP deployment: + - 6 job templates (3 remediation, 1 compliance, 1 patching, 1 reporting) + - 3 projects (remediation, compliance, reporting) + - 3 inventories (production 30 hosts, staging 15 hosts, all-managed 63 hosts) + - 12 recent jobs with varied statuses + +Follows the same mock-server pattern as mock-lightspeed-mcp.py. +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +mcp = FastMCP("aap-mcp") + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +def _ts(delta: timedelta) -> str: + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +# --------------------------------------------------------------------------- +# Mock data: Projects +# --------------------------------------------------------------------------- + +MOCK_PROJECTS = [ + { + "id": 6, + "type": "project", + "name": "Remediation Playbooks", + "description": "CVE and security remediation playbooks managed via Git", + "scm_type": "git", + "scm_url": "https://github.com/org/remediation-playbooks.git", + "scm_branch": "main", + "scm_revision": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + "status": "successful", + "last_job_run": _ts(timedelta(hours=2)), + "last_update_failed": False, + "created": _ts(timedelta(days=90)), + "modified": _ts(timedelta(hours=2)), + }, + { + "id": 7, + "type": "project", + "name": "Compliance Checks", + "description": "STIG and CIS compliance scanning playbooks", + "scm_type": "git", + "scm_url": "https://github.com/org/compliance-playbooks.git", + "scm_branch": "main", + "scm_revision": "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3", + "status": "successful", + "last_job_run": _ts(timedelta(days=1)), + "last_update_failed": False, + "created": _ts(timedelta(days=120)), + "modified": _ts(timedelta(days=1)), + }, + { + "id": 8, + "type": "project", + "name": "Fleet Reporting", + "description": "System inventory and health reporting playbooks", + "scm_type": "git", + "scm_url": "https://github.com/org/fleet-reports.git", + "scm_branch": "main", + "scm_revision": "c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "status": "successful", + "last_job_run": _ts(timedelta(days=3)), + "last_update_failed": False, + "created": _ts(timedelta(days=180)), + "modified": _ts(timedelta(days=3)), + }, +] + +# --------------------------------------------------------------------------- +# Mock data: Inventories & Hosts +# --------------------------------------------------------------------------- + +MOCK_INVENTORIES = [ + { + "id": 1, + "type": "inventory", + "name": "Production Systems", + "description": "All production RHEL systems across data centers", + "total_hosts": 30, + "has_active_failures": False, + "hosts_with_active_failures": 0, + "total_groups": 5, + "groups_with_active_failures": 0, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=365)), + "modified": _ts(timedelta(days=1)), + }, + { + "id": 2, + "type": "inventory", + "name": "Staging Systems", + "description": "Pre-production staging environment", + "total_hosts": 15, + "has_active_failures": False, + "hosts_with_active_failures": 0, + "total_groups": 3, + "groups_with_active_failures": 0, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=300)), + "modified": _ts(timedelta(days=7)), + }, + { + "id": 3, + "type": "inventory", + "name": "All Managed Systems", + "description": "Complete fleet: production, staging, development, QA, legacy", + "total_hosts": 63, + "has_active_failures": True, + "hosts_with_active_failures": 2, + "total_groups": 8, + "groups_with_active_failures": 1, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=365)), + "modified": _ts(timedelta(hours=6)), + }, +] + + +def _generate_hosts(inventory_id: int) -> list[dict]: + """Generate realistic hosts for an inventory.""" + hosts: list[dict] = [] + if inventory_id == 1: + roles = ["web", "db", "app", "lb", "monitoring", "cache"] + for i, role in enumerate(roles): + for j in range(5 if role in ("web", "app") else 4 if role == "db" else 3 if role == "monitoring" else 2): + hosts.append({ + "id": len(hosts) + 1, + "type": "host", + "name": f"{role}-{j+1:02d}.prod.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "production", "role": "{role}"}}', + }) + if len(hosts) >= 30: + break + if len(hosts) >= 30: + break + elif inventory_id == 2: + for i in range(15): + role = ["web", "db", "app"][i % 3] + hosts.append({ + "id": 100 + i, + "type": "host", + "name": f"{role}-{i+1:02d}.staging.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "staging", "role": "{role}"}}', + }) + elif inventory_id == 3: + for i in range(30): + hosts.append({ + "id": 200 + i, + "type": "host", + "name": f"host-{i+1:02d}.prod.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": i in (45, 58), + "variables": f'{{"rhel_version": "9.3", "environment": "production"}}', + }) + for i in range(15): + hosts.append({ + "id": 230 + i, + "type": "host", + "name": f"host-{i+1:02d}.staging.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "staging"}}', + }) + for i in range(10): + hosts.append({ + "id": 245 + i, + "type": "host", + "name": f"dev-{i+1:02d}.dev.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "8.9", "environment": "development"}}', + }) + for i in range(5): + hosts.append({ + "id": 255 + i, + "type": "host", + "name": f"qa-{i+1:02d}.qa.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.2", "environment": "qa"}}', + }) + for i in range(3): + hosts.append({ + "id": 260 + i, + "type": "host", + "name": f"legacy-{i+1:02d}.corp.example.com", + "inventory": inventory_id, + "enabled": i < 2, + "has_active_failures": i == 2, + "variables": f'{{"rhel_version": "7.9", "environment": "legacy"}}', + }) + return hosts + + +# --------------------------------------------------------------------------- +# Mock data: Job Templates +# --------------------------------------------------------------------------- + +MOCK_JOB_TEMPLATES = [ + { + "id": 10, + "type": "job_template", + "name": "CVE Remediation - Kernel Update", + "description": "Kernel update with boom snapshot for rollback safety", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": True, + "job_type": "check", + "verbosity": 1, + "timeout": 3600, + "forks": 5, + "status": "successful", + "last_job_run": _ts(timedelta(hours=4)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1001, "status": "successful", "finished": _ts(timedelta(hours=4))}, + }, + "created": _ts(timedelta(days=60)), + "modified": _ts(timedelta(days=2)), + }, + { + "id": 11, + "type": "job_template", + "name": "CVE Remediation - Package Update", + "description": "General package update for CVE remediation with needs-restarting check", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-package-update.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": False, + "job_type": "check", + "verbosity": 1, + "timeout": 1800, + "forks": 10, + "status": "successful", + "last_job_run": _ts(timedelta(hours=12)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1005, "status": "successful", "finished": _ts(timedelta(hours=12))}, + }, + "created": _ts(timedelta(days=45)), + "modified": _ts(timedelta(days=5)), + }, + { + "id": 12, + "type": "job_template", + "name": "CVE Remediation - Generic", + "description": "Generic CVE remediation template for ad-hoc patches", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-remediation.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": True, + "job_type": "check", + "verbosity": 1, + "timeout": 3600, + "forks": 5, + "status": "never updated", + "last_job_run": None, + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + }, + "created": _ts(timedelta(days=30)), + "modified": _ts(timedelta(days=30)), + }, + { + "id": 20, + "type": "job_template", + "name": "Compliance Check - STIG", + "description": "Run STIG compliance scan across fleet", + "inventory": 3, + "project": 7, + "playbook": "playbooks/compliance/check-all.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": False, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 7200, + "forks": 20, + "status": "successful", + "last_job_run": _ts(timedelta(days=1)), + "summary_fields": { + "project": {"id": 7, "name": "Compliance Checks", "status": "successful"}, + "inventory": {"id": 3, "name": "All Managed Systems", "total_hosts": 63}, + "credentials": [ + {"id": 2, "name": "compliance-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1010, "status": "successful", "finished": _ts(timedelta(days=1))}, + }, + "created": _ts(timedelta(days=180)), + "modified": _ts(timedelta(days=14)), + }, + { + "id": 25, + "type": "job_template", + "name": "Emergency Patching", + "description": "Emergency patch application — NO become enabled (misconfigured)", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/emergency-patch.yml", + "become_enabled": False, + "ask_job_type_on_launch": False, + "ask_variables_on_launch": False, + "ask_limit_on_launch": False, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 600, + "forks": 25, + "status": "failed", + "last_job_run": _ts(timedelta(days=7)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1020, "status": "failed", "finished": _ts(timedelta(days=7))}, + }, + "created": _ts(timedelta(days=200)), + "modified": _ts(timedelta(days=200)), + }, + { + "id": 30, + "type": "job_template", + "name": "Fleet Health Report", + "description": "Generate fleet health and inventory report", + "inventory": 3, + "project": 8, + "playbook": "playbooks/reporting/fleet-health.yml", + "become_enabled": False, + "ask_job_type_on_launch": False, + "ask_variables_on_launch": True, + "ask_limit_on_launch": False, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 1800, + "forks": 30, + "status": "successful", + "last_job_run": _ts(timedelta(hours=6)), + "summary_fields": { + "project": {"id": 8, "name": "Fleet Reporting", "status": "successful"}, + "inventory": {"id": 3, "name": "All Managed Systems", "total_hosts": 63}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1025, "status": "successful", "finished": _ts(timedelta(hours=6))}, + }, + "created": _ts(timedelta(days=120)), + "modified": _ts(timedelta(days=14)), + }, +] + +# --------------------------------------------------------------------------- +# Mock data: Jobs (recent runs) +# --------------------------------------------------------------------------- + +PROD_HOSTS = [ + "web-01.prod.example.com", + "web-02.prod.example.com", + "db-01.prod.example.com", + "db-02.prod.example.com", + "app-01.prod.example.com", + "app-02.prod.example.com", +] + +MOCK_JOBS = [ + { + "id": 1001, + "type": "job", + "name": "CVE Remediation - Kernel Update", + "job_type": "check", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=4, minutes=30)), + "finished": _ts(timedelta(hours=4)), + "elapsed": 1800.0, + "job_template": 10, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "limit": "web-01.prod.example.com,web-02.prod.example.com,db-01.prod.example.com", + "extra_vars": '{"target_cve": "CVE-2024-12345", "remediation_mode": "automated", "verify_after": true}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 10, "name": "CVE Remediation - Kernel Update"}, + }, + }, + { + "id": 1002, + "type": "job", + "name": "CVE Remediation - Kernel Update", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=3, minutes=45)), + "finished": _ts(timedelta(hours=3)), + "elapsed": 2700.0, + "job_template": 10, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "limit": "web-01.prod.example.com,web-02.prod.example.com,db-01.prod.example.com", + "extra_vars": '{"target_cve": "CVE-2024-12345", "remediation_mode": "automated", "verify_after": true}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 10, "name": "CVE Remediation - Kernel Update"}, + }, + }, + { + "id": 1005, + "type": "job", + "name": "CVE Remediation - Package Update", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=12, minutes=20)), + "finished": _ts(timedelta(hours=12)), + "elapsed": 1200.0, + "job_template": 11, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-package-update.yml", + "limit": "", + "extra_vars": '{"target_cve": "CVE-2024-54321"}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 11, "name": "CVE Remediation - Package Update"}, + }, + }, + { + "id": 1010, + "type": "job", + "name": "Compliance Check - STIG", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(days=1, hours=2)), + "finished": _ts(timedelta(days=1)), + "elapsed": 7200.0, + "job_template": 20, + "inventory": 3, + "project": 7, + "playbook": "playbooks/compliance/check-all.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "scheduled", + "summary_fields": { + "job_template": {"id": 20, "name": "Compliance Check - STIG"}, + }, + }, + { + "id": 1020, + "type": "job", + "name": "Emergency Patching", + "job_type": "run", + "status": "failed", + "failed": True, + "started": _ts(timedelta(days=7, hours=1)), + "finished": _ts(timedelta(days=7)), + "elapsed": 3600.0, + "job_template": 25, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/emergency-patch.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 25, "name": "Emergency Patching"}, + }, + }, + { + "id": 1025, + "type": "job", + "name": "Fleet Health Report", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=6, minutes=30)), + "finished": _ts(timedelta(hours=6)), + "elapsed": 1800.0, + "job_template": 30, + "inventory": 3, + "project": 8, + "playbook": "playbooks/reporting/fleet-health.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "scheduled", + "summary_fields": { + "job_template": {"id": 30, "name": "Fleet Health Report"}, + }, + }, +] + +_next_job_id = 2000 + + +# --------------------------------------------------------------------------- +# Mock stdout generators +# --------------------------------------------------------------------------- + +def _generate_stdout(job: dict) -> str: + """Generate realistic Ansible playbook stdout for a job.""" + playbook_name = job.get("name", "Unknown") + job_type = job.get("job_type", "run") + status = job.get("status", "successful") + limit = job.get("limit", "") + hosts = limit.split(",") if limit else PROD_HOSTS[:3] + hosts = [h.strip() for h in hosts if h.strip()] + extra_vars = job.get("extra_vars", "{}") + mode = " (CHECK MODE)" if job_type == "check" else "" + + lines = [] + lines.append(f"PLAY [{playbook_name}] *****") + lines.append("") + + lines.append(f"TASK [Gathering Facts{mode}] *****") + for h in hosts: + lines.append(f"ok: [{h}]") + lines.append("") + + if "kernel" in playbook_name.lower(): + lines.append(f"TASK [Create boom snapshot for rollback{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}] => {{\"msg\": \"boom create --title pre-remediation-CVE-2024-12345\"}}") + lines.append("") + + lines.append(f"TASK [Check disk space for kernel update{mode}] *****") + for h in hosts: + lines.append(f"ok: [{h}] => {{\"msg\": \"Disk space OK: 45% used\"}}") + lines.append("") + + lines.append(f"TASK [Update kernel package{mode}] *****") + for h in hosts: + result = "changed" if status == "successful" else "fatal" + if result == "changed": + lines.append(f'changed: [{h}] => {{"msg": "kernel-5.14.0-362.24.1.el9_3 -> kernel-5.14.0-362.24.2.el9_3"}}') + else: + lines.append(f'fatal: [{h}]: FAILED! => {{"msg": "Permission denied", "rc": 1}}') + lines.append("") + + lines.append(f"TASK [Check if reboot is needed (needs-restarting -r){mode}] *****") + for h in hosts: + lines.append(f'changed: [{h}] => {{"rc": 1, "msg": "Reboot is required to fully utilize updates."}}') + lines.append("") + + elif "package" in playbook_name.lower(): + lines.append(f"TASK [Update target packages for CVE remediation{mode}] *****") + for h in hosts: + lines.append(f'changed: [{h}] => {{"msg": "httpd-2.4.53-7.el9 -> httpd-2.4.57-8.el9"}}') + lines.append("") + + lines.append(f"TASK [Restart affected services{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}]") + lines.append("") + + lines.append(f"TASK [Verify service health{mode}] *****") + for h in hosts: + lines.append(f'ok: [{h}] => {{"msg": "Service httpd is running"}}') + lines.append("") + + elif "emergency" in playbook_name.lower() and status == "failed": + lines.append(f"TASK [Apply emergency patch{mode}] *****") + for h in hosts: + lines.append(f'fatal: [{h}]: FAILED! => {{"msg": "Missing sudo password (become_enabled not set)", "rc": 1}}') + lines.append("") + lines.append("NO MORE HOSTS LEFT *****") + lines.append("") + + else: + lines.append(f"TASK [Execute playbook tasks{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}]") + lines.append("") + + lines.append("PLAY RECAP *****") + for h in hosts: + if status == "successful": + ok_count = random.randint(3, 6) + changed_count = random.randint(1, 3) + lines.append(f"{h:<45} : ok={ok_count} changed={changed_count} unreachable=0 failed=0 skipped=0 rescued=0 ignored=0") + elif status == "failed": + lines.append(f"{h:<45} : ok=1 changed=0 unreachable=0 failed=1 skipped=0 rescued=0 ignored=0") + lines.append("") + + return "\n".join(lines) + + +def _generate_events(job: dict) -> list[dict]: + """Generate realistic Ansible task events for a job.""" + hosts = (job.get("limit", "").split(",") if job.get("limit") else PROD_HOSTS[:3]) + hosts = [h.strip() for h in hosts if h.strip()] + events: list[dict] = [] + eid = 1 + + task_names = ["Gathering Facts"] + if "kernel" in job.get("name", "").lower(): + task_names += [ + "Create boom snapshot for rollback", + "Check disk space for kernel update", + "Update kernel package", + "Check if reboot is needed (needs-restarting -r)", + ] + elif "package" in job.get("name", "").lower(): + task_names += [ + "Update target packages for CVE remediation", + "Restart affected services", + "Verify service health", + ] + else: + task_names += ["Execute playbook tasks"] + + for task_name in task_names: + for host in hosts: + is_failed = job.get("status") == "failed" and task_name != "Gathering Facts" + events.append({ + "id": eid, + "type": "job_event", + "event": "runner_on_ok" if not is_failed else "runner_on_failed", + "task": task_name, + "host": host, + "host_name": host, + "play": job.get("name", ""), + "changed": task_name != "Gathering Facts" and not is_failed, + "failed": is_failed, + "event_data": { + "task": task_name, + "host": host, + "res": { + "changed": task_name != "Gathering Facts" and not is_failed, + "msg": "Task completed" if not is_failed else "Permission denied", + }, + }, + "created": _ts(timedelta(hours=4, minutes=30 - eid)), + }) + eid += 1 + + return events + + +def _generate_host_summaries(job: dict) -> list[dict]: + """Generate per-host summaries for a job.""" + hosts = (job.get("limit", "").split(",") if job.get("limit") else PROD_HOSTS[:3]) + hosts = [h.strip() for h in hosts if h.strip()] + summaries: list[dict] = [] + + for i, host in enumerate(hosts): + is_failed = job.get("status") == "failed" + summaries.append({ + "id": i + 1, + "type": "job_host_summary", + "host": i + 1, + "host_name": host, + "ok": 1 if is_failed else random.randint(3, 6), + "changed": 0 if is_failed else random.randint(1, 3), + "dark": 0, + "failures": 1 if is_failed else 0, + "skipped": 0, + "processed": 1, + "failed": is_failed, + }) + + return summaries + + +# --------------------------------------------------------------------------- +# MCP Tools: Job Management +# --------------------------------------------------------------------------- + +@mcp.tool() +def job_templates_list( + page_size: int = 10, + search: Optional[str] = None, +) -> dict: + """List available job templates in AAP. + + Args: + page_size: Number of results per page (default 10, max 200). + search: Optional search string to filter templates by name. + """ + results = MOCK_JOB_TEMPLATES + if search: + s = search.lower() + results = [t for t in results if s in t["name"].lower() or s in t.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def job_templates_retrieve(id: str) -> dict: + """Retrieve detailed information about a specific job template. + + Args: + id: Job template ID (as string). + """ + tid = int(id) + template = next((t for t in MOCK_JOB_TEMPLATES if t["id"] == tid), None) + if not template: + return {"detail": f"Not found. Job template {id} does not exist."} + return template + + +@mcp.tool() +def projects_list( + page_size: int = 50, + search: Optional[str] = None, +) -> dict: + """List available projects in AAP. + + Args: + page_size: Number of results per page. + search: Optional search string to filter projects by name. + """ + results = MOCK_PROJECTS + if search: + s = search.lower() + results = [p for p in results if s in p["name"].lower() or s in p.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def job_templates_launch_retrieve( + id: str, + requestBody: Optional[dict] = None, +) -> dict: + """Launch a job from a job template. + + Args: + id: Job template ID to launch. + requestBody: Optional launch parameters including job_type ('run' or 'check'), + extra_vars (dict), and limit (comma-separated host list). + """ + global _next_job_id + tid = int(id) + template = next((t for t in MOCK_JOB_TEMPLATES if t["id"] == tid), None) + if not template: + return {"detail": f"Not found. Job template {id} does not exist."} + + body = requestBody or {} + job_type = body.get("job_type", template.get("job_type", "run")) + + if not template.get("ask_job_type_on_launch") and job_type != template.get("job_type"): + return { + "error": f"Cannot override job_type: ask_job_type_on_launch is disabled on template {id}", + } + + job_id = _next_job_id + _next_job_id += 1 + + new_job = { + "id": job_id, + "type": "job", + "name": template["name"], + "job_type": job_type, + "status": "pending", + "failed": False, + "started": _ts(timedelta(seconds=0)), + "finished": None, + "elapsed": 0.0, + "job_template": tid, + "inventory": template["inventory"], + "project": template["project"], + "playbook": template["playbook"], + "limit": body.get("limit", ""), + "extra_vars": str(body.get("extra_vars", {})), + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": tid, "name": template["name"]}, + }, + } + MOCK_JOBS.append(new_job) + + # Simulate job completion after launch + new_job["status"] = "successful" + new_job["finished"] = _ts(timedelta(seconds=-300)) + new_job["elapsed"] = 300.0 + + return { + "job": job_id, + "status": "pending", + "type": "job", + "url": f"/api/controller/v2/jobs/{job_id}/", + "related": { + "stdout": f"/api/controller/v2/jobs/{job_id}/stdout/", + "job_events": f"/api/controller/v2/jobs/{job_id}/job_events/", + "job_host_summaries": f"/api/controller/v2/jobs/{job_id}/job_host_summaries/", + }, + } + + +@mcp.tool() +def jobs_retrieve(id: int) -> dict: + """Get the status and details of a job run. + + Args: + id: Job ID to retrieve. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + return job + + +@mcp.tool() +def jobs_list(page_size: int = 10) -> dict: + """List recent job runs. + + Args: + page_size: Number of results to return. + """ + results = sorted(MOCK_JOBS, key=lambda j: j.get("started", ""), reverse=True) + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def jobs_stdout_retrieve(id: int, format: str = "txt") -> dict: + """Get the stdout (console output) from a job run. + + Args: + id: Job ID. + format: Output format ('txt' or 'json'). Default 'txt'. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + return { + "content": _generate_stdout(job), + "range": {"start": 0, "end": 1}, + } + + +@mcp.tool() +def jobs_job_events_list(id: int, page_size: int = 50) -> dict: + """Get task-level events for a job run. + + Args: + id: Job ID. + page_size: Number of events to return. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + events = _generate_events(job) + return { + "count": len(events), + "next": None, + "previous": None, + "results": events[:page_size], + } + + +@mcp.tool() +def jobs_job_host_summaries_list(id: int) -> dict: + """Get per-host execution summaries for a job run. + + Args: + id: Job ID. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + summaries = _generate_host_summaries(job) + return { + "count": len(summaries), + "next": None, + "previous": None, + "results": summaries, + } + + +@mcp.tool() +def jobs_relaunch_retrieve( + id: int, + hosts: str = "all", + job_type: str = "run", +) -> dict: + """Relaunch a previously completed or failed job. + + Args: + id: Original job ID to relaunch. + hosts: Which hosts to target ('all' or 'failed'). + job_type: Job type for relaunch ('run' or 'check'). + """ + global _next_job_id + original = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not original: + return {"detail": f"Not found. Job {id} does not exist."} + + new_id = _next_job_id + _next_job_id += 1 + + new_job = { + **original, + "id": new_id, + "job_type": job_type, + "status": "successful", + "failed": False, + "started": _ts(timedelta(seconds=0)), + "finished": _ts(timedelta(seconds=-300)), + "elapsed": 300.0, + "launch_type": "relaunch", + } + MOCK_JOBS.append(new_job) + + return { + "job": new_id, + "status": "pending", + "type": "job", + "url": f"/api/controller/v2/jobs/{new_id}/", + } + + +# --------------------------------------------------------------------------- +# MCP Tools: Inventory Management +# --------------------------------------------------------------------------- + +@mcp.tool() +def inventories_list( + page_size: int = 10, + search: Optional[str] = None, +) -> dict: + """List available inventories in AAP. + + Args: + page_size: Number of results per page. + search: Optional search string to filter inventories. + """ + results = MOCK_INVENTORIES + if search: + s = search.lower() + results = [inv for inv in results if s in inv["name"].lower() or s in inv.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def hosts_list( + inventory_id: Optional[int] = None, + page_size: int = 50, + search: Optional[str] = None, +) -> dict: + """List hosts in an inventory. + + Args: + inventory_id: Filter by inventory ID. If not provided, lists hosts from all inventories. + page_size: Number of results per page. + search: Optional search string to filter hosts by name. + """ + inv_id = inventory_id or 1 + hosts = _generate_hosts(inv_id) + if search: + s = search.lower() + hosts = [h for h in hosts if s in h["name"].lower()] + return { + "count": len(hosts), + "next": None if len(hosts) <= page_size else f"/api/controller/v2/hosts/?page=2", + "previous": None, + "results": hosts[:page_size], + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/with_skills/rh-sre__playbook-executor/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..fe5d549c --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/mcp-aap-validator/SKILL.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/mcp-aap-validator/SKILL.md new file mode 100644 index 00000000..a1c4f708 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/mcp-aap-validator/SKILL.md @@ -0,0 +1,66 @@ +--- +name: mcp-aap-validator +description: | + Validate AAP (Ansible Automation Platform) MCP server connectivity. Use when the user asks to "validate AAP MCP", "check AAP connection", or when other skills need to verify AAP MCP availability before job management or inventory operations. +model: haiku +color: yellow +--- + +# MCP AAP Validator + +Validates connectivity to AAP MCP servers by running lightweight tool calls. + +## When to Use This Skill + +Use when validating AAP MCP before job template operations, troubleshooting connection issues, or when other skills (e.g. playbook-executor) need to verify availability. Do NOT use for creating templates—use job-template-creator. + +## Workflow + +1. **Test connectivity**: Call these tools to verify each server responds: + - `job_templates_list` (page_size: 10) from aap-mcp-job-management + - `inventories_list` (page_size: 10) from aap-mcp-inventory-management +2. **If any fails**: Provide a comprehensive message with possible root causes (see below). +3. **Report**: Output a table with validated servers and outcome (emojis). + +## Failure Message (Root Causes) + +When a tool call fails, include: + +``` +❌ AAP MCP connection failed + +**Possible root causes:** +- **Credentials**: AAP_MCP_SERVER or AAP_API_TOKEN not set or invalid +- **401 Unauthorized**: Token expired or invalid → regenerate in AAP Web UI +- **403 Forbidden**: Token lacks RBAC permissions (need Job Templates, Inventories) +- **404 Not Found**: Wrong AAP_MCP_SERVER URL (must point to MCP gateway, not main AAP UI) +- **Connection timeout**: Server unreachable, firewall, or network issue +- **SSL/TLS error**: Certificate verification problem + +**Troubleshooting:** +1. Verify env vars: AAP_MCP_SERVER, AAP_API_TOKEN (never echo values) +2. Get token: AAP Web UI → Users → [Your User] → Tokens → Create +3. Ensure AAP_MCP_SERVER points to MCP gateway endpoint +4. Restart host after config changes +``` + +## Report Format + +Always end with a table: + +| Server | Outcome | +|--------|---------| +| aap-mcp-job-management | ✅ PASSED | +| aap-mcp-inventory-management | ✅ PASSED | + +Use ✅ for success, ❌ for failure, ⚠️ for partial (e.g. one server OK, one failed). + +## Dependencies + +### Required MCP Tools +- `job_templates_list` (from aap-mcp-job-management) - Connectivity test +- `inventories_list` (from aap-mcp-inventory-management) - Connectivity test + +### Required MCP Servers +- `aap-mcp-job-management` - AAP job template and execution +- `aap-mcp-inventory-management` - AAP inventory management diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/SKILL.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/SKILL.md new file mode 100644 index 00000000..a29c9443 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/SKILL.md @@ -0,0 +1,499 @@ +--- +name: playbook-executor +description: | + **CRITICAL**: Use for Ansible playbook execution via AAP. DO NOT call AAP MCP tools directly. + + Execute remediation playbooks with job management, dry-run, and reporting. Use after playbook-generator. + + **Git Flow**: If template playbook path ≠ generated playbook, perform Git Flow (commit, push, sync) BEFORE launch. +--- + +# AAP Playbook Executor Skill + +This skill executes Ansible remediation playbooks through AAP (Ansible Automation Platform) with full job management capabilities. + +**Integration with Remediation Skill**: The `/remediation` skill orchestrates this skill as part of its Step 5 (Execute Playbook) workflow. For standalone playbook execution, you can invoke this skill directly. + +## Prerequisites + +**Required MCP Servers**: `aap-mcp-job-management`, `aap-mcp-inventory-management` ([setup guide](https://docs.redhat.com/)) + +**Required MCP Tools**: +- `job_templates_list` (from aap-mcp-job-management) - List job templates +- `job_templates_retrieve` (from aap-mcp-job-management) - Get template details +- `projects_list` (from aap-mcp-job-management) - Get project name and scm_url for Git Flow +- `job_templates_launch_retrieve` (from aap-mcp-job-management) - Launch jobs +- `jobs_retrieve` (from aap-mcp-job-management) - Get job status +- `jobs_stdout_retrieve` (from aap-mcp-job-management) - Get console output +- `jobs_job_events_list` (from aap-mcp-job-management) - Get task events +- `jobs_job_host_summaries_list` (from aap-mcp-job-management) - Get host statistics +- `inventories_list` (from aap-mcp-inventory-management) - List inventories +- `hosts_list` (from aap-mcp-inventory-management) - List inventory hosts + +**Required Environment Variables**: +- `AAP_MCP_SERVER` - Base URL for the MCP endpoint of the AAP server (must point to the AAP MCP gateway) +- `AAP_API_TOKEN` - AAP API authentication token + +### Prerequisite Validation + +**CRITICAL**: Before executing operations, execute the `/mcp-aap-validator` skill to verify AAP MCP server availability. + +**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-aap-validator/SKILL.md#validation-freshness-policy). + +**How to invoke**: Execute the `/mcp-aap-validator` skill + +**Handle validation result**: +- **If validation PASSED**: Continue with playbook execution workflow +- **If validation PARTIAL**: Warn user and ask to proceed +- **If validation FAILED**: Stop execution, provide setup instructions from validator + +**Human Notification on Failure**: +If prerequisites are not met: +- ❌ "Cannot proceed: AAP MCP servers are not available" +- 📋 "Setup required: Configure AAP_MCP_SERVER and AAP_API_TOKEN environment variables" +- ❓ "How would you like to proceed? (setup now / skip / abort)" +- ⏸️ Wait for user decision + +## When to Use This Skill + +**Use this skill directly when you need**: +- Execute a previously generated Ansible playbook via AAP +- Track the status of a running AAP job +- Monitor playbook job completion +- Run dry-run (check mode) before production execution +- Verify playbook execution succeeded + +**Use the `/remediation` skill when you need**: +- Full remediation workflow including playbook execution +- Integrated CVE analysis → playbook generation → execution → verification +- End-to-end remediation orchestration + +**How they work together**: The `/remediation` skill invokes this skill after generating a remediation playbook, managing the full workflow from analysis to verification. + +## Workflow + +**Git Flow is MANDATORY**: When the job template's playbook path differs from the generated playbook (or content must be updated), you MUST perform Git Flow (write, commit, push, sync) and receive "sync complete" from the user BEFORE launching any job. Do NOT skip this—launching without it executes the wrong playbook. + +### Phase 0: Validate AAP MCP Prerequisites + +**Action**: Execute the `/mcp-aap-validator` skill + +**Note**: Can skip if validation was performed earlier in this session and succeeded. + +**How to invoke**: Execute the `/mcp-aap-validator` skill + +**Handle validation result**: +- **If validation PASSED**: Continue to Phase 1 +- **If validation PARTIAL**: Warn user and ask to proceed +- **If validation FAILED**: Stop execution, user must set up AAP MCP servers + +### Phase 1: Job Template Selection and Playbook Preparation + +**Goal**: Identify an AAP job template suitable for executing the remediation playbook. **Git Flow is MANDATORY** before Phase 3 when the template points to a different playbook or when content must be updated. + +**Input**: Playbook content and metadata from playbook-generator (filename, CVE ID, target systems). The playbook YAML is already in context—do NOT regenerate it during Git Flow. Playbook path is derived from metadata: `playbooks/remediation/` (e.g., `playbooks/remediation/remediation-CVE-2025-49794.yml`). + +**BLOCKING**: You MUST NOT launch any job (dry-run or production) until the playbook is in the Git repo and the user has confirmed "sync complete". AAP executes from the synced project—there is no "override at launch". Launching without Git Flow executes the WRONG playbook. + +#### Step 1.1: Derive Playbook Path + +From playbook metadata (filename from playbook-generator): +- Use convention `playbooks/remediation/` +- Support both `remediation-CVE-*.yml` and `remediation-CVE-*-playbook.yml` patterns. +- Example: CVE-2026-26103 → `playbooks/remediation/remediation-CVE-2026-26103.yml` + +#### Step 1.2: List Templates and Validate Each Candidate + +**MCP Tool**: `job_templates_list` (from aap-mcp-job-management) + +**Parameters**: +- `page_size`: 50 (retrieve up to 50 templates) +- `search`: "" (search for all templates) + +**REQUIRED**: For each template in results: +1. Call `job_templates_retrieve(id)` to get full details +2. **Invoke the `/job-template-remediation-validator` skill** with the template ID to verify it meets remediation requirements (inventory, project, playbook, credentials, become_enabled) +3. Only include templates that PASS validation in the lists below + +Build two lists: +- **exact_match**: `template.playbook` equals `our_playbook_path` (normalize slashes; match if equal or basenames match) +- **compatible_other**: Passes job-template-remediation-validator but **different playbook path** (template points to e.g. `cve-remediation.yml` while we have `remediation-CVE-2026-26103.yml`) + +**Path normalization**: Normalize slashes, handle `playbooks/remediation/` prefix. Match if `template.playbook` equals `our_playbook_path` or if basenames match. **Different filenames = different path = Scenario 2.** + +#### Step 1.3: Scenario Selection (MANDATORY - Do Not Skip) + +**Scenario 1 - Same playbook path** (exact_match not empty): + +The template already points to our playbook path. The project may need the latest content. **Read [references/05-git-flow-prompts.md](references/05-git-flow-prompts.md)** for Scenario 1 prompt, options (A/B), and Git Flow steps. + +- **If A**: Execute Git Flow (see Git Flow section below). **BLOCK Phase 3** until user confirms "sync complete" or "done". +- **If B**: Wait for user confirmation. **BLOCK Phase 3** until user confirms. + +**Scenario 2 - Different playbook path** (compatible_other not empty, exact_match empty): + +**CRITICAL**: The template points to a DIFFERENT playbook than our generated playbook. You MUST NOT launch the job without Git Flow—AAP executes from synced content; there is no override at launch. **Read [references/05-git-flow-prompts.md](references/05-git-flow-prompts.md)** for Scenario 2 prompt and Git Flow steps. + +- **If yes**: Execute Git Flow. **BLOCK Phase 3** until Git Flow completes and user confirms "sync complete". +- **If no**: Fall through to Scenario 3. + +**Anti-pattern**: Do NOT say "I'll override with our playbook" and then launch—that is impossible. The playbook MUST be in the repo before launch. + +**Scenario 3 - No suitable template** (exact_match and compatible_other both empty, or user chose "no" in Scenario 2): + +Execute the `/job-template-creator` skill with instruction: +``` +"Create a job template for this remediation playbook. Playbook: [content]. Filename: [filename]. Path: [our_playbook_path]. CVE: [cve_id]. Target systems: [list]." +``` + +The job-template-creator skill guides the user through: (1) Adding playbook to Git repository, (2) Syncing AAP project, (3) Creating job template via AAP Web UI with correct path, inventory, credentials, privilege escalation. + +After `/job-template-creator` completes, retrieve the template ID (from skill output or user confirmation). Execute `/job-template-remediation-validator` to validate the newly created template. If passed, proceed to Phase 3 (Dry-Run). If failed, report issues and ask user to fix in AAP Web UI. + +**Multiple matches**: If multiple exact matches, present list and ask user to choose by number. If multiple different-path matches, prefer by project name containing "remediation" or "CVE", else first. + +**Phase 1 Checkpoint** (BLOCKING - must pass before Phase 3): +- **Git Flow required**: If Scenario 1 or 2, you MUST complete Git Flow and receive "sync complete" from the user before proceeding. Do NOT skip. +- **No override**: There is no way to "override" the playbook at launch. AAP runs whatever is in the synced project. +- **Never launch** if the playbook has not been committed, pushed, and synced + +#### Git Flow (for Scenario 1 Override and Scenario 2) - MANDATORY HITL + +**When**: Scenario 1 (same path, update content) or Scenario 2 (different path, replace playbook). **Do not skip**—execution with wrong playbook content will remediate the wrong CVE. + +**Target path**: +- Scenario 1: `our_playbook_path` (e.g. `playbooks/remediation/remediation-CVE-2026-26103.yml`) +- Scenario 2: `template.playbook` (e.g. `playbooks/remediation/cve-remediation.yml`)—we replace the template's playbook with our generated content + +**Prerequisite**: Ask user for the local path to the Git repository. Use `projects_list` for project name and `scm_url`. **Read [references/05-git-flow-prompts.md](references/05-git-flow-prompts.md)** for repo path question, HITL checkpoint text, and after-push message. + +**Steps** (execute in order; HITL at checkpoint): +1. **Write playbook to file** (FAST—do NOT regenerate): + - The playbook content is ALREADY in context from playbook-generator (or remediation skill). Use it directly. + - **⚠️ ABSOLUTE PATH REQUIRED**: The Write path MUST start with `/`. Use: `/`. Example: `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project/playbooks/remediation/cve-remediation.yml` + - **WRONG** (causes "Error writing file"): `test-aap-project/playbooks/...` or `playbooks/remediation/...` — these are relative and fail when repo is outside workspace. + - **Before Write**: Confirm path starts with `/`. If not, prepend the user's repo path. + - Do NOT invoke playbook-generator, do NOT call MCP tools, do NOT re-fetch. This should take seconds, not minutes. +2. Use Run tool: `git add ` (from repo root, e.g. `git add playbooks/remediation/cve-remediation.yml`) +3. **HITL Checkpoint** (REQUIRED): Display summary per reference file. Wait for "yes" or "proceed" +4. If confirmed: `git commit -m "Add/update remediation playbook for CVE-YYYY-NNNNN"` +5. `git push origin main` (or branch from project's scm_branch if available) + +**Note**: Git must be configured. Use Run tool for git commands. + +**Do NOT proceed to Phase 3 (Dry-Run) until user confirms sync complete.** + +### Phase 2: Git Flow (MANDATORY before Phase 3) + +**BLOCKING**: You MUST NOT proceed to Phase 3 (Dry-Run) until Git Flow is complete. + +**When**: Scenario 1 (same path, update content) or Scenario 2 (different path). See Phase 1 Step 1.3. + +**Checkpoint**: Before Phase 3, confirm: +- [ ] Playbook written to repo at target path +- [ ] Git commit and push completed (with user confirmation) +- [ ] User confirmed "sync complete" after AAP project sync + +**If any unchecked**: STOP. Do Git Flow. Do NOT launch the job. + +### Phase 3: Dry-Run Execution (Recommended) + +**Prerequisite**: Phase 2 (Git Flow) MUST be complete. User must have confirmed "sync complete". + +**Goal**: Test playbook in check mode before actual execution to simulate changes. + +**Read [references/04-dry-run-display-templates.md](references/04-dry-run-display-templates.md)** for: Playbook Preview, Dry-Run Offer, Dry-Run Results Display, Proceed prompt. + +#### Step 3.1–3.2: Display Preview and Offer Dry-Run + +Show playbook structure per reference. Offer dry-run with options: yes / no / abort. **ONLY if user confirms**, proceed. + +#### Step 3.3: Launch Dry-Run Job + +**Pre-launch check** (BLOCKING): If Scenario 1 or 2 applied, you MUST have completed Git Flow and received "sync complete" from the user. If not, STOP—do not launch. Return to Phase 2 / Git Flow. + +**MCP Tool**: `job_templates_launch_retrieve` (from aap-mcp-job-management) + +**Parameters**: `id`, `requestBody` with `job_type: "check"`, `extra_vars`, `limit` + +**Key**: `job_type: "check"` - Runs Ansible in check mode (dry-run) + +#### Step 3.4: Monitor Dry-Run Progress + +Poll `jobs_retrieve` every 2 seconds. Use `jobs_job_events_list` for live task updates. + +#### Step 3.5: Display Dry-Run Results + +**MCP Tools**: `jobs_stdout_retrieve` (id, format: "txt"), `jobs_job_host_summaries_list` (id). Use display format from reference. + +#### Step 3.6: Proceed to Actual Execution? + +Ask per reference. Wait for "yes" or "execute". + +### Phase 4: Actual Execution + +**ONLY execute if user explicitly confirms** (either after dry-run or directly if they skipped dry-run). + +#### Step 4.1: Final Confirmation + +``` +⚠️ CRITICAL: Playbook Execution Confirmation Required + +This playbook will: +- Execute on: 3 production systems +- Update packages: httpd (2.4.53-7.el9 → 2.4.57-8.el9) +- Restart services: httpd +- Estimated downtime: ~10 seconds per system +- Requires reboot: No + +Job Template: CVE Remediation Template (ID: 10) +AAP URL: https://aap.example.com/jobs/ + +❓ Execute this playbook now? + +Options: +- "yes" or "execute" - Proceed with execution +- "abort" - Cancel execution + +Please respond with your choice. +``` + +Wait for explicit "yes" or "execute" response. + +#### Step 4.2: Launch Production Job + +**Pre-launch check** (BLOCKING): Same as Phase 3—if Scenario 1 or 2 applied, Git Flow must be complete and user must have confirmed "sync complete". Do NOT launch without it. + +**MCP Tool**: `job_templates_launch_retrieve` (from aap-mcp-job-management) + +**Parameters**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run", + "extra_vars": { + "target_cve": "CVE-2025-49794", + "remediation_mode": "automated", + "verify_after": true + }, + "limit": "prod-web-01,prod-web-02,prod-web-03" + } +} +``` + +**Key Parameter**: `job_type: "run"` - Runs Ansible in execution mode (actual changes) + +**Expected Output**: +```json +{ + "job": 1235, + "status": "pending", + "url": "/api/controller/v2/jobs/1235/" +} +``` + +#### Step 4.3: Monitor Execution Progress + +**Polling Strategy**: +1. Call `jobs_retrieve(id=job_id)` every 2 seconds +2. Get task events with `jobs_job_events_list(id=job_id)` for progress updates +3. Display real-time task completion status +4. Continue until status is "successful", "failed", or "error" + +**Progress Display**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running +Elapsed: 1m 23s +AAP URL: https://aap.example.com/#/jobs/playbook/1235 + +Recent Events: +- ✓ Gathering Facts (completed - all hosts) +- ✓ Check Disk Space (completed - all hosts) +- ✓ Backup Configuration (completed - all hosts) +- ⏳ Update Package: httpd (running - prod-web-01, prod-web-02) + └─ prod-web-01: Installing httpd-2.4.57-8.el9... + └─ prod-web-02: Installing httpd-2.4.57-8.el9... +- ⏸ Restart Service: httpd (pending) +``` + +**Update every 2 seconds** until completion. + +### Phase 5: Execution Report + +**Goal**: Generate comprehensive report with job details, per-host results, and full output. + +**Read [references/01-execution-report-templates.md](references/01-execution-report-templates.md)** for JSON examples, comprehensive report template, and Success/Partial Success/Failure output templates. + +#### Step 5.1–5.4: Gather Data + +**MCP Tools** (all from aap-mcp-job-management): +- `jobs_retrieve` (id) - Job details +- `jobs_job_host_summaries_list` (id) - Per-host stats +- `jobs_job_events_list` (id) - Task timeline +- `jobs_stdout_retrieve` (id, format: "txt") - Full console output + +#### Step 5.5: Generate Report + +Format all gathered data per reference. Use Success / Partial Success / Failure template based on job status. + +#### Step 5.6: Validate Job Log for CVE Handling (MANDATORY) + +**Goal**: Confirm from the job stdout that the playbook actually addressed the target CVE(s). + +**Input**: Target CVE ID(s) from invocation (e.g. CVE-2025-49794). Job stdout from `jobs_stdout_retrieve` (already gathered in Step 5.4). + +**Parse stdout for**: +- Target CVE ID(s) in output (vars, task names, audit logs, playbook metadata) +- Package update tasks for affected packages (dnf/yum install/update, package module) +- Remediation-related task names (e.g. "Update package", "Restart service", "remediation") + +**Report** (add to execution report): +- **✓ Job log confirms CVE-XXXX-YYYY was addressed** — CVE ID or package updates found in stdout +- **⚠️ Job log did not show clear evidence of CVE handling** — No CVE ID or package updates found; recommend manual verification or `/remediation-verifier` + +**Batch**: For multiple CVEs, validate each. Report per-CVE confirmation or warning. + +### Phase 6: Error Handling + +**If job status is "failed" or "error"**, provide detailed troubleshooting. + +**Read [references/02-error-handling-guide.md](references/02-error-handling-guide.md)** for: Error categories, error report template, troubleshooting steps, relaunch parameters. + +#### Step 6.1: Parse Error Output + +**MCP Tool**: `jobs_stdout_retrieve`. Analyze output for error categories per reference. + +#### Step 6.2: Generate Error Report + +Use error report template from reference. Include per-host results, failed task details, troubleshooting steps, relaunch options. + +#### Step 6.3: Offer Relaunch + +If user chooses relaunch: **MCP Tool** `jobs_relaunch_retrieve` with `hosts: "failed"`, `job_type: "run"` per reference. + +## Reference Files + +| File | Use When | +|------|----------| +| [01-execution-report-templates.md](references/01-execution-report-templates.md) | Phase 5 reports, Success/Partial/Failure output | +| [02-error-handling-guide.md](references/02-error-handling-guide.md) | Phase 6 error reports, relaunch | +| [03-workflow-examples.md](references/03-workflow-examples.md) | Demo full workflow, failure handling, skip dry-run | +| [04-dry-run-display-templates.md](references/04-dry-run-display-templates.md) | Phase 3 preview, offer, results, proceed prompt | +| [05-git-flow-prompts.md](references/05-git-flow-prompts.md) | Scenario 1/2 prompts, Git Flow HITL, after-push | + +## Dependencies + +### Required MCP Servers +- `aap-mcp-job-management` - AAP job management and execution +- `aap-mcp-inventory-management` - AAP inventory management + +### Required MCP Tools +- `job_templates_list` (from aap-mcp-job-management) - List templates +- `job_templates_retrieve` (from aap-mcp-job-management) - Get template details +- `projects_list` (from aap-mcp-job-management) - Get project name and scm_url for Git Flow +- `job_templates_launch_retrieve` (from aap-mcp-job-management) - Launch jobs +- `jobs_retrieve` (from aap-mcp-job-management) - Get job status +- `jobs_stdout_retrieve` (from aap-mcp-job-management) - Get console output +- `jobs_job_events_list` (from aap-mcp-job-management) - Get task events +- `jobs_job_host_summaries_list` (from aap-mcp-job-management) - Get host statistics +- `inventories_list` (from aap-mcp-inventory-management) - List inventories +- `hosts_list` (from aap-mcp-inventory-management) - List hosts + +### Related Skills +- `mcp-aap-validator` - **PREREQUISITE** - Validates AAP MCP servers (invoke in Phase 0) +- `job-template-remediation-validator` - **REQUIRED** - Invoke for each candidate template in Phase 1 Step 1.2 to verify remediation requirements +- `job-template-creator` - Creates/guides AAP job template setup +- `playbook-generator` - Generates playbooks for execution +- `remediation-verifier` - Verifies success after execution + +### Reference Documentation +- [references/](references/) - Step-numbered reference files (01–05) for templates and examples +- [AAP Job Execution Guide](../../docs/ansible/aap-job-execution.md) - AAP job execution best practices +- [Playbook Integration with AAP](../../docs/ansible/playbook-integration-aap.md) - Playbook-to-AAP workflow + +## Critical: Human-in-the-Loop Requirements + +This skill executes code on production systems. **Explicit user confirmation is REQUIRED** at multiple stages. + +**Before Git commit/push** (Scenario 1 Override, Scenario 2): +1. **Display change summary**: File path, diff or file size +2. **Ask for confirmation**: "Ready to commit and push these changes? Reply 'yes' or 'proceed' to continue, or 'abort' to cancel." +3. **Wait for explicit "yes" or "proceed"**: Do not commit/push without confirmation + +**Before Dry-Run Execution** (if user chooses dry-run): +1. **Display Playbook Preview**: Show tasks and explain changes +2. **Ask for Dry-Run Confirmation**: + ``` + ❓ Run dry-run to simulate changes? + + Options: + - "yes" - Run dry-run (recommended) + - "no" - Skip to actual execution + - "abort" - Cancel + + Please respond with your choice. + ``` +3. **Wait for Explicit Response**: Do not proceed without confirmation + +**Before Actual Execution** (REQUIRED): +1. **Display Execution Summary**: Show systems, changes, downtime estimate +2. **Ask for Final Confirmation**: + ``` + ⚠️ CRITICAL: Execute playbook on production systems? + + This will make real changes to N systems. + + Options: + - "yes" or "execute" - Proceed + - "abort" - Cancel + + Please respond with your choice. + ``` +3. **Wait for Explicit "yes" or "execute"**: Do not proceed without confirmation + +**Never assume approval** - always wait for explicit user confirmation before executing playbooks. + +## Best Practices + +1. **Write path must be absolute** - When Git Flow writes the playbook to the user's repo, use `/playbooks/remediation/`. The path MUST start with `/`. Relative paths cause "Error writing file". +2. **Always validate AAP prerequisites** - Invoke mcp-aap-validator in Phase 0 +3. **Validate each template** - Invoke job-template-remediation-validator for each candidate before selection +4. **Never skip Git Flow** - If template playbook path ≠ generated playbook path (Scenario 2) or content must be updated (Scenario 1), you MUST complete Git Flow and receive "sync complete" before Phase 3. Do NOT launch without it. +5. **Recommend dry-run** - Offer check mode before production execution +6. **Filter compatible templates** - Check inventory, project, and credentials match +7. **Monitor in real-time** - Display task progress during execution +8. **Comprehensive reporting** - Include per-host stats, task timeline, full output +9. **Error categorization** - Parse errors and provide specific troubleshooting +10. **Relaunch capability** - Offer to retry failed hosts +11. **Link to AAP** - Provide direct URL to job in AAP Web UI +12. **Suggest verification** - Always recommend remediation-verifier after success +13. **Document job details** - Save job ID and template info for audit trail + +## Integration with Other Skills + +- **playbook-generator**: Generates playbooks that this skill executes +- **job-template-creator**: Creates AAP job templates when needed +- **remediation-verifier**: Verifies success after this skill completes execution +- **`/remediation` skill**: Orchestrates full workflow including playbook execution + +**Orchestration Example** (from `/remediation` skill): +1. Agent invokes playbook-generator skill → Creates playbook YAML +2. playbook-generator asks for confirmation → User approves playbook content +3. Agent invokes playbook-executor skill (this skill) → Execution workflow +4. Skill validates templates via job-template-remediation-validator → Filters valid candidates +5. Skill checks path match → If different path, offers Git Flow (HITL: commit/push, sync AAP) +6. Skill waits for "sync complete" before proceeding (if Git Flow was used) +7. Skill offers dry-run → User runs check mode +8. Skill asks for execution confirmation → User approves +9. Skill executes and monitors → Reports completion +10. Agent invokes remediation-verifier skill → Confirms CVE resolved + +**Note**: Both playbook-generator and playbook-executor require separate confirmations for different purposes: +- playbook-generator: Confirms playbook content is acceptable +- playbook-executor: Confirms execution on production systems is approved + +This two-step approval ensures user control over both what to run and when to run it. diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/01-execution-report-templates.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/01-execution-report-templates.md new file mode 100644 index 00000000..a6773c5f --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/01-execution-report-templates.md @@ -0,0 +1,168 @@ +# Step 01: Execution Report Templates + +Read this reference when generating Phase 5 execution reports or output templates. + +## Phase 5: Job Details (JSON Examples) + +### jobs_retrieve Expected Output + +```json +{ + "id": 1235, + "name": "CVE Remediation Template", + "status": "successful", + "started": "2026-02-24T15:35:02Z", + "finished": "2026-02-24T15:40:25Z", + "elapsed": 323.45, + "job_template": 10, + "inventory": 1, + "limit": "prod-web-01,prod-web-02,prod-web-03", + "playbook": "playbooks/remediation/remediation-CVE-2025-49794.yml" +} +``` + +### jobs_job_host_summaries_list Expected Output + +```json +{ + "results": [ + { + "host_name": "prod-web-01", + "ok": 8, + "changed": 3, + "failed": 0, + "unreachable": 0 + }, + { + "host_name": "prod-web-02", + "ok": 8, + "changed": 3, + "failed": 0, + "unreachable": 0 + }, + { + "host_name": "prod-web-03", + "ok": 5, + "changed": 0, + "failed": 1, + "unreachable": 0 + } + ] +} +``` + +## Comprehensive Report Template + +```markdown +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 5m 23s +**Started**: 2026-02-24 15:35:02 UTC +**Completed**: 2026-02-24 15:40:25 UTC +**Job Template**: CVE Remediation Template +**Playbook**: playbooks/remediation/remediation-CVE-2025-49794.yml +**AAP URL**: [View in AAP](https://aap.example.com/#/jobs/playbook/1235) + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| prod-web-01 | 8 | 3 | 0 | 0 | ✅ Success | +| prod-web-02 | 8 | 3 | 0 | 0 | ✅ Success | +| prod-web-03 | 8 | 3 | 0 | 0 | ✅ Success | + +**Summary**: 3 of 3 hosts successfully remediated + +## Task Timeline +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) + - prod-web-01: 2.4.53-7.el9 → 2.4.57-8.el9 + - prod-web-02: 2.4.53-7.el9 → 2.4.57-8.el9 + - prod-web-03: 2.4.53-7.el9 → 2.4.57-8.el9 +5. ✅ Restart httpd service (15s) +6. ✅ Verify service status (2s) +7. ✅ Update audit log (1s) + +## Full Console Output +
+Click to expand (187 lines) + +[Full stdout from jobs_stdout_retrieve] + +
+ +## Job Log CVE Validation (Step 5.6) +✓ Job log confirms CVE-XXXX-YYYY was addressed + +*(Or: ⚠️ Job log did not show clear evidence of CVE handling—verify manually or use remediation-verifier)* + +## Next Steps +1. ✅ All systems successfully remediated +2. ☐ Verify remediation with remediation-verifier skill +3. ☐ Update vulnerability tracking system +4. ☐ Schedule follow-up verification in 24-48 hours + +--- + +**Recommendation**: Run remediation-verifier skill to confirm CVE status has been updated in Red Hat Lightspeed. +``` + +## Output Templates + +### Success Template + +```markdown +✅ Playbook Execution Successful + +Job ID: 1235 +Duration: 5m 23s +Systems Remediated: 3 of 3 + +View full report above for details. + +Next Steps: +- Run remediation-verifier skill to confirm CVE resolution +- Update vulnerability tracking system +- Monitor systems for 24-48 hours + +AAP URL: https://aap.example.com/#/jobs/playbook/1235 +``` + +### Partial Success Template + +```markdown +⚠️ Playbook Execution Completed with Failures + +Job ID: 1235 +Duration: 2m 45s +Systems Remediated: 2 of 3 +Failed Systems: prod-web-03 + +See error details above for troubleshooting steps. + +Options: +- Relaunch for failed hosts +- Manual remediation +- Skip failed hosts + +AAP URL: https://aap.example.com/#/jobs/playbook/1235 +``` + +### Failure Template + +```markdown +❌ Playbook Execution Failed + +Job ID: 1235 +Duration: 1m 15s +Systems Remediated: 0 of 3 + +Critical errors prevented execution. +See error details above for troubleshooting. + +AAP URL: https://aap.example.com/#/jobs/playbook/1235 +``` diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/02-error-handling-guide.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/02-error-handling-guide.md new file mode 100644 index 00000000..90492f00 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/02-error-handling-guide.md @@ -0,0 +1,108 @@ +# Step 02: Error Handling Guide + +Read this reference when generating Phase 6 error reports or troubleshooting. + +## Error Categories + +**Parse error output** from `jobs_stdout_retrieve` for these common patterns: + +1. **Connection Failures**: SSH timeout, host unreachable, authentication failed +2. **Permission Errors**: sudo required, insufficient privileges, SELinux denials +3. **Package Manager Issues**: repo unavailable, package not found, dependency conflicts +4. **Service Failures**: service not found, restart failed, timeout +5. **Disk Space**: insufficient space for updates +6. **General Failures**: playbook syntax errors, task failures + +## Error Report Template + +```markdown +# Playbook Execution Failed + +## Job Summary +**Job ID**: 1235 +**Status**: ❌ Failed +**Duration**: 2m 45s +**Started**: 2026-02-24 15:35:02 UTC +**Failed At**: 2026-02-24 15:37:47 UTC +**Job Template**: CVE Remediation Template +**AAP URL**: [View in AAP](https://aap.example.com/#/jobs/playbook/1235) + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| prod-web-01 | 8 | 3 | 0 | 0 | ✅ Success | +| prod-web-02 | 8 | 3 | 0 | 0 | ✅ Success | +| prod-web-03 | 5 | 0 | 1 | 0 | ❌ Failed | + +**Summary**: 2 of 3 hosts succeeded, 1 failed + +## Failed Tasks Details + +### Host: prod-web-03 + +**Task**: Restart httpd service +**Error**: "Failed to restart httpd.service: Unit httpd.service not found." + +**Error Category**: Service Failure + +**Root Cause**: The httpd service is not installed or not recognized by systemd. + +**Troubleshooting Steps**: +1. Check if httpd is installed: + ```bash + ssh prod-web-03 'rpm -q httpd' + ``` +2. If not installed, the package update may have failed: + ```bash + ssh prod-web-03 'dnf info httpd' + ``` +3. Check systemd service status: + ```bash + ssh prod-web-03 'systemctl status httpd' + ``` +4. Review package manager logs: + ```bash + ssh prod-web-03 'tail -50 /var/log/dnf.log' + ``` + +**Recommended Action**: +- Verify httpd package installation on prod-web-03 +- Check if package update completed successfully +- Manually install httpd if needed: `dnf install httpd` +- Relaunch job for failed host only + +## Console Output (Last 50 Lines) +
+Click to expand error context + +[Relevant error output from jobs_stdout_retrieve] + +
+ +## Relaunch Options + +Would you like to: +1. **Relaunch for failed hosts only** - Run job again with limit="prod-web-03" +2. **Fix issues manually and relaunch** - Resolve problems first, then relaunch +3. **View full job output** - See complete execution logs +4. **Abort** - Stop remediation workflow + +Please choose an option (1-4): +``` + +## Relaunch Parameters + +**MCP Tool**: `jobs_relaunch_retrieve` (from aap-mcp-job-management) + +**Parameters**: +```json +{ + "id": "1235", + "requestBody": { + "hosts": "failed", + "job_type": "run" + } +} +``` + +This relaunches the job for only the failed hosts. diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/03-workflow-examples.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/03-workflow-examples.md new file mode 100644 index 00000000..f5caa0d1 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/03-workflow-examples.md @@ -0,0 +1,119 @@ +# Step 03: Workflow Examples + +Read this reference when demonstrating end-to-end workflow patterns. + +## Example 1: Full Workflow with Dry-Run + +**User Request**: "Execute the CVE-2025-49794 remediation playbook" + +**Skill Response**: + +1. **Validate AAP Prerequisites**: + - Invoke mcp-aap-validator skill → PASSED + +2. **List Job Templates**: + - Call `job_templates_list()` → Found 2 templates + - Filter compatible templates → 1 matches requirements + +3. **User Selects Template**: + ``` + Found 1 compatible job template: + 1. "CVE Remediation Template" (ID: 10) + + Select template (1) or "create" for new: 1 + ``` + +4. **Playbook Preparation**: + ``` + Guide user to add playbook to Git: + - Commands provided + - User syncs AAP project + - Verification: Playbook available ✓ + ``` + +5. **Offer Dry-Run**: + ``` + Run dry-run first? yes + ``` + +6. **Execute Dry-Run**: + - Launch with `job_type="check"` + - Monitor progress → COMPLETED + - Display dry-run results: + ``` + Would change 3 tasks on 3 hosts + No errors detected + ``` + +7. **Proceed to Actual Execution**: + ``` + Proceed with actual execution? yes + ``` + +8. **Execute Playbook**: + - Launch with `job_type="run"` + - Monitor progress → COMPLETED + - Generate report: + ``` + ✅ 3 of 3 hosts successfully remediated + Duration: 5m 23s + ``` + +9. **Suggest Verification**: + ``` + Recommend: Run remediation-verifier skill to confirm CVE resolution + ``` + +## Example 2: Handle Execution Failure + +**User Request**: "Execute remediation playbook" + +**Skill Response**: + +1-7. [Same as Example 1 through execution] + +8. **Execute Playbook**: + - Launch job → FAILED on 1 of 3 hosts + - Generate error report: + ``` + ❌ prod-web-03 failed + Error: httpd service not found + Troubleshooting steps provided + ``` + +9. **Offer Relaunch**: + ``` + Relaunch for failed host only? yes + ``` + +10. **Relaunch Job**: + - Call `jobs_relaunch_retrieve` with hosts="failed" + - Monitor → COMPLETED + - Final report: + ``` + ✅ All 3 hosts successfully remediated (1 after retry) + ``` + +## Example 3: Skip Dry-Run + +**User Request**: "Execute playbook directly, skip dry-run" + +**Skill Response**: + +1-4. [Same as Example 1 through template selection] + +5. **Offer Dry-Run**: + ``` + Run dry-run first? no + ``` + +6. **Final Confirmation**: + ``` + ⚠️ Execute on production without dry-run? + This will make changes immediately. + Confirm: yes + ``` + +7. **Execute Playbook**: + - Launch with `job_type="run"` + - Monitor and report as in Example 1 diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/04-dry-run-display-templates.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/04-dry-run-display-templates.md new file mode 100644 index 00000000..d9b6f0dc --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/04-dry-run-display-templates.md @@ -0,0 +1,93 @@ +# Step 04: Dry-Run Display Templates + +Read this reference when displaying Phase 3 dry-run content. + +## Playbook Preview + +```markdown +# Playbook Preview + +**Playbook**: remediation-CVE-2025-49794.yml +**Target Systems**: 5 systems + +## Tasks Overview: +1. **Gather Facts** - Collect system information +2. **Check Disk Space** - Ensure sufficient space for updates (>500MB) +3. **Backup Configuration** - Snapshot critical configs +4. **Update Package: httpd** - Upgrade to version 2.4.57-8.el9 +5. **Restart Service: httpd** - Apply changes +6. **Verify Service Status** - Confirm httpd is running +7. **Update Audit Log** - Record remediation event + +**Estimated Duration**: 3-5 minutes per system +**Requires Reboot**: No +**Downtime**: Brief (~10 seconds during service restart) +``` + +## Dry-Run Offer + +``` +⚠️ Recommended: Run dry-run first + +Dry-run mode (--check) simulates changes without applying them. +This helps identify: +- Package availability issues +- Permission problems +- Configuration conflicts +- Unexpected side effects + +❓ Run dry-run before actual execution? +- "yes" - Run dry-run first (recommended) +- "no" - Skip to actual execution +- "abort" - Cancel execution + +Please respond with your choice. +``` + +## Dry-Run Results Display + +```markdown +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s +**Completed**: 2024-01-20 15:32:17 UTC + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| prod-web-01 | 3 | 8 | 0 | ✓ Ready | +| prod-web-02 | 3 | 8 | 0 | ✓ Ready | +| prod-web-03 | 3 | 8 | 0 | ✓ Ready | + +## Changes That Would Be Made: +1. **httpd package** - Would update from 2.4.53-7.el9 to 2.4.57-8.el9 +2. **httpd service** - Would restart +3. **audit log** - Would add remediation entry + +## Dry-Run Output: +
+Click to expand full output + +[Full stdout from jobs_stdout_retrieve] + +
+ +✓ No errors detected in dry-run +✓ All systems passed pre-flight checks +``` + +## Proceed to Actual Execution Prompt + +``` +❓ Dry-run completed successfully. Proceed with actual execution? + +Options: +- "yes" or "execute" - Proceed with actual remediation +- "review" - Show dry-run output again +- "abort" - Cancel execution + +Please respond with your choice. +``` diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/05-git-flow-prompts.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/05-git-flow-prompts.md new file mode 100644 index 00000000..41945d0e --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/05-git-flow-prompts.md @@ -0,0 +1,97 @@ +# Step 05: Git Flow Prompts + +Read this reference when executing Git Flow (Scenario 1 Override or Scenario 2). + +## Scenario 1 Prompt (Same path) + +The template already points to our playbook path. The project may need the latest content. + +``` +Found template [name] (ID: X) with matching playbook path. The project may need to be updated with the latest playbook. + +Options: +(A) Override: I'll add/update the playbook in the project via git. You sync the AAP project, then confirm. +(B) Manual: You add the playbook and sync. Confirm when done. + +❓ Choose (A) or (B): +``` + +- **If A**: Execute Git Flow (see Git Flow section below). Wait for user: "Sync complete" or "done". +- **If B**: Wait for user confirmation. + +## Scenario 2 Prompt (Different path) + +**CRITICAL**: The template points to a DIFFERENT playbook than our generated playbook. You MUST NOT launch the job without Git Flow. AAP executes from synced project content—there is no "override at launch". The playbook MUST be in the repo before any job launch. + +``` +Found template [name] (ID: X) pointing to [template.playbook]. Our generated playbook is [our_playbook_path]. + +⚠️ The template's playbook path does NOT match. We must update the playbook in the project before execution. + +Options: +- "yes" or "proceed" - I'll add our playbook to the project via git (you'll confirm commit/push, then sync AAP) +- "no" - Create a new template via `/job-template-creator` skill + +❓ Proceed with playbook update (git flow)? +``` + +- **If yes**: Execute Git Flow. **Do NOT proceed to Phase 3 until Git Flow completes.** +- **If no**: Fall through to Scenario 3 (job-template-creator). + +## Repo Path Question + +``` +What is the local path to the Git repository for project [Project Name] (scm_url)? +``` + +Use `projects_list` to get project name and `scm_url`; display to help user identify the repo. + +**Path format**: Ask for the **absolute path** (e.g. `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project`). When writing the playbook, the Write tool path MUST be `/playbooks/remediation/` — the full absolute path. Do NOT use a relative path like `test-aap-project/playbooks/...`; that causes "Error writing file". + +## Git Flow: Write Step (FAST) + +**CRITICAL**: The playbook is ALREADY generated. During Git Flow you must WRITE the existing content to disk—nothing more. + +- **DO**: Single file write of the playbook content already in context (from playbook-generator or remediation) +- **DO NOT**: Invoke playbook-generator again, call create_vulnerability_playbook, re-fetch from MCP, or validate/transform the content +- **Expected duration**: Seconds. If it takes minutes, you are doing unnecessary work. + +### Write Path (ABSOLUTE REQUIRED) + +**⚠️ WRITE PATH MUST START WITH `/`** — The Write tool path MUST be an absolute path. Relative paths cause "Error writing file" because the repo is often outside the workspace. + +**Formula**: `write_path = user_provided_path + "/" + target_path` + +- `user_provided_path` = exactly what the user typed (e.g. `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project`) +- `target_path` = e.g. `playbooks/remediation/cve-remediation.yml` + +**Correct**: `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project/playbooks/remediation/cve-remediation.yml` + +**WRONG** (will fail): +- `test-aap-project/playbooks/remediation/cve-remediation.yml` +- `playbooks/remediation/cve-remediation.yml` + +**Before calling Write**: Verify the path starts with `/`. If it does not, prepend the user's repo path. + +## Git Flow HITL Checkpoint + +**REQUIRED** before commit/push: + +``` +Ready to commit and push these changes? +- File: [target_path] +- CVE: [cve_id] +- This will update the playbook in the AAP project. + +Reply 'yes' or 'proceed' to continue, or 'abort' to cancel. +``` + +**Wait for user confirmation.** If "yes" or "proceed": `git commit -m "Add/update remediation playbook for CVE-YYYY-NNNNN"` then `git push origin main`. + +## After Push Message + +``` +I've pushed the playbook. Sync the AAP project: Automation Execution > Projects > [Project] > Sync. Reply 'sync complete' when done. +``` + +**Do NOT proceed to Phase 3 (Dry-Run) until user confirms sync complete.** diff --git a/evaluation/with_skills/rh-sre__playbook-executor/instruction.md b/evaluation/with_skills/rh-sre__playbook-executor/instruction.md new file mode 100644 index 00000000..5cced969 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/instruction.md @@ -0,0 +1,18 @@ +# Playbook Execution Task + +You are a Red Hat SRE. A remediation playbook needs to be executed against production systems through Ansible Automation Platform. You are responsible for the safe execution and monitoring of this job. + +## Scenario +A CVE remediation playbook has been prepared and a job template exists in AAP. You need to execute it safely: validate the template first, consider running a dry-run, launch the production job, monitor its progress, and report the results. + +## Requirements +- Locate and validate the job template in AAP (check it has the right inventory, project, credentials, and privilege escalation) +- Document a pre-flight checklist: template readiness, target hosts, and any prerequisites +- Plan the execution: whether to run a dry-run (check mode) first, how to monitor job progress, and what to do if it fails +- Launch the job (or document the launch procedure) and monitor its status +- Report per-host results: which hosts succeeded, which failed, and any error details +- Include guidance for handling failures (retry, rollback, escalation) + +Document your methodology, execution plan, and results in `/root/report.md`. + +Use MCP tools to interact with AAP. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-sre__playbook-executor/solution/solve.sh b/evaluation/with_skills/rh-sre__playbook-executor/solution/solve.sh new file mode 100644 index 00000000..090c2294 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/solution/solve.sh @@ -0,0 +1,21 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Playbook Execution Report + +## Execution Steps +1. Dry-run: job_type='check' (Ansible check mode) +2. Review results +3. Execute: job_type='run' + +## Git Flow +Playbook stored at playbooks/remediation/cve-2024-12345.yml. Commit, push, wait for sync complete before launch. No override at launch—AAP runs from synced project. + +## Job Template Validation +Invoke job-template-remediation-validator for each candidate template. + +## Execution Report +- Status: Success +- Systems patched: 4/4 +- Validate job log (jobs_stdout_retrieve) for CVE handling +- Suggest remediation-verifier after success +REPORT_EOF diff --git a/evaluation/with_skills/rh-sre__playbook-executor/task.toml b/evaluation/with_skills/rh-sre__playbook-executor/task.toml new file mode 100644 index 00000000..eaa9b790 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__playbook-executor" +name = "rh-sre Playbook Execution Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "playbook-executor", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-sre__playbook-executor/tests/llm_judge.py b/evaluation/with_skills/rh-sre__playbook-executor/tests/llm_judge.py new file mode 100644 index 00000000..15da24ed --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "launch_config_and_git_flow", "file": "/root/report.md", "question": "Does the report configure launch-time prompts for flexibility (variables, host limits, job type) and require Git synchronization before execution?", "reference": "A skilled report configures launch-time prompts and requires Git sync. An unskilled report hardcodes execution settings and skips synchronization requirements."}, + {"id": "relaunch_failed_hosts", "file": "/root/report.md", "question": "Does the report mention relaunching with hosts: failed to retry only failed hosts?", "reference": "A skilled report uses jobs_relaunch_retrieve with hosts: failed. An unskilled report suggests full re-execution."}, + {"id": "dry_run_and_monitoring", "file": "/root/report.md", "question": "Does the report recommend dry-run first and include per-host execution monitoring?", "reference": "A skilled report follows check mode before run and monitors per-host. An unskilled report runs directly without dry-run."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-sre__playbook-executor/tests/test.sh b/evaluation/with_skills/rh-sre__playbook-executor/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-sre__playbook-executor/tests/test_outputs.py b/evaluation/with_skills/rh-sre__playbook-executor/tests/test_outputs.py new file mode 100644 index 00000000..dab37078 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-executor/tests/test_outputs.py @@ -0,0 +1,89 @@ +""" +Tests for rh-sre__playbook-executor per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['playbook', 'execut', 'job']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_git_flow_mandatory(self): + """Skill: When template playbook path differs from generated playbook, Git Flow (commit, push, sync) is MANDATORY before launch.""" + c = read_report().lower() + has_git = any(t in c for t in ["git", "commit", "push", "sync"]) + has_block = any(t in c for t in ["before launch", "mandatory", "must", "block", "sync complete"]) + assert has_git or has_block, ( + "should require Git Flow when path differs (skill: no override at launch)" + ) + + def test_launch_configuration(self): + """Skill teaches configuring launch-time prompts for execution flexibility + (job type, variables, host limiting). Without skill, agents run playbooks + with hardcoded settings.""" + c = read_report().lower() + has_launch = any(t in c for t in ["launch", "prompt", "on launch"]) + has_config = any(t in c for t in [ + "variable", "limit", "job type", "configur", + ]) + assert has_launch and has_config, ( + "should configure launch-time prompts for execution flexibility" + ) + + def test_relaunch_failed_hosts(self): + """Skill: jobs_relaunch_retrieve with hosts: 'failed' to retry only failed hosts.""" + c = read_report().lower() + assert any(t in c for t in ["relaunch", "failed hosts", "retry failed"]), ( + "should mention relaunch for failed hosts (skill: jobs_relaunch_retrieve)" + ) + + def test_dry_run_first(self): + """Skill: Recommend dry-run (check mode) before production execution.""" + c = read_report().lower() + assert any(t in c for t in ["dry", "check mode", "check_mode", "preview", "before launch"]), ( + "should recommend dry-run first (skill: Phase 3)" + ) + + def test_per_host_results(self): + """Skill: Report per-host results (succeeded, failed, error details).""" + c = read_report().lower() + has_per_host = any(t in c for t in ["per host", "each host", "host result", "stdout", "host summary"]) + has_ansible_outcome = any(t in c for t in ["succeeded", "failed", "unreachable", "skipped", "changed"]) + assert has_per_host or has_ansible_outcome, ( + "should report per-host execution results (skill: host summaries)" + ) + + def test_error_taxonomy(self): + """Docs teach error taxonomy: connection/permissions/package/service/disk + failure categories with specific recovery paths. + Without docs, agents treat all errors generically.""" + c = read_report().lower() + categories = ["connection", "permission", "package", "service", "disk"] + mentioned = sum(1 for cat in categories if cat in c) + assert mentioned >= 2, ( + "should categorize errors by type (connection/permissions/package/service/disk)" + ) diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/Dockerfile b/evaluation/with_skills/rh-sre__playbook-generator/environment/Dockerfile new file mode 100644 index 00000000..484ebb33 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/Dockerfile @@ -0,0 +1,52 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + + + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..1092dd1d --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- pass rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/with_skills/rh-sre__playbook-generator/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..2269a235 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,722 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2026-1234": { + "cve_id": "CVE-2026-1234", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Critical kernel vulnerability: remote code execution in kernel network stack allows unauthenticated attackers to execute arbitrary code via crafted packets", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2026-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 8, + "total_remediated": 2, + "total_vulnerable": 6, + }, + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/skills/playbook-generator/SKILL.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/skills/playbook-generator/SKILL.md new file mode 100644 index 00000000..a9234cdd --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/environment/skills/playbook-generator/SKILL.md @@ -0,0 +1,377 @@ +--- +name: playbook-generator +description: | + **CRITICAL**: This skill ONLY GENERATES playbooks. It does NOT EXECUTE them. For execution, use /playbook-executor skill. + + Generate production-ready Ansible remediation playbooks for CVE vulnerabilities with Red Hat best practices, error handling, and Kubernetes safety patterns. Use this skill when you need to create remediation playbooks that follow Red Hat Lightspeed patterns and incorporate RHEL-specific considerations. + + This skill calls the MCP tool (remediations__create_vuln_playbook) and returns the playbook **AS IS**. Do NOT modify, enhance, or add to the generated playbook. Any change requires explicit user validation first. + + **IMPORTANT**: + - ALWAYS use this skill instead of calling create_vulnerability_playbook directly + - NEVER execute playbooks using ansible-playbook CLI + - ALWAYS delegate execution to /playbook-executor skill +--- + +# Ansible Playbook Generator Skill + +This skill generates Ansible remediation playbooks for CVE vulnerabilities, applying Red Hat best practices, RHEL-specific patterns, and Kubernetes safety considerations. + +**Integration with Remediation Skill**: The `/remediation` skill orchestrates this skill as part of its Step 4 (Generate Playbook) workflow. For standalone playbook generation, you can invoke this skill directly. + +## When to Use This Skill + +**🚨 CRITICAL SCOPE LIMITATION**: This skill **ONLY GENERATES** playbooks. It does **NOT EXECUTE** them. + +**Use this skill directly when you need**: +- Generate a remediation playbook for a specific CVE +- Create batch remediation playbooks for multiple CVEs +- Get a remediation playbook from Red Hat Lightspeed (returned unmodified) +- Standalone playbook generation without full remediation workflow + +**Do NOT use this skill when you need**: +- "Create playbook and execute it" → Use `/remediation` skill (orchestrates this skill + playbook-executor) +- "Remediate CVE-X" (full workflow) → Use `/remediation` skill +- Execute playbooks → Use `/playbook-executor` skill instead +- Run ansible-playbook CLI → Use `/playbook-executor` skill via AAP MCP +- Monitor job execution → Use `/playbook-executor` skill instead + +**Use the `/remediation` skill when you need**: +- End-to-end CVE remediation (analysis → validation → playbook → execution → verification) +- Integrated impact analysis before playbook generation +- System context gathering and remediation strategy determination +- Execution guidance and verification workflows + +**How they work together**: +1. The `/remediation` skill orchestrates this skill after gathering system context +2. This skill generates the optimized playbook +3. The remediation skill then invokes `/playbook-executor` for execution via AAP MCP +4. Finally, `/remediation-verifier` confirms success + +## Workflow + +**🚨 Tool Failure Rule**: If `create_vulnerability_playbook` (or `create_vuln_playbook`) fails, STOP. Present options (retry / generate from knowledge with user confirmation / exit). Never auto-generate from your knowledge. + +### 1. Playbook Generation (MCP Tool) + +**MCP Tool**: `create_vulnerability_playbook` or `remediations__create_vulnerability_playbook` (from lightspeed-mcp) + +**Parameters** (tool may use `cve_ids`/`cves` and `system_ids`/`uuids`—check tool schema): +- `cves` or `cve_ids`: Array of CVE identifiers + - Example: `["CVE-2024-1234"]` + - Format: CVE-YYYY-NNNNN strings +- `uuids` or `system_ids`: Array of system UUIDs from Red Hat Lightspeed inventory + - Example: `["uuid-1", "uuid-2"]` + - Format: UUID strings (get from system-context skill) +- `playbook_name`: Name for the playbook (if required by tool) + +**Expected Output**: Ansible playbook YAML from Red Hat Lightspeed. + +**CRITICAL — Return AS IS**: You MUST return the playbook exactly as the MCP tool provides it. Do NOT add pre-flight checks, backups, service restarts, audit logging, or any other modifications. The MCP tool description states: "Don't process the playbook. You MUST return the YAML as is." Any enhancement requires explicit user approval—offer modifications only after user requests them. + +#### When MCP Tool Fails (REQUIRED Error Handling) + +**🚨 CRITICAL**: When `create_vulnerability_playbook` (or `remediations__create_vulnerability_playbook`) returns an error, you MUST NOT generate a playbook from your own knowledge. Stop and present options to the user. + +**If the tool returns error** (e.g., "Unhandled error", timeout, 500, connection failure): + +1. **Report the failure** to the user with the error message +2. **Present these options** and wait for explicit user choice: + +``` +❌ Red Hat Lightspeed playbook generation failed: [error message] + +**Next steps** (choose one): + +(A) **Retry** - Try the MCP tool again (may succeed if transient) +(B) **Generate from knowledge** - Create a playbook using documentation templates (⚠️ NOT from Red Hat Lightspeed; requires your explicit approval) +(C) **Exit** - Stop playbook generation; user can retry later or use manual remediation + +❓ Reply with A, B, or C: +``` + +3. **Execute based on user choice**: + - **A (Retry)**: Call the MCP tool again. If it fails again, present options again (limit retries to 2; after 2 failures, present B and C only) + - **B (Generate from knowledge)**: ONLY proceed if user explicitly chose B. Use documentation (cve-remediation-templates.md, package-management.md) to build a playbook. Add disclaimer: "Generated from documentation templates—Red Hat Lightspeed API was unavailable. Review carefully before execution." + - **C (Exit)**: Stop. Do not generate any playbook. Suggest: "You can retry later when Lightspeed MCP is available, or create a manual remediation playbook." + +**NEVER** auto-generate a playbook from your knowledge when the tool fails without explicit user confirmation for option B. + +### 5. Return Playbook AS IS (No Modifications) + +**CRITICAL**: Return the playbook exactly as the MCP tool provides it. Do NOT add, remove, or modify any content. + +**Do NOT**: +- Add pre-flight checks (RHEL validation, subscription check) +- Add backup/snapshot creation +- Add service restart logic +- Add audit logging +- Add Kubernetes pod eviction +- Replace or wrap the MCP output with documentation templates + +**If the user requests enhancements** (e.g. "add pre-flight checks", "add backup step"): +1. Show the original playbook first +2. Ask: "The playbook above is from Red Hat Lightspeed. You requested [enhancement]. Should I create a modified version with these additions? (yes/no)" +3. Only if user confirms "yes", create a modified version and show the diff +4. Require explicit approval before any modified playbook is used + +### 6. Playbook Validation (Minimal) + +Before returning, verify only: +- YAML is returned (the MCP tool output) +- No modifications were applied + +Do NOT validate for "best practices" or add missing elements—return AS IS. + +## Critical: Human-in-the-Loop Requirements + +This skill generates code that will execute on production systems. **Explicit user confirmation is REQUIRED** before returning the playbook. + +**When MCP Tool Fails** (REQUIRED): +- Do NOT generate a playbook from your own knowledge without explicit user confirmation +- Present options: (A) Retry, (B) Generate from knowledge (requires user approval), (C) Exit +- Wait for user to choose A, B, or C before proceeding +- If user chooses B: Add disclaimer that playbook was generated from documentation, not Red Hat Lightspeed + +**Before Playbook Return** (REQUIRED): +1. **Display Playbook Preview**: Show complete playbook YAML to user +2. **Display Metadata**: Show CVE IDs, target systems, reboot requirements, Kubernetes considerations +3. **Ask for Confirmation**: + ``` + ❓ Review the playbook above. This playbook will: + - Update packages on N systems + - Require reboot: [Yes/No] + - Affect Kubernetes pods: [Yes/No] + + Should I provide this playbook for execution? + + Options: + - "yes" or "proceed" - Provide playbook for execution + - "modify" - Request changes to playbook + - "abort" - Cancel playbook generation + + Please respond with your choice. + ``` +4. **Wait for Explicit Confirmation**: Do not provide playbook without "yes" or "proceed" + +**Never assume approval** - always wait for explicit user confirmation before providing executable playbooks. + +### 7. Return Playbook + +**🚨 CRITICAL**: This skill **ONLY GENERATES** playbooks. It does **NOT EXECUTE** them. + +**ONLY after receiving explicit user confirmation**, return the production-ready playbook with metadata: + +```yaml +# Playbook metadata to return: +playbook: + file: remediation-CVE-YYYY-NNNNN.yml + path: playbooks/remediation/remediation-CVE-YYYY-NNNNN.yml # Full path for playbook-executor template matching + content: | + [Complete YAML playbook] + + metadata: + cve_ids: ["CVE-YYYY-NNNNN"] + target_systems: ["uuid-1", "uuid-2"] + rhel_versions_supported: ["7", "8", "9"] + requires_reboot: true/false + kubernetes_safe: true/false + estimated_duration_minutes: 15 + risk_level: "medium" # based on reboot requirement + + execution_notes: + - "Test in staging environment first" + - "Schedule maintenance window if reboot required" + - "Ensure kubectl access if Kubernetes systems" + - "Back up critical data before execution" +``` + +## Critical: Execution Handoff + +**🚨 THIS SKILL DOES NOT EXECUTE PLAYBOOKS** + +After generating the playbook, if the user requests execution: + +❌ **WRONG** - Do NOT use `ansible-playbook` CLI: +```bash +ansible-playbook remediation.yml --check # ❌ This skill cannot do this +``` + +✅ **CORRECT** - Delegate to the `/playbook-executor` skill: +```markdown +I've generated the remediation playbook. To execute it in dry-run mode, I'll invoke the playbook-executor skill: + +[Invoke /playbook-executor skill with the playbook content] +``` + +**When user asks to execute**: +1. Save the playbook to a file (if needed for reference) +2. Invoke `/playbook-executor` skill with instruction: + ``` + "Execute this playbook for CVE-XXXX-YYYY in dry-run mode using AAP job template [ID]. Monitor job status and report results." + ``` +3. The playbook-executor skill handles all execution via AAP MCP tools + +**Never attempt to**: +- Run `ansible-playbook` command directly +- Execute playbooks via Shell/Bash tool +- Use any local Ansible execution method + +**Always delegate execution to** `/playbook-executor` skill. + +## Output Template + +When completing playbook generation, provide output in this format: + +```markdown +# Remediation Playbook Generated + +## CVE Information +**CVE ID**: CVE-YYYY-NNNNN +**Target Systems**: N systems +**RHEL Versions**: 7, 8, 9 +**Requires Reboot**: Yes/No +**Kubernetes Safe**: Yes/No + +## Playbook Features +✓ Generated by Red Hat Lightspeed (returned AS IS, no modifications) + +## Playbook File: remediation-CVE-YYYY-NNNNN.yml + +```yaml +[Complete playbook YAML] +``` + +## Next Steps: Execution + +**🔴 IMPORTANT**: Do NOT execute this playbook using `ansible-playbook` CLI. + +**✅ To execute this playbook**, invoke the `/playbook-executor` skill: + +```markdown +Ready to execute? The playbook-executor skill will: +1. Add this playbook to your AAP Git project +2. Create/use an AAP job template +3. Execute in dry-run mode first (if requested) +4. Launch actual execution (with your approval) +5. Monitor job status and report results + +Would you like me to invoke the playbook-executor skill now? +Options: +- "yes" or "execute" - Invoke playbook-executor skill +- "dry-run first" - Execute in check mode first +- "save only" - Just save the playbook file for later +``` + +**Execution Flow**: +1. **This skill** → Generates playbook (DONE ✓) +2. **playbook-executor skill** → Executes via AAP MCP tools +3. **remediation-verifier skill** → Verifies success after execution + +**Safety Notes**: +- Playbook is from Red Hat Lightspeed—review before execution +- No modifications were applied; user may request enhancements separately +``` + +## Examples + +### Example 1: Simple CVE + +**User Request**: "Generate playbook for CVE-2024-1234 on 5 RHEL 8 systems" + +**Skill Response**: +1. Call `remediations__create_vuln_playbook` with cves, uuids, playbook_name +2. Return the playbook **exactly as received**—no modifications +3. Ask for user confirmation before handoff to playbook-executor + +### Example 2: Batch CVEs + +**User Request**: "Generate playbook for CVE-2024-1234, CVE-2024-5678 on 20 systems" + +**Skill Response**: +1. Call `remediations__create_vuln_playbook` with multiple CVE IDs and system UUIDs +2. Return the playbook **exactly as received**—no modifications +3. Ask for user confirmation before handoff to playbook-executor + +## Error Handling + +**CVE has no automated remediation**: +``` +CVE-YYYY-NNNNN does not have an automated remediation playbook available in Red Hat Lightspeed. + +Manual remediation required: +1. Affected packages: package-name-version +2. Recommended action: dnf update package-name +3. Verification: package-name --version + +Would you like me to create a manual playbook template based on Red Hat best practices? +``` + +**Unsupported RHEL version**: +``` +Target systems include RHEL 6, which is not supported by this skill. + +Supported RHEL versions: 7, 8, 9 + +Please filter target systems to supported versions or consult Red Hat documentation for RHEL 6 remediation guidance. +``` + +**Kubernetes context missing**: +``` +Target systems appear to be Kubernetes nodes but kubectl access is not configured. + +To generate Kubernetes-safe playbooks, ensure: +1. kubectl is installed and configured +2. Access to cluster is available +3. Appropriate RBAC permissions for node operations + +Proceeding with standard playbook (without pod eviction). Add pod eviction manually if needed. +``` + +## Dependencies + +### Required MCP Servers +- `lightspeed-mcp` - Red Hat Lightspeed platform access + +### Required MCP Tools +- `remediations__create_vuln_playbook` (from lightspeed-mcp) - Generate remediation playbook from Red Hat Lightspeed + - Parameters: playbook_name, cves (array), uuids (array of system UUIDs) + - Returns: Ansible playbook YAML—**return AS IS**, do not modify + +### Related Skills +- `cve-impact` - Provides CVE severity and risk assessment to inform playbook complexity +- `system-context` - Provides system inventory and deployment context for playbook targeting +- `remediation-verifier` - Verifies playbook execution success after deployment +- `playbook-executor` - Executes generated playbooks and tracks job status + +### Reference Documentation +- [cve-remediation-templates.md](../../docs/ansible/cve-remediation-templates.md) - Ansible playbook templates for different CVE types +- [package-management.md](../../docs/rhel/package-management.md) - RHEL package management best practices (DNF vs YUM, reboot detection) + +## Best Practices + +1. **🚨 NEVER EXECUTE PLAYBOOKS** - This skill generates only. Always delegate execution to `/playbook-executor` skill +2. **🚨 RETURN AS IS** - Do NOT modify the MCP-generated playbook. No enhancements without explicit user request and approval +3. **🚨 NEVER auto-generate on tool failure** - When the MCP tool fails, present options (retry / generate from knowledge with user confirmation / exit). Do NOT silently generate from your own knowledge +4. **Require user approval** - ALWAYS get explicit confirmation before providing playbooks for execution +5. **Clear handoff** - After generation, explicitly tell user to invoke `/playbook-executor` for execution + +## Tools Reference + +This skill uses: +- `remediations__create_vuln_playbook` (from lightspeed-mcp) - Generate playbook from Red Hat Lightspeed. Returns YAML **as is**—do not modify. + +All MCP tools are provided by the lightspeed-mcp server configured in `.mcp.json`. + +## Integration with Other Skills + +- **cve-impact**: Provides CVE severity and risk assessment to inform playbook complexity +- **system-context**: Provides system inventory and deployment context for playbook targeting +- **remediation-verifier**: Verifies playbook execution success after deployment + +**Orchestration Example** (from `/remediation` skill): +1. Agent invokes cve-impact skill → Gets risk assessment +2. Agent gathers context → Determines deployment requirements +3. Agent invokes playbook-generator skill → Generates production-ready playbook +4. Agent provides execution guidance → User deploys playbook +5. Agent invokes remediation-verifier skill → Confirms success diff --git a/evaluation/with_skills/rh-sre__playbook-generator/instruction.md b/evaluation/with_skills/rh-sre__playbook-generator/instruction.md new file mode 100644 index 00000000..585c6f73 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/instruction.md @@ -0,0 +1,17 @@ +# Playbook Generation Task + +You are a Red Hat SRE. A critical CVE has been identified affecting systems in your fleet. You need to generate a remediation playbook that can be used to patch the affected hosts. + +## Scenario +CVE-2026-1234 has been confirmed as a critical kernel vulnerability affecting multiple production RHEL systems. You need to generate an Ansible playbook that remediates this CVE on the affected hosts. + +## Requirements +- Use available tools to generate a remediation playbook for the CVE +- Review the generated playbook content: what packages it updates, whether it requires a reboot, and any risk factors +- Document the playbook metadata: target CVE, affected systems, reboot requirements, and delegation safety notes +- If playbook generation fails, document the failure and describe alternative approaches +- The playbook should be ready to hand off for execution (do not execute it yourself) + +Document the generated playbook and your analysis in `/root/report.md`. You MUST write the report file — do not just display the content. + +Use MCP tools to interact with the environment. If reference documentation or skills are available in this environment, consult them before beginning work. Complete the entire analysis autonomously — do not stop after preliminary steps. Proceed through playbook generation, review, and report writing without waiting for user input. diff --git a/evaluation/with_skills/rh-sre__playbook-generator/solution/solve.sh b/evaluation/with_skills/rh-sre__playbook-generator/solution/solve.sh new file mode 100644 index 00000000..2543cf93 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/solution/solve.sh @@ -0,0 +1,38 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Playbook Generation Report + +## Methodology +Used `create_vulnerability_playbook` MCP tool (remediations endpoint via Lightspeed) to generate the remediation playbook for CVE-2026-1234. The playbook is returned AS IS — unmodified from the generation tool output. No pre-flight, backup, or restart steps were added. + +## Generated Playbook (returned AS IS, unmodified) +```yaml +- hosts: affected_systems + tasks: + - block: + - name: Create boom snapshot + command: boom create --title "pre-cve-{{ cve_id }}" + - name: Apply patch + dnf: + name: '*' + state: latest + rescue: + - name: Rollback + command: boom rollback + always: + - name: Check reboot needed + command: needs-restarting -r + register: needs_restarting +``` + +## Key Patterns +- block/rescue/always for error handling +- needs-restarting -r for reboot detection (RHEL 8/9) +- boom create for kernel/snapshot before remediation + +## Failure Handling +If the create_vulnerability_playbook tool fails, do not auto-generate a playbook from general knowledge. Present the user with options: (A) Retry the tool, (B) Generate from knowledge with explicit user approval, or (C) Exit and escalate. + +## Execution +Do NOT run ansible-playbook directly. Delegate execution to the playbook-executor skill/workflow. +REPORT_EOF diff --git a/evaluation/with_skills/rh-sre__playbook-generator/task.toml b/evaluation/with_skills/rh-sre__playbook-generator/task.toml new file mode 100644 index 00000000..1ea4f1ac --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__playbook-generator" +name = "rh-sre Playbook Generation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "playbook-generator", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-sre__playbook-generator/tests/llm_judge.py b/evaluation/with_skills/rh-sre__playbook-generator/tests/llm_judge.py new file mode 100644 index 00000000..05cd660f --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "mcp_tool_for_generation", "file": "/root/report.md", "question": "Does the report describe using an MCP tool (such as create_vulnerability_playbook or a remediations/Lightspeed endpoint) to generate the playbook, rather than writing it manually from scratch?", "reference": "A skilled report uses the Lightspeed MCP create_vulnerability_playbook tool. An unskilled report writes the playbook manually from general Ansible knowledge without using an MCP generation tool."}, + {"id": "return_as_is", "file": "/root/report.md", "question": "Does the report explicitly state that the generated playbook should be returned AS IS or unmodified, without adding extra steps like pre-flight checks, backup tasks, or restart handlers?", "reference": "A skilled report emphasizes returning the tool output unmodified. An unskilled report adds pre-flight checks, backup steps, restart handlers, or other enhancements to the generated playbook."}, + {"id": "delegation_not_execution", "file": "/root/report.md", "question": "Does the report explicitly state that playbook execution should be delegated to a separate execution workflow and NOT run directly via ansible-playbook?", "reference": "A skilled report delegates execution to a dedicated execution workflow rather than running ansible-playbook directly. An unskilled report runs ansible-playbook directly or doesn't address the execution boundary."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-sre__playbook-generator/tests/test.sh b/evaluation/with_skills/rh-sre__playbook-generator/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-sre__playbook-generator/tests/test_outputs.py b/evaluation/with_skills/rh-sre__playbook-generator/tests/test_outputs.py new file mode 100644 index 00000000..00518d36 --- /dev/null +++ b/evaluation/with_skills/rh-sre__playbook-generator/tests/test_outputs.py @@ -0,0 +1,74 @@ +""" +Tests for rh-sre__playbook-generator per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['playbook', 'generat', 'cve']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_mcp_tool_for_generation(self): + """Skill: Use create_vulnerability_playbook MCP tool, not manual playbook writing.""" + c = read_report().lower() + assert any(t in c for t in [ + "create_vulnerability_playbook", "create_vuln_playbook", + "remediations", "lightspeed", + ]) and any(t in c for t in ["tool", "mcp", "generat"]), ( + "should reference MCP tool usage for playbook generation (not manual writing)" + ) + + def test_no_modifications_to_playbook(self): + """Skill: Return playbook AS IS, no modifications—never add pre-flight, backup, restart.""" + c = read_report().lower() + assert any(t in c for t in [ + "as is", "as-is", "unmodified", "do not modify", "no modification", + "unchanged", "without modification", "returned unchanged", + "original output", "generated output", + ]), "should return playbook unmodified (skill: no enhancements without user approval)" + + def test_no_auto_generate_on_failure(self): + """Skill: Never auto-generate playbooks from general knowledge without approval.""" + c = read_report().lower() + has_constraint = any(t in c for t in [ + "do not auto", "never auto", "not auto-generat", + "without approval", "explicit approval", "user approval", + "do not generat", "never generat", + ]) + has_options = any(t in c for t in ["retry", "option", "escalat"]) + assert has_constraint or has_options, ( + "should state not to auto-generate playbooks without user approval" + ) + + def test_delegation_to_executor(self): + """Skill: This skill ONLY generates; execution delegated to playbook-executor.""" + c = read_report().lower() + assert any(t in c for t in [ + 'delegat', 'executor', 'playbook-executor', 'hand off', + 'not execute', 'do not run', 'do not execute', + 'not run ansible-playbook', 'not ansible-playbook', + ]), "should delegate execution (not run ansible-playbook directly)" diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/Dockerfile b/evaluation/with_skills/rh-sre__remediation-verifier/environment/Dockerfile new file mode 100644 index 00000000..484ebb33 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/Dockerfile @@ -0,0 +1,52 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + + + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..1092dd1d --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- pass rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/with_skills/rh-sre__remediation-verifier/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..e826c96e --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,759 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def _system_profile_for_host(host_type: str, rhel_version: str, sid: int) -> dict: + """Generate system_profile fields for a host based on type and RHEL version.""" + el = "el9" if rhel_version.startswith("9") else "el8" + kernel = f"5.14.0-362.24.1.{el}_3.x86_64" if "9" in rhel_version else f"4.18.0-477.27.1.{el}.x86_64" + base_pkgs = [ + {"name": "kernel-core", "version": f"5.14.0-362.24.1.{el}.x86_64"}, + {"name": "httpd", "version": f"2.4.57-5.{el}"}, + {"name": "sshd", "version": f"8.9p1-23.{el}"}, + {"name": "firewalld", "version": f"1.2.5-4.{el}"}, + {"name": "systemd", "version": f"250-19.{el}"}, + ] + if "web" in host_type or "lb" in host_type: + base_pkgs.extend([ + {"name": "nginx", "version": f"1.24.1-3.{el}"}, + {"name": "openssl", "version": f"3.0.7-24.{el}"}, + ]) + elif "db" in host_type: + base_pkgs.extend([ + {"name": "postgresql", "version": f"15.4-1.{el}"}, + {"name": "openssl", "version": f"3.0.7-24.{el}"}, + ]) + elif "mon" in host_type: + base_pkgs.extend([ + {"name": "prometheus", "version": f"2.45.0-1.{el}"}, + {"name": "node_exporter", "version": f"1.6.1-2.{el}"}, + ]) + else: + base_pkgs.extend([ + {"name": "java-17-openjdk", "version": f"17.0.8-4.{el}"}, + {"name": "openssl", "version": f"3.0.7-24.{el}"}, + ]) + services = ["sshd.service", "firewalld.service", "chronyd.service"] + if "web" in host_type or "lb" in host_type: + services.append("httpd.service") + elif "db" in host_type: + services.extend(["postgresql.service", "postgresql-15.service"]) + elif "mon" in host_type: + services.extend(["prometheus.service", "node_exporter.service"]) + else: + services.append("httpd.service") + ip_octet = 10 + (sid % 245) + mac_hex = f"{(sid % 256):02x}" + return { + "installed_packages": base_pkgs[:8], + "running_services": services, + "network_interfaces": [ + {"name": "eth0", "ipv4": [f"10.0.1.{ip_octet}"], "mac": f"52:54:00:a1:b2:{mac_hex}"}, + {"name": "lo", "ipv4": ["127.0.0.1"], "mac": "00:00:00:00:00:00"}, + ], + "kernel_version": kernel, + } + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + # Add system_profile to each host + for idx, s in enumerate(systems): + host_type = "app" # default + for ht in ["web", "db", "app", "lb", "mon", "cache"]: + if ht in s["id"]: + host_type = ht + break + s["system_profile"] = _system_profile_for_host( + host_type, s["rhel_version"], idx + 1 + ) + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/skills/remediation-verifier/SKILL.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/skills/remediation-verifier/SKILL.md new file mode 100644 index 00000000..e393f5ba --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/environment/skills/remediation-verifier/SKILL.md @@ -0,0 +1,399 @@ +--- +name: remediation-verifier +description: | + **CRITICAL**: This skill must be used for remediation verification. DO NOT use raw MCP tools like get_cve or get_host_details directly for verification. + + Verify CVE remediation success by checking Red Hat Lightspeed CVE status, validating package versions, and confirming service health. Use this skill after executing remediation playbooks to ensure vulnerabilities are properly fixed. + + This skill orchestrates MCP tools (get_cve, get_cve_systems, get_host_details) to provide comprehensive remediation verification including CVE status checking, package version validation, and service health confirmation. + + **IMPORTANT**: ALWAYS use this skill instead of calling verification MCP tools directly. +--- + +# Remediation Verification Skill + +This skill verifies CVE remediation success by validating that vulnerabilities have been properly fixed on target systems. + +**Integration with Remediation Skill**: The `/remediation` skill orchestrates this skill as part of its Step 6 (Verify Deployment) workflow. For standalone verification after manual remediation, you can invoke this skill directly. + +## When to Use This Skill + +**Use this skill directly when you need**: +- Verify CVE remediation after playbook execution +- Confirm package updates were applied successfully +- Check service health after remediation +- Validate Kubernetes pod recovery after node updates +- Generate verification reports for compliance + +**Use the `/remediation` skill when you need**: +- Full remediation workflow including verification +- Integrated remediation → execution → verification + +**How they work together**: The `/remediation` skill invokes this skill after the user executes the remediation playbook, providing final confirmation that the CVE is resolved. + +## Workflow + +### 1. CVE Status Verification + +**MCP Tool**: `get_cve` or `vulnerability__get_cve` (from lightspeed-mcp) + +**Parameters for get_cve**: +- `cve_id`: Exact CVE identifier (format: `"CVE-YYYY-NNNNN"`) + - Example: `"CVE-2024-1234"` +- `include_details`: `true` (retrieve complete metadata including remediation status) + +**Expected After Remediation**: +- CVE metadata still exists (CVE doesn't disappear from database) +- Remediation marked as applied +- Fixed version recorded + +**MCP Tool**: `get_cve_systems` or `vulnerability__get_cve_systems` (from lightspeed-mcp) + +**Parameters for get_cve_systems**: +- `cve_id`: Exact CVE identifier (format: `"CVE-YYYY-NNNNN"`) + - Example: `"CVE-2024-1234"` +- `limit`: Optional number of systems to return (default: all) + - Example: `100` +- `offset`: Optional pagination offset (default: 0) + - Example: `0` + +**Expected After Remediation**: +- Target systems removed from affected list OR +- Systems marked as "patched" status OR +- Systems show fixed package version + +**Verification Logic**: +``` +✓ System UUID not in affected systems list → PASS +✓ System status = "patched" → PASS +✗ System still in affected list with "vulnerable" status → FAIL +``` + +**Important**: Red Hat Lightspeed inventory updates may take time (up to 24 hours after remediation). Consider this timing when interpreting results. + +### 2. Package Version Verification + +**MCP Tool**: `get_host_details` or `inventory__get_host_details` (from lightspeed-mcp) + +**Parameters**: +- `system_id`: UUID of the system to verify (from Red Hat Lightspeed inventory) + - Example: `"uuid-1"` + - Format: UUID string (get from system-context skill or get_cve_systems result) +- `include_system_profile`: `true` (retrieve installed packages and service status) + - Example: `true` + +**Expected Output**: System details including: +- `system_profile.installed_packages` - List of installed RPM packages with versions +- `system_profile.enabled_services` - Services enabled at boot +- `system_profile.running_processes` - Currently running processes + +**Verification Workflow**: +``` +For each target system: + +1. Get current installed packages + Tool: get_host_details(system_id="uuid-1", include_system_profile=true) + Extract: system_profile.installed_packages + +2. Compare against expected fixed versions + CVE Fix Example: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 + + Installed Packages Check: + ✓ httpd-2.4.37-2.el8 (or newer) installed → PASS + ✗ httpd-2.4.37-1.el8 (old version) still present → FAIL + ✗ httpd not found → FAIL (package removed unexpectedly) + +3. Handle version comparison edge cases + - Epoch numbers (e.g., 1:httpd-2.4.37) + - Release suffixes (e.g., 2.4.37-2.el8_9.1) + - Architecture (x86_64, aarch64) +``` + +**Package Version Comparison Logic**: +```python +def verify_package_version(installed, expected_fixed): + """ + installed: "httpd-2.4.37-2.el8.x86_64" + expected_fixed: "httpd-2.4.37-2.el8" + + Returns: True if installed >= expected_fixed + """ + # Parse version components using RPM version comparison + # Account for epoch, version, release + # Use >= comparison (newer versions are acceptable) +``` + +### 3. Service Health Verification + +**MCP Tool**: `get_host_details` or `inventory__get_host_details` (from lightspeed-mcp) + +**Parameters**: Same as Step 2 - system_id with include_system_profile=true + +Verify affected services are running properly: + +``` +For each affected service (e.g., httpd): + +1. Check service status + Extract from: system_profile.enabled_services + Extract from: system_profile.running_processes + + Service Health Checks: + ✓ Service in enabled_services list → Service will start on boot + ✓ Service process in running_processes → Service currently running + ✗ Service not enabled → WARN (service won't start on reboot) + ✗ Service process not running → FAIL (service down) + +2. Check for service errors + Look for: system_profile.systemd_failed_units + ✓ Service not in failed units → PASS + ✗ Service in failed units → FAIL (service failed to start) + +3. Verify uptime (if applicable) + Check: Service started recently (after remediation) + ✓ Service uptime < remediation time + 10 minutes → Service restarted + ⚠ Service uptime > remediation time → Service may not have restarted +``` + +### 4. Remediation Summary Generation + +Generate comprehensive verification report: + +```json +{ + "verification_status": "success", # or "partial_success", "failed" + + "cve_id": "CVE-2024-1234", + "verification_date": "2024-01-20T15:30:00Z", + + "systems_remediated": { + "total": 10, + "successful": 10, + "failed": 0, + "partial": 0 + }, + + "verification_details": [ + { + "system_id": "uuid-1", + "hostname": "web-server-01", + "status": "verified", + + "checks": { + "cve_status": { + "result": "pass", + "details": "System removed from affected systems list" + }, + "package_version": { + "result": "pass", + "expected": "httpd-2.4.37-2.el8", + "installed": "httpd-2.4.37-2.el8", + "details": "Package updated successfully" + }, + "service_health": { + "result": "pass", + "service": "httpd", + "status": "running", + "details": "Service restarted and healthy" + } + } + } + ], + + "compliance": { + "all_systems_patched": true, + "services_healthy": true + }, + + "recommendations": [ + "Remediation verified successfully on all systems", + "Consider re-scanning with insights-client for updated inventory", + "Document remediation in change management system" + ] +} +``` + +### 6. Handle Verification Failures + +If verification fails, provide troubleshooting guidance: + +**Package Version Mismatch**: +``` +Verification Failed: Package Not Updated + +System: web-server-01 (uuid-1) +Expected: httpd-2.4.37-2.el8 +Found: httpd-2.4.37-1.el8 (OLD VERSION) + +Possible causes: +1. Playbook execution failed (check Ansible output) +2. Package repository doesn't have fixed version +3. Package update was skipped due to dependency conflict + +Troubleshooting: +1. Check Ansible playbook output for errors +2. Verify package availability: + sudo dnf info httpd-2.4.37-2.el8 +3. Manually update package: + sudo dnf update httpd +4. Check for package holds: + sudo dnf versionlock list +``` + +**Service Not Running**: +``` +Verification Failed: Service Not Running + +System: web-server-01 (uuid-1) +Service: httpd +Status: Failed + +Troubleshooting: +1. Check service status: + sudo systemctl status httpd +2. View service logs: + sudo journalctl -u httpd --since "10 minutes ago" +3. Check for configuration errors: + sudo httpd -t +4. Restart service manually: + sudo systemctl restart httpd +``` + +## Output Template + +When completing verification, provide output in this format: + +```markdown +# Remediation Verification Report + +## CVE: CVE-YYYY-NNNNN +**Verification Date**: 2024-01-20 15:30 UTC +**Overall Status**: ✓ SUCCESS + +## Summary +**Total Systems**: 10 +**Successfully Remediated**: 10 +**Failed**: 0 +**Partial Success**: 0 + +## Verification Results + +### System: web-server-01 (uuid-1) +**Status**: ✓ VERIFIED + +**Checks Performed**: +✓ CVE Status: System removed from affected list +✓ Package Version: httpd-2.4.37-2.el8 (updated from 2.4.37-1.el8) +✓ Service Health: httpd running and healthy + +--- + +### System: web-server-02 (uuid-2) +**Status**: ✓ VERIFIED + +**Checks Performed**: +✓ CVE Status: System marked as patched +✓ Package Version: httpd-2.4.37-2.el8 installed +✓ Service Health: httpd running + +--- + +[Additional systems...] + +## Compliance Status +✓ All systems successfully patched +✓ All services running and healthy + +## Recommendations +1. Remediation verified successfully on all 10 systems +2. Re-scan systems with Red Hat Lightspeed for updated inventory: + ```bash + sudo insights-client --check-results + ``` +3. Document remediation in change management system +4. Consider scheduling next vulnerability scan in 7 days + +## Next Steps +- Remediation complete, no further action required +- Monitor systems for 24 hours to ensure stability +- Update vulnerability tracking system +``` + +## Examples + +### Example 1: Successful Verification + +**User Request**: "Verify remediation for CVE-2024-1234 on 5 systems" + +**Skill Response**: +1. Call `get_cve_systems` → 0 systems affected (down from 5) +2. Call `get_host_details` for each → All have httpd-2.4.37-2.el8 +3. Check service status → All httpd services running +4. Return: "✓ All 5 systems verified, CVE remediated successfully" + +### Example 2: Partial Success + +**User Request**: "Verify batch remediation on 20 systems" + +**Skill Response**: +1. Call `get_cve_systems` → 2 systems still affected (18 fixed) +2. Call `get_host_details` → 2 systems have old package version +3. Identify failed systems: web-server-18, web-server-19 +4. Return: "⚠ 18/20 systems verified. 2 systems failed package update. Troubleshooting guidance provided." + +## Dependencies + +### Required MCP Servers +- `lightspeed-mcp` - Red Hat Lightspeed platform access + +### Required MCP Tools +- `get_cve` or `vulnerability__get_cve` (from lightspeed-mcp) - Get CVE metadata and remediation status + - Parameters: cve_id (string, format CVE-YYYY-NNNNN), include_details (boolean) + - Returns: CVE metadata including remediation status +- `get_cve_systems` or `vulnerability__get_cve_systems` (from lightspeed-mcp) - List systems affected by CVE + - Parameters: cve_id (string), limit (number), offset (number) + - Returns: List of systems with vulnerability status +- `get_host_details` or `inventory__get_host_details` (from lightspeed-mcp) - Get system details including packages and services + - Parameters: system_id (UUID string), include_system_profile (boolean) + - Returns: System profile with installed_packages, enabled_services, running_processes + +### Related Skills +- `playbook-generator` - Generates playbooks that this skill verifies +- `system-context` - Provides system context for verification scope +- `cve-impact` - Initial impact assessment to compare against verification results +- `playbook-executor` - Executes playbooks that this skill verifies + +### Reference Documentation +- None required (verification skill uses MCP tool data) + +## Best Practices + +1. **Wait before verification** - Allow 5-10 minutes after playbook execution for system updates to register +2. **Check multiple indicators** - CVE status + package version + service health (defense in depth) +3. **Re-scan with Lightspeed** - Recommend `insights-client --check-results` to update inventory +4. **Document failures** - Provide detailed troubleshooting for any verification failures +5. **Consider timing** - Lightspeed inventory updates may take up to 24 hours to propagate +6. **Verify at scale** - Use batch verification for large deployments (call get_host_details in parallel) + +## Integration with Other Skills + +- **playbook-generator**: Generates playbooks that this skill verifies +- **system-context**: Provides system context for verification scope +- **cve-impact**: Initial impact assessment to compare against verification results + +**Orchestration Example** (from `/remediation` skill): +1. User requests CVE remediation +2. Agent invokes playbook-generator → Creates playbook +3. User executes playbook manually +4. Agent invokes remediation-verifier skill → Confirms success +5. Agent reports: "✓ CVE remediated and verified on all systems" + +**Verification-First Principle**: +``` +Never assume remediation worked. Always verify: +1. CVE status in Lightspeed +2. Package versions updated +3. Services running + +Trust, but verify. +``` diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/instruction.md b/evaluation/with_skills/rh-sre__remediation-verifier/instruction.md new file mode 100644 index 00000000..845397fa --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/instruction.md @@ -0,0 +1,18 @@ +# Post-Remediation Verification Task + +You are a Red Hat SRE. A CVE remediation playbook was recently executed against production systems. You need to verify that the fix was successfully applied and that the systems are healthy. + +## Scenario +The operations team ran a remediation playbook for CVE-2026-1234 against affected RHEL systems earlier today. You need to confirm the remediation actually worked and that no systems were left in a broken state. + +## Requirements +- Check whether the affected systems are still listed as vulnerable to the CVE +- Verify that the relevant packages have been updated to the fixed versions +- Confirm that critical services on each system are running and healthy after the patch +- Identify any systems where remediation failed or is incomplete +- For any failures, provide troubleshooting guidance (package conflicts, services not restarting, partial patches) +- Produce a verification summary: how many systems passed, how many failed, and any remaining action items + +Document your methodology, verification results, and assessment in `/root/report.md`. + +Use MCP tools to query system data. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/solution/solve.sh b/evaluation/with_skills/rh-sre__remediation-verifier/solution/solve.sh new file mode 100644 index 00000000..49714723 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/solution/solve.sh @@ -0,0 +1,20 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Remediation Verification + +## CVE-2024-12345 Status +| System | RPM Status | CVE Status | Service Health | +|--------|-----------|------------|----------------| +| web-01 | installed >= fixed | Patched | Healthy | + +## Checks Performed +- get_cve_systems: System removed from affected list or status=patched +- get_host_details: system_profile.installed_packages >= expected fixed version +- systemd_failed_units: No service disruptions +- enabled_services, running_processes: verified + +## Notes +- Lightspeed inventory lag: up to 24 hours +- Recommend: insights-client --check-results to update inventory +- RPM comparison: installed version >= expected fixed version +REPORT_EOF diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/task.toml b/evaluation/with_skills/rh-sre__remediation-verifier/task.toml new file mode 100644 index 00000000..23f81673 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__remediation-verifier" +name = "rh-sre Remediation Verification Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "remediation-verifier", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/tests/llm_judge.py b/evaluation/with_skills/rh-sre__remediation-verifier/tests/llm_judge.py new file mode 100644 index 00000000..15b8919b --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "inventory_24h_lag", "file": "/root/report.md", "question": "Does the report note that Lightspeed inventory can take up to 24 hours to update and recommend insights-client --check-results for re-sync?", "reference": "A skilled report warns about inventory lag. An unskilled report expects immediate updates."}, + {"id": "system_profile_checks", "file": "/root/report.md", "question": "Does the report use get_host_details with include_system_profile for installed packages and service health verification?", "reference": "A skilled report uses system profile data. An unskilled report only checks CVE status."}, + {"id": "three_verification_layers", "file": "/root/report.md", "question": "Does the report verify at least 2 of: CVE status, package version, service health?", "reference": "A skilled report performs defense-in-depth verification. An unskilled report only checks one layer."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/tests/test.sh b/evaluation/with_skills/rh-sre__remediation-verifier/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/tests/test_outputs.py b/evaluation/with_skills/rh-sre__remediation-verifier/tests/test_outputs.py new file mode 100644 index 00000000..00ddada6 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation-verifier/tests/test_outputs.py @@ -0,0 +1,75 @@ +""" +Tests for rh-sre__remediation-verifier per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['verif', 'remediation', 'confirm']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_three_checks(self): + """Skill: Verify CVE status + package version + service health (defense in depth).""" + c = read_report().lower() + has_cve = any(t in c for t in ["cve", "vulnerab", "patched", "affected"]) + has_pkg = any(t in c for t in ["package", "version", "installed", "fixed"]) + has_svc = any(t in c for t in ["service", "running", "health", "enabled"]) + assert (has_cve and has_pkg) or (has_cve and has_svc) or (has_pkg and has_svc), ( + "should perform at least 2 of 3 checks (skill: CVE status, package, service)" + ) + + def test_package_version_comparison(self): + """Skill: Compare installed package version to expected fixed version (RPM-style).""" + c = read_report().lower() + has_compare = any(t in c for t in ["compare", "version", "expected", "installed"]) + has_fixed = any(t in c for t in ["fixed", "updated", "el8", "el9"]) + assert has_compare or has_fixed, ( + "should compare package versions (skill: verify_package_version)" + ) + + def test_inventory_24h_lag(self): + """Skill: Lightspeed inventory can take up to 24 hours to reflect updated package versions.""" + c = read_report().lower() + has_24 = "24" in c + has_timing = any(t in c for t in ["hour", "propagat", "delay"]) + assert has_24 and has_timing, ( + "should note inventory 24h lag (skill: Best Practices)" + ) + + def test_include_system_profile(self): + """Skill: get_host_details with include_system_profile: true returns installed_packages, enabled_services.""" + c = read_report().lower() + assert any(t in c for t in ["include_system_profile", "system_profile", "installed_packages"]), ( + "should reference include_system_profile for packages/services (skill)" + ) + + def test_insights_client_resync(self): + """Skill: insights-client --check-results triggers inventory re-sync.""" + c = read_report().lower() + assert any(t in c for t in ["insights-client", "check-results"]), ( + "should mention insights-client for inventory resync (skill)" + ) diff --git a/evaluation/with_skills/rh-sre__remediation/environment/Dockerfile b/evaluation/with_skills/rh-sre__remediation/environment/Dockerfile new file mode 100644 index 00000000..484ebb33 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/Dockerfile @@ -0,0 +1,52 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + + + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..1092dd1d --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- pass rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__remediation/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/with_skills/rh-sre__remediation/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..2269a235 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,722 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2026-1234": { + "cve_id": "CVE-2026-1234", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Critical kernel vulnerability: remote code execution in kernel network stack allows unauthenticated attackers to execute arbitrary code via crafted packets", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2026-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 8, + "total_remediated": 2, + "total_vulnerable": 6, + }, + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/SKILL.md b/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/SKILL.md new file mode 100644 index 00000000..fde713b0 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/SKILL.md @@ -0,0 +1,340 @@ +--- +name: cve-validation +description: | + **CRITICAL**: This skill must be used for CVE validation queries. DO NOT use raw MCP tools like get_cve directly. + + Validate CVE identifiers and check remediation availability in Red Hat Lightspeed. Use this skill when you need to verify a CVE exists, check its severity, and confirm automated remediation is available before proceeding with remediation planning. + + **DO NOT use this skill when** user requests full remediation - use `/remediation` skill instead: + - "Create a remediation playbook for CVE-X" → `/remediation` skill + - "Create playbook and execute it" → `/remediation` skill + - "Remediate CVE-X" / "Patch CVE-X" → `/remediation` skill + + This skill orchestrates MCP tools (get_cve) to provide comprehensive CVE validation. The `/remediation` skill invokes this skill as Step 2 of its workflow. +--- + +# CVE Validation Skill + +This skill validates CVE identifiers and checks remediation availability in Red Hat Lightspeed, ensuring CVEs are valid and remediable before investing effort in remediation planning. + +**Integration with Remediation Skill**: The `/remediation` skill orchestrates this skill as part of its Step 2 (Validate CVE) workflow. For standalone CVE validation, you can invoke this skill directly. + +## Invocation Note (Host-Specific) + +When invoked by another skill (e.g. remediation), use the Skill tool—do NOT use "Task Output" with the skill name as task ID. That causes "No task found with ID: cve-validation". See [skill-invocation.md](../../../docs/references/skill-invocation.md). + +## Prerequisites + +**Required MCP Servers**: `lightspeed-mcp` ([setup guide](https://console.redhat.com/)) + +**Required MCP Tools**: +- `get_cve` (from lightspeed-mcp) - Get CVE metadata and validation + +**Required Environment Variables**: +- `LIGHTSPEED_CLIENT_ID` - Red Hat Lightspeed service account client ID +- `LIGHTSPEED_CLIENT_SECRET` - Red Hat Lightspeed service account secret + +### Prerequisite Validation + +**CRITICAL**: Before executing any operations, execute the `/mcp-lightspeed-validator` skill to verify MCP server availability. + +**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-lightspeed-validator/SKILL.md#validation-freshness-policy). + +**How to invoke**: Execute the `/mcp-lightspeed-validator` skill + +**Handle validation result**: +- **If validation PASSED**: Continue with CVE validation +- **If validation PARTIAL**: Warn user and ask to proceed +- **If validation FAILED**: Stop execution, provide setup instructions + +## When to Use This Skill + +**Use this skill directly when you need**: +- Quick validation of CVE identifier format and existence (standalone query) +- Check if automated remediation is available +- Verify CVE metadata before analysis +- Validate CVE lists for batch operations + +**DO NOT use this skill when** - use `/remediation` skill instead: +- User says "create a remediation playbook" or "remediate CVE-X" or "patch CVE-X" +- User says "create playbook and execute it" - agent orchestrates full workflow +- Any request that implies playbook generation or execution + +**Use the `/remediation` skill when you need**: +- Full remediation workflow (validation + analysis + playbook + execution) +- Integrated CVE validation as part of remediation planning + +**How they work together**: The `/remediation` skill invokes this skill early in the workflow to fail fast if a CVE is invalid or has no automated remediation, saving time and effort. + +**When invoked by remediation**: Return remediatable status prominently so the orchestrator can gate. Include `remediation_status.automated_remediation_available` (boolean) and `validation_status` ("valid" | "not_remediable" | "invalid" | "not_found") in the output. + +## Workflow + +### Step 0: Validate Lightspeed MCP Prerequisites + +**Action**: Execute the `/mcp-lightspeed-validator` skill + +**Note**: Can skip if validation was performed earlier in this session and succeeded. See [Validation Freshness Policy](../mcp-lightspeed-validator/SKILL.md#validation-freshness-policy). + +**How to invoke**: Execute the `/mcp-lightspeed-validator` skill + +**Handle validation result**: +- **If validation PASSED**: Continue to Step 1 +- **If validation PARTIAL**: Warn user and ask to proceed +- **If validation FAILED**: Stop execution, user must set up MCP server + +### Step 1: CVE Format Validation + +Validate CVE identifier format before calling MCP tools. **Format only**—do NOT reject based on year or sequence magnitude. + +```python +CVE Format: CVE-YYYY-NNNNN +Where: +- YYYY = 4-digit year (1999-2030; current and recent years are valid) +- NNNNN = 4-7 digit sequence number (e.g. 1234, 24882, 1234567) + +Valid Examples: +- CVE-2024-1234 +- CVE-2026-24882 # 2026 CVEs exist; 24882 is 5 digits (valid) +- CVE-2023-12345 +- CVE-2021-1234567 + +Invalid Examples (format only): +- CVE-24-1234 (year must be 4 digits) +- CVE-2024-ABC (sequence must be numeric) +- 2024-1234 (missing CVE- prefix) +- CVE-2024-123 (sequence must be 4-7 digits) +``` + +**Quick Regex Check**: +``` +Pattern: ^CVE-\d{4}-\d{4,7}$ + +If invalid format: +→ Return error immediately +→ Suggest format correction +→ Do not proceed to MCP tool calls +``` + +**CRITICAL - Do NOT add extra checks**: If the format matches the regex, you MUST call `get_cve`. Do NOT reject based on: +- "Future" or "current year" assumptions (e.g. "2026 CVE might not exist yet") +- Sequence number magnitude (e.g. "24882 seems high")—5 digits is valid +- Your training data about typical CVE ranges + +Let the API determine existence. A 404 from get_cve means "not found"; format validation only catches malformed IDs. + +### Step 2: CVE Metadata Retrieval + +**CRITICAL**: Document consultation MUST happen BEFORE tool invocation. + +**Document Consultation** (REQUIRED - Execute FIRST): +1. **Action**: Read [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) using the Read tool to understand CVE validation criteria +2. **Action**: Read [references/01-remediation-indicators.md](references/01-remediation-indicators.md) to interpret get_cve response—**CRITICAL** to avoid misinterpreting remediation availability +3. **Output to user**: "I consulted [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) and [01-remediation-indicators.md](references/01-remediation-indicators.md) for CVE validation and remediation indicator interpretation." + +**MCP Tool**: `get_cve` or `vulnerability__get_cve` (from lightspeed-mcp) + +**Do NOT use** `vulnerability__explain_cves` for validation. That tool requires `system_uuid` from inventory; at validation you may not have it. `get_cve` provides remediation availability. Never pass `system_uuid: "undefined"` or placeholders. + +**Parameters**: +- `cve`: Exact CVE identifier from user query (format: `"CVE-YYYY-NNNNN"`) + - Example: `"CVE-2024-1234"` +- `advisory_available`: `"true"` (retrieve CVE with advisory/remediation info) + +**Expected Output**: CVE metadata including CVSS score, severity, affected packages, remediation availability + +Retrieve CVE metadata from Red Hat Lightspeed: + +```json +{ + "cve_id": "CVE-2024-1234", + "cvss_score": 7.5, + "cvss_vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", + "severity": "Important", # Red Hat severity rating + "description": "A vulnerability in Apache HTTPD...", + "published_date": "2024-01-15", + "modified_date": "2024-01-20", + + "affected_packages": [ + { + "name": "httpd", + "version": "2.4.37-1.el8", + "fixed_version": "2.4.37-2.el8" + } + ], + + "references": [ + "https://access.redhat.com/errata/RHSA-2024:1234", + "https://nvd.nist.gov/vuln/detail/CVE-2024-1234" + ], + + "cwe": "CWE-400: Uncontrolled Resource Consumption", + + "exploitability": "Proof of concept available", + "remediation_available": true, # KEY FIELD + "reboot_required": false +} +``` + +### Step 3: Validation Checks + +**CRITICAL**: Document consultation MUST happen BEFORE validation logic. + +**Document Consultation** (REQUIRED - Execute FIRST): +1. **Action**: Read [cvss-scoring.md](../../docs/references/cvss-scoring.md) using the Read tool to understand Red Hat severity classification and CVSS score ranges +2. **Output to user**: "I consulted [cvss-scoring.md](../../docs/references/cvss-scoring.md) to understand Red Hat severity classification and CVSS score ranges." + +Perform comprehensive validation: + +**A. Existence Check**: +``` +✓ CVE exists in Red Hat CVE database +✗ CVE not found → Return error with suggestions +``` + +**B. Red Hat Relevance Check**: +``` +✓ CVE affects RHEL systems +✗ CVE is Windows/macOS specific → Not applicable to RHEL +``` + +**C. Severity Classification**: +``` +Red Hat Severity Levels: +- Critical (CVSS 9.0-10.0): Immediate action required +- Important (CVSS 7.0-8.9): Urgent remediation needed +- Moderate (CVSS 4.0-6.9): Plan remediation +- Low (CVSS 0.1-3.9): Low priority +``` + +**D. Remediation Availability Check** (READ [references/01-remediation-indicators.md](references/01-remediation-indicators.md)): +``` +Key Question: Can Red Hat Lightspeed generate an automated playbook? + +✅ USE these fields: + - advisory_available === true → Remediation available + - remediation === 2 → Automated remediation available + - advisories_list non-empty → RHSA exists, remediation available + +❌ DO NOT use rules[] for remediation decision: + - rules: [] (empty) does NOT mean "no remediation" + - Advisor rules are separate from vulnerability remediation + - Remediation comes from Security Advisories (RHSA), not Advisor rules + +✓ If advisory_available=true OR remediation=2 OR advisories_list has entries + → Proceed with automated remediation + → Use create_vulnerability_playbook tool + +✗ If advisory_available=false AND (remediation=0 or advisories_list empty) + → Manual remediation required + → Provide manual steps based on affected packages +``` + +**E. Package Information Validation**: +``` +Check affected_packages array: +✓ Packages identified: httpd-2.4.37-1.el8 +✓ Fixed version available: httpd-2.4.37-2.el8 +✓ Package exists in RHEL repositories + +This information will be used by playbook-generator skill. +``` + +### Step 4: Return Validation Result + +Return structured validation result. **When invoked by remediation skill**: Ensure `validation_status` and `remediation_status.automated_remediation_available` are explicit—the orchestrator gates on these. + +```json +{ + "validation_status": "valid", # or "invalid", "not_found", "not_remediable" + + "cve_metadata": { + "cve_id": "CVE-2024-1234", + "cvss_score": 7.5, + "severity": "Important", + "description": "Brief description...", + "published_date": "2024-01-15" + }, + + "remediation_status": { + "automated_remediation_available": true, + "reboot_required": false, + "affected_packages": [ + { + "name": "httpd", + "current_version": "2.4.37-1.el8", + "fixed_version": "2.4.37-2.el8" + } + ] + }, + + "recommendations": [ + "Automated remediation available via Red Hat Lightspeed", + "No reboot required for this CVE", + "Severity: Important - Urgent remediation recommended", + "Test in staging environment before production deployment" + ], + + "next_steps": [ + "Analyze CVE impact (use cve-impact skill)", + "Gather system context (use system-context skill)", + "Generate remediation playbook (use playbook-generator skill)" + ] +} +``` + +## Output, Examples, Error Handling + +**Read [references/03-output-template.md](references/03-output-template.md)** for report format. +**Read [references/04-examples.md](references/04-examples.md)** for validation examples. +**Read [references/05-error-handling.md](references/05-error-handling.md)** for format, not-found, no-remediation, and API errors. + +## Best Practices + +Validate format first; if regex matches, ALWAYS call get_cve (do not reject on year/sequence). Check remediation availability; fail fast if none. Provide clear next steps and manual guidance when automated unavailable. Link to NVD and Red Hat Security. Cache results to avoid redundant calls. + +## Dependencies + +### Required MCP Servers +- `lightspeed-mcp` - Red Hat Lightspeed platform access + +### Required MCP Tools +- `get_cve` (from lightspeed-mcp) - Get CVE metadata and validation + - Parameters: cve_id (string, format CVE-YYYY-NNNNN), include_details (boolean), validate_format (boolean) + - Returns: CVE metadata with CVSS score, severity, affected packages, remediation availability + +### Related Skills +- `mcp-lightspeed-validator` - **PREREQUISITE** - Validates Lightspeed MCP server before operations + - Use before: ALL cve-validation operations (Step 0 in workflow) + - Purpose: Ensures MCP server is available before attempting tool calls + +- `cve-impact` - Analyze CVE impact after validation + - Use after: Step 4 when CVE is validated and user wants impact analysis + - Purpose: Assess risk and affected systems for validated CVE + +- `system-context` - Get system details after validating CVE affects systems + - Use after: Validation confirms CVE has affected systems + - Purpose: Understand deployment context before remediation + +- `playbook-generator` - Generate remediation playbooks for validated CVEs + - Use after: Validation confirms remediation_available = true + - Purpose: Create automated remediation for valid, remediable CVEs + +### Reference Documentation +- [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) - CVE validation criteria +- [references/01-remediation-indicators.md](references/01-remediation-indicators.md) - **REQUIRED** - Correct interpretation of get_cve response (advisory_available, remediation, advisories_list). Do NOT use rules[] for remediation decision. +- [cvss-scoring.md](../../docs/references/cvss-scoring.md) - Red Hat severity classification and CVSS score ranges +- [cve-remediation-templates.md](../../docs/ansible/cve-remediation-templates.md) - Manual remediation templates for CVEs without automated playbooks + +## Tools Reference + +This skill uses: +- `get_cve` (vulnerability toolset) - Get CVE metadata and remediation availability from Red Hat Lightspeed + +**Do NOT use** `vulnerability__explain_cves` in this skill—it requires `system_uuid` which may not be available at validation time. Use `get_cve` only. + +All tools are provided by the lightspeed-mcp MCP server configured in `.mcp.json`. + +## Integration with Other Skills + +cve-impact, playbook-generator, system-context, remediation-verifier all depend on validation first. The `/remediation` skill invokes cve-validation as Step 2. Validate → proceed if valid; stop and return error if invalid. diff --git a/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/01-remediation-indicators.md b/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/01-remediation-indicators.md new file mode 100644 index 00000000..17f9afe8 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/01-remediation-indicators.md @@ -0,0 +1,66 @@ +# Remediation Availability Indicators (get_cve Response) + +Read this reference when interpreting `vulnerability__get_cve` or `get_cve` response to determine if automated remediation is available. + +## CRITICAL: Correct vs Incorrect Indicators + +### ✅ USE These Fields for Remediation Availability + +| Field | Meaning | Remediation Available When | +|-------|---------|-----------------------------| +| `advisory_available` | Red Hat Security Advisory exists | `true` | +| `remediation` | Remediation status code | `2` = automated remediation available | +| `advisories_list` | List of RHSA/errata IDs | Non-empty (e.g. `["RHSA-2026:2719"]`) | + +**Decision rule**: Remediation IS available when `advisory_available === true` OR `remediation === 2` OR `advisories_list` is non-empty. + +### ❌ DO NOT Use These Fields for Remediation + +| Field | Meaning | Why NOT to Use | +|-------|---------|----------------| +| `rules` | Red Hat Insights **Advisor** rules | Advisor rules are separate from vulnerability remediation. Empty `rules: []` does NOT mean no remediation. Remediation comes from Security Advisories (RHSA), not Advisor rules. | + +**Common mistake**: Agent sees `rules: []` (empty) and incorrectly concludes "no remediation available." This is WRONG. Always check `advisory_available` and `remediation` first. + +## Interpretation Checklist + +When evaluating `get_cve` response: + +1. **Check `advisory_available`**: If `true` → remediation available ✓ +2. **Check `remediation`**: If `2` → automated remediation available ✓ +3. **Check `advisories_list`**: If non-empty (e.g. RHSA-*) → remediation available ✓ +4. **Ignore `rules`**: Do NOT use for remediation decision. Empty rules ≠ no remediation. + +## Example: Remediation Available (rules empty) + +```json +{ + "advisory_available": true, + "advisories_list": ["RHSA-2026:2719"], + "remediation": 2, + "rules": [] +} +``` + +**Correct interpretation**: Remediation IS available. `rules: []` only means no Advisor rule—remediation comes from RHSA-2026:2719. + +## Example: No Remediation + +```json +{ + "advisory_available": false, + "advisories_list": [], + "remediation": 0, + "rules": [] +} +``` + +**Correct interpretation**: No automated remediation. Manual steps required. + +## get_cve_systems Response (per-system) + +When using `get_cve_systems` for system-level check, each system entry may include: +- `attributes.advisory_available` — same meaning as get_cve +- `attributes.remediation` — same meaning as get_cve + +Use the same interpretation rules. Do NOT use `rules` for remediation decision. diff --git a/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/03-output-template.md b/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/03-output-template.md new file mode 100644 index 00000000..51bb3992 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/03-output-template.md @@ -0,0 +1,36 @@ +# CVE Validation Output Template + +Read when completing CVE validation to format the report. + +```markdown +# CVE Validation Result + +## CVE: CVE-YYYY-NNNNN +**Status**: ✓ Valid + +## CVE Information +**CVSS Score**: X.X (Severity) +**Published**: YYYY-MM-DD +**Description**: [Brief description] + +## Affected Packages +- package-current → package-fixed (fixed) + +## Remediation Status +✓ **Automated Remediation Available** (or ✗ Manual required) +✓ Package updates available +✗ Reboot NOT required + +## Severity Assessment +**Red Hat Severity**: Critical/Important/Moderate/Low +**Priority**: P0/P1/P2 +**Response Time**: [guidance] + +## Recommendations +1. [Automated/manual remediation guidance] +2. Test in staging first +3. Schedule deployment during change window + +## Next Steps +1. cve-impact → system-context → playbook-generator → remediation-verifier +``` diff --git a/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/04-examples.md b/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/04-examples.md new file mode 100644 index 00000000..2a16ce85 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/04-examples.md @@ -0,0 +1,35 @@ +# CVE Validation Examples + +## Example 1: Valid CVE with Automated Remediation + +**Request**: "Validate CVE-2024-1234" +1. Format check → Valid +2. get_cve → found +3. advisory_available/remediation/advisories_list → remediation available (ignore rules[]) +4. Return: "Valid, automated remediation available" + +## Example 2: Valid CVE, No Automated Remediation + +**Request**: "Validate CVE-2024-5678" +1. Format → Valid, get_cve → found +2. advisory_available/remediation/advisories_list → no remediation +3. Return: "Valid but manual remediation: yum update custom-app" + +## Example 3: Invalid Format + +**Request**: "Validate CVE-24-1234" +1. Format → Invalid (year must be 4 digits) +2. Return error without MCP call; suggest CVE-2024-1234 + +## Example 4: CVE Not Found + +**Request**: "Validate CVE-2024-999999" +1. Format → Valid, get_cve → 404 +2. Return: "Not found. Check NVD, access.redhat.com, or wait 24-48h if recent" + +## Example 5: Batch Validation + +**Request**: "Validate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +1. Validate each sequentially +2. Return summary per CVE (valid/remediable, valid/manual, invalid format) +3. Suggest next steps per CVE diff --git a/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/05-error-handling.md b/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/05-error-handling.md new file mode 100644 index 00000000..201c193a --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/05-error-handling.md @@ -0,0 +1,37 @@ +# CVE Validation Error Handling + +Read when errors occur during CVE validation. + +## CVE Format Invalid + +``` +CVE Validation Failed: Invalid Format +Provided: CVE-24-1234 +Expected: CVE-YYYY-NNNNN (YYYY=4-digit year, NNNNN=4-7 digit sequence) +Suggestion: Did you mean CVE-2024-1234? +``` + +## CVE Not Found in Database + +``` +CVE-YYYY-NNNNN was not found in Red Hat CVE database. +Possible reasons: Too recent, doesn't affect RHEL, incorrect ID. +Next steps: Verify at NVD, access.redhat.com/security/cve/CVE-YYYY-NNNNN, wait 24-48h if recent. +``` + +## CVE Exists But No Automated Remediation + +``` +CVE Validation: Valid (No Automated Remediation) +CVE-YYYY-NNNNN is valid but has no automated playbook. +Manual steps: dnf/yum update package-name, restart service if needed, verify fix. +Offer: "Would you like a manual playbook template?" +``` + +## API Access Error + +``` +CVE Validation Failed: API Access Error +Possible causes: Network, auth failure, service unavailable. +Troubleshooting: ping console.redhat.com, verify credentials, status.redhat.com, retry. +``` diff --git a/evaluation/with_skills/rh-sre__remediation/environment/skills/remediation/SKILL.md b/evaluation/with_skills/rh-sre__remediation/environment/skills/remediation/SKILL.md new file mode 100644 index 00000000..49dcb23f --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/skills/remediation/SKILL.md @@ -0,0 +1,279 @@ +--- +name: remediation +description: | + **CRITICAL**: Use this skill for ALL CVE remediation workflows. DO NOT use individual skills piecemeal for end-to-end remediation. + + Use when users request: + - CVE remediation playbooks or security patch deployment + - Multi-step remediation (validation → context → playbook → execution) + - Batch remediation across multiple systems or CVEs + - End-to-end CVE management (analysis + remediation + verification) + - Prioritizing and remediating CVEs (not just listing them) + - Emergency security response with immediate remediation plans + + DO NOT use for simple queries: + - "List critical CVEs" → Use `/cve-impact` skill + - "What's the CVSS score for CVE-X?" → Use `/cve-impact` or `/cve-validation` + - Standalone impact analysis without remediation → Use `/cve-impact` + + This skill orchestrates 6 specialized skills (cve-impact, cve-validation, system-context, playbook-generator, playbook-executor, remediation-verifier) for complete remediation workflows. +--- +model: inherit +color: red +metadata: + author: "Red Hat Ecosystem Engineering" + priority: "high" +--- + +# Remediation Skill + +End-to-end CVE remediation workflow. Orchestrates specialized skills for validation, context gathering, playbook generation, execution, and verification. + +## Prerequisites + +**Required MCP Servers**: `lightspeed-mcp` (CVE data, playbook generation), `aap-mcp-job-management`, `aap-mcp-inventory-management` (execution) + +**Related Skills** (this skill invokes them): +- `/mcp-lightspeed-validator` - Verify Lightspeed MCP before CVE operations +- `/mcp-aap-validator` - Verify AAP MCP before playbook execution +- `/cve-impact` - CVE risk assessment +- `/cve-validation` - CVE validation and remediation availability +- `/system-context` - System inventory and deployment context +- `/playbook-generator` - Ansible playbook generation +- `/playbook-executor` - Playbook execution via AAP +- `/remediation-verifier` - Post-remediation verification + +**Verification**: See Step 0 for MCP validation. Execute `/mcp-aap-validator` before Step 5 (playbook execution) if not already validated. + +## When to Use This Skill + +**Use this skill when**: +- User requests CVE remediation (playbook creation, patching, deployment) +- Full workflow needed: analysis → validation → playbook → execution → verification +- Batch remediation across multiple CVEs or systems + +**Do NOT use when**: +- User only wants CVE listing or impact analysis → Use `/cve-impact` +- User only wants CVE validation → Use `/cve-validation` +- User only wants playbook generation (no execution) → Use `/playbook-generator` directly + +## Workflow + +Execute skills in this order. **MANDATORY**: Use actual Skill tool invocations, NOT text pretending to invoke skills. **Each step must complete before the next begins**—do not start Step N+1 until Step N has returned its result. + +### Upfront: Planned Tasks (Before Step 0) + +**When**: Before executing any step. **Do NOT start Step 0 until the user validates the plan.** + +**Action**: Present the planned task list using **Part A** of [references/01-remediation-plan-template.md](references/01-remediation-plan-template.md). Show the 7 tasks (validate MCP → impact → validate CVE → context → playbook → execute → verify) and ask "Proceed with this plan?" + +**Task list ordering** (CRITICAL): If using TodoWrite or task list UI, create tasks **in workflow order** (Step 0, 1, 2, 3, 4, 5, 6). Do NOT create in completion order or random order—display order must match execution order. + +**Wait for explicit user response** ("yes" or "proceed") before invoking Step 0. If "abort" → stop. + +### Step 0: Validate MCP Prerequisites + +**Action**: Execute `/mcp-lightspeed-validator` (and `/mcp-aap-validator` before Step 5 if executing playbooks) + +**When**: Before any CVE or remediation operations. Can skip if already validated this session. + +**Sequencing (MANDATORY)**: Invoke validators **one at a time**. **Do NOT proceed to Step 1 until Step 0 is complete.** Wait for each validator to return explicit results (PASSED / FAILED / PARTIAL) before moving on. "Successfully loaded skill" alone does NOT mean validation completed—you must see the actual validation outcome. + +**Invocation**: Use the Skill tool for ALL sub-skill invocations (validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier). **Do NOT use "Task Output" with the skill name as task ID**—that causes "No task found" errors (e.g. "No task found with ID: cve-validation"). See [skill-invocation.md](../../docs/references/skill-invocation.md). + +**Handle result**: If validation fails, stop and provide setup instructions. If passed, proceed to Step 1. **If any skill invocation fails** (e.g. "No task found with ID: ..."): Proceed with a warning—do not block. Later steps will surface real errors if MCP is unavailable. + +### Step 1: Impact Analysis (If Requested or Needed) + +**Action**: Execute the `/cve-impact` skill + +**Invoke**: +``` +"Analyze CVE-XXXX-YYYY and assess its impact on affected systems" +``` + +**Expected**: Risk assessment, affected systems list, CVSS interpretation. Integrate into remediation planning. If user only wanted impact analysis, provide assessment and offer remediation options. + +### Step 2: Validate CVE (Remediatable Gate) + +**Action**: Execute the `/cve-validation` skill + +**Invoke**: +``` +"Validate CVE-XXXX-YYYY format, existence, and remediation availability" +``` + +**Expected**: Validation status including `remediation_status.automated_remediation_available` or `validation_status`. + +**Remediatable Gate** (MANDATORY): Trust cve-validation skill output. Do NOT re-interpret raw get_cve response—cve-validation uses advisory_available, remediation, advisories_list (not rules[]). See [cve-validation references/01-remediation-indicators.md](../cve-validation/references/01-remediation-indicators.md). +- **If remediatable** (`remediation_available: true` or `validation_status: "valid"`): Proceed to Step 3. +- **If NOT remediatable** (`remediation_available: false` or `validation_status: "not_remediable"`): + 1. Explain: "CVE-XXXX-YYYY has no automated remediation in Red Hat Lightspeed. Execution may have no effect." + 2. Suggest alternatives: manual patching, check Red Hat errata. + 3. Offer: "Continue anyway? (yes/no)" + 4. **If user says "yes"**: Proceed to Step 3 with warning: "⚠️ Proceeding despite no automated remediation—playbook generation or execution may have no effect." + 5. **If user says "no"**: Stop. Do not proceed to Steps 3–5. + +**Batch**: For multiple CVEs, validate each. Proceed only with remediatable CVEs unless user explicitly confirms to include non-remediatable ones (with same warning). + +### Step 3: Gather Context + +**Action**: Execute the `/system-context` skill + +**Invoke**: +``` +"Gather system context for CVE-XXXX-YYYY: identify affected systems, RHEL versions, and deployment environments" +``` + +**Expected**: Context summary with remediation strategy. Use to inform playbook generation and execution planning. + +### Step 4: Generate Playbook + +**Action**: Execute the `/playbook-generator` skill + +**CRITICAL**: You MUST invoke `/playbook-generator`, NOT generate playbook text yourself. + +**Invoke**: +``` +"Generate an Ansible remediation playbook for CVE-XXXX-YYYY targeting systems [list of system UUIDs]. Apply Red Hat best practices and RHEL-specific patterns from documentation." +``` + +**Expected**: Ansible playbook from Red Hat Lightspeed (returned AS IS by playbook-generator—no modifications). Present to user. **The playbook-generator ONLY GENERATES**—it does NOT execute. After presenting the playbook, present the Remediation Plan for user validation (see below). + +### Remediation Plan (User Validation) — MANDATORY before Step 5 + +**When**: After Step 4 completes. **Do NOT proceed to Step 5 until the user validates the plan.** + +**Action**: Present the plan using the Summary + Table + Checklist format. **Read [references/01-remediation-plan-template.md](references/01-remediation-plan-template.md)** for the exact template. + +**Format**: +1. **Summary** — 1–2 sentences: what will happen and why +2. **Table** — CVE | Target Systems | Key Action +3. **Checklist** — Ordered steps (mark completed as "— done") +4. **Confirm prompt** — "yes"/"proceed", "dry-run only", or "abort" + +**Wait for explicit user response.** If "yes" or "proceed" → invoke playbook-executor. If "abort" → stop. If "dry-run only" → invoke playbook-executor with instruction to run dry-run only and stop. + +### Step 5: Execute Playbook (With User Confirmation) + +**Prerequisite**: Remediation Plan must be presented and user must have responded "yes" or "proceed" (or "dry-run only"). Do NOT invoke playbook-executor until plan validation is complete. + +**CRITICAL**: Before execution, you MUST: +1. Have presented the Remediation Plan (summary + table + checklist) +2. Have received user confirmation ("yes", "proceed", or "dry-run only") +3. Show playbook preview and key tasks when invoking playbook-executor +4. Recommend dry-run first; wait for explicit approval before actual execution + +**Action**: Execute the `/playbook-executor` skill + +**Invoke** (pass playbook metadata from playbook-generator and system-context): +``` +"Execute the generated playbook for CVE-XXXX-YYYY. Playbook file: [filename from playbook-generator]. Content: [in context from playbook-generator output]. Target systems: [list of system UUIDs from system-context]. Start with dry-run (check mode) if user requested it. Monitor job status until completion and report results." +``` + +**Git Flow path**: When playbook-executor performs Git Flow (write playbook to repo), it MUST use the absolute path for the Write tool: `/playbooks/remediation/`. Never use a relative path like `test-aap-project/playbooks/...`—that causes "Error writing file" when the repo is outside the workspace. + +**Expected**: playbook-executor validates AAP, matches templates, offers dry-run, executes on approval, streams progress, generates report. **Validates job log for CVE handling**—confirms from stdout that the playbook addressed the target CVE(s); reports ✓ confirmation or ⚠️ warning if no evidence found. After success, suggest verification with `/remediation-verifier`. + +### Step 6: Verify Deployment (Optional) + +**Action**: Execute the `/remediation-verifier` skill (if user requests verification) + +**Invoke**: +``` +"Verify remediation success for CVE-XXXX-YYYY on systems [list of system UUIDs]. Check CVE status, package versions, and service health." +``` + +**Expected**: Verification report with pass/fail. Present results to user. + +## Dependencies + +### Required MCP Tools +- None (orchestration skill—delegates to other skills that use MCP tools) + +### Required MCP Servers +- `lightspeed-mcp` - CVE data, playbook generation +- `aap-mcp-job-management` - Job launch and monitoring +- `aap-mcp-inventory-management` - Inventory for execution + +### Related Skills +- `cve-impact` - Step 1 +- `cve-validation` - Step 2 +- `system-context` - Step 3 +- `playbook-generator` - Step 4 +- `playbook-executor` - Step 5 +- `remediation-verifier` - Step 6 + +### Reference Documentation +- [references/01-remediation-plan-template.md](references/01-remediation-plan-template.md) - Plan format for user validation +- [lightspeed-mcp-tool-failures.md](../../docs/references/lightspeed-mcp-tool-failures.md) - Backend errors (e.g. explain_cves), user-friendly message, workarounds +- [cve-remediation-templates.md](../../docs/ansible/cve-remediation-templates.md) +- [package-management.md](../../docs/rhel/package-management.md) + +## Critical: Human-in-the-Loop Requirements + +This skill requires explicit user confirmation at: + +1. **Upfront Planned Tasks** (before Step 0) + - Present the 7-task plan. Wait for "yes" or "proceed" before starting any step. + - Do NOT invoke validators or other skills until the user confirms. + +2. **Remediation Plan Validation** (before Step 5) + - Present the plan: Summary + Table + Checklist + - Wait for user response: "yes"/"proceed", "dry-run only", or "abort" + - Do NOT invoke playbook-executor until the user validates the plan + +3. **Before Playbook Execution (Step 5)** + - Display playbook preview and key tasks + - Recommend dry-run first; wait for explicit approval before actual execution + +4. **Before Destructive Actions** + - Offer dry-run (check mode) before actual execution + - If dry-run approved, run first and show results + - Only proceed to actual execution after user confirms + +**Never assume approval**—always wait for explicit user confirmation before execution. + +## MCP Tool Usage + +**vulnerability__explain_cves**: Requires a valid `system_uuid` from inventory. Do NOT call it unless you have the resolved UUID from Step 3 (system-context) or Step 1 (cve-impact). Never pass `system_uuid: "undefined"` or placeholder values—this causes validation errors. For remediation availability at Step 2, use `get_cve` via cve-validation only. + +**Lightspeed tool failures**: If a tool fails with a cryptic backend error (e.g. `'dnf_modules'`), do NOT retry or expose the raw error. Use workarounds from [lightspeed-mcp-tool-failures.md](../../docs/references/lightspeed-mcp-tool-failures.md). + +## Error Handling + +- **Invalid CVE**: "CVE-XXXX-YYYY is not valid or doesn't exist. Please verify the CVE ID." +- **No Remediation Available**: "CVE-XXXX-YYYY doesn't have an automated remediation playbook. Manual patching required." +- **System Not Found**: "System XXXX is not in the Lightspeed inventory. Please ensure it's registered." +- **Batch Partial Failure**: "Successfully processed X of Y CVEs. Failed: [list]. Reason: [explanations]" +- **Lightspeed tool failures** (e.g. explain_cves `'dnf_modules'`): Do NOT show raw error. Use user-friendly message and workaround from [lightspeed-mcp-tool-failures.md](../../docs/references/lightspeed-mcp-tool-failures.md). + +## Output Format + +**Single CVE**: +``` +CVE-XXXX-YYYY Remediation Summary +CVSS Score: X.X (Severity) +Affected Packages: package-name-version +Ansible Playbook Generated: ✓ +Target Systems: N systems +[Playbook YAML or AAP link] +[Execution instructions] +``` + +**Batch**: +``` +Batch Remediation Summary +CVEs: CVE-A, CVE-B, CVE-C +Target Systems: N systems +Total Fixes: X package updates +[Consolidated playbook] +[Execution instructions] +``` + +## Important Reminders + +- **Use actual tool calls**—invoke skills via Skill tool, not text. If tool use count is 0, you are doing it wrong. +- **Orchestrate skills, don't call MCP tools directly**—skills handle docs and tools. +- **Always ask for execution confirmation** before Step 5. +- **Safety**: Test in non-prod first, back up systems, schedule maintenance windows, verify after execution. diff --git a/evaluation/with_skills/rh-sre__remediation/environment/skills/remediation/references/01-remediation-plan-template.md b/evaluation/with_skills/rh-sre__remediation/environment/skills/remediation/references/01-remediation-plan-template.md new file mode 100644 index 00000000..343d4359 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/environment/skills/remediation/references/01-remediation-plan-template.md @@ -0,0 +1,85 @@ +# Remediation Plan Template + +Read this reference when presenting plans for user validation. + +## Part A: Upfront Planned Tasks (Before Step 0) + +**When**: Before executing any step. Present immediately after the user requests remediation. + +**Purpose**: Let the user validate the approach before any work begins. + +**Format**: +``` +## Remediation: CVE-XXXX-YYYY + +**Planned tasks** (in order—use this exact order for TodoWrite/task lists; display order must match execution order): +1. Validate MCP (Lightspeed, AAP) +2. Impact analysis (assess CVE risk) +3. CVE validation (remediatable gate) +4. System context (affected systems, RHEL versions) +5. Generate playbook +6. Dry-run → User confirms → Execute +7. Verify (optional) + +❓ Proceed with this plan? +- "yes" or "proceed" — I'll start with Step 0 (validate MCP) +- "abort" — Cancel +``` + +**Wait for user response** before invoking Step 0. Do NOT start any step until the user confirms. + +--- + +## Part B: Execution Plan (After Step 4, Before Step 5) + +**When**: After Step 4 (playbook generated) and before Step 5 (execution). The user must validate before proceeding. + +## Part B Format + +### 1. Summary (1–2 sentences) + +``` +## Remediation Plan: CVE-XXXX-YYYY + +**Summary**: [One sentence describing what will happen and why.] +Example: "Remediate CVE-2026-24882 on ip-172-31-32-201 via Ansible playbook (httpd update to address CVE)." +``` + +### 2. Table (CVE, systems, key actions) + +``` +| CVE | Target Systems | Key Action | +|-----|----------------|------------| +| CVE-XXXX-YYYY | hostname-1, hostname-2 | Update package: httpd 2.4.x → 2.4.y | +``` + +For batch: one row per CVE or combined row if same action. + +### 3. Checklist (ordered steps) + +``` +**Execution steps**: +☐ Step 0: Validate MCP (Lightspeed, AAP) — done +☐ Step 1: Impact analysis — done +☐ Step 2: CVE validation — done +☐ Step 3: System context — done +☐ Step 4: Generate playbook — done +☐ Step 5: Dry-run → Confirm → Execute +☐ Step 6: Verify (optional) +``` + +Mark completed steps as "— done". Show only remaining steps as checkboxes if preferred. + +### 4. Confirm Prompt + +``` +❓ Confirm to proceed? + +- "yes" or "proceed" — Run dry-run first, then execute +- "dry-run only" — Run dry-run only, no execution +- "abort" — Cancel remediation + +Please respond with your choice. +``` + +**Wait for explicit user response** before invoking playbook-executor. diff --git a/evaluation/with_skills/rh-sre__remediation/instruction.md b/evaluation/with_skills/rh-sre__remediation/instruction.md new file mode 100644 index 00000000..ffd80028 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/instruction.md @@ -0,0 +1,19 @@ +# CVE Remediation Workflow Task + +You are a Red Hat SRE. A critical CVE has been reported and you need to plan and document a complete end-to-end remediation workflow, from initial validation through execution and verification. + +## Scenario +CVE-2026-1234 (Critical, CVSS 9.8) has been identified as affecting production RHEL systems in your fleet. Management wants a comprehensive remediation plan that covers every phase of the response. + +## Requirements +- Validate the CVE: confirm it is real, assess its severity, and determine if a remediation is available +- Assess the impact: identify which systems are affected and their criticality +- Gather system context: understand each affected system's role, dependencies, and constraints before patching +- Plan playbook generation: how the remediation playbook will be created +- Plan execution: how the playbook will be run (dry-run first, then production), including approval gates and rollback strategy +- Plan verification: how you will confirm remediation was successful after execution +- Present a phased workflow with clear decision points and user confirmation steps at each gate + +Document the complete workflow plan in `/root/report.md`. + +Use MCP tools to query data. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-sre__remediation/solution/solve.sh b/evaluation/with_skills/rh-sre__remediation/solution/solve.sh new file mode 100644 index 00000000..2721e5ff --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/solution/solve.sh @@ -0,0 +1,21 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Remediation Plan + +## Orchestration Order +1. Validate MCP connectivity +2. CVE impact analysis +3. Validate CVE remediation availability +4. Gather system context +5. Generate playbook +6. Execute playbook +7. Verify remediation + +## CVE-2024-12345 +- Remediatable: Yes +- Systems: 4 production +- Template: Kernel update with boom snapshot + +## Execution +Wait for user confirmation (yes/proceed) before Step 5 (Execute playbook). Dry-run first, then production run. +REPORT_EOF diff --git a/evaluation/with_skills/rh-sre__remediation/task.toml b/evaluation/with_skills/rh-sre__remediation/task.toml new file mode 100644 index 00000000..1922d4d5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__remediation" +name = "rh-sre CVE Remediation Planning Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "remediation", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-sre__remediation/tests/llm_judge.py b/evaluation/with_skills/rh-sre__remediation/tests/llm_judge.py new file mode 100644 index 00000000..c5278840 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "remediation_gate", "file": "/root/report.md", "question": "Does the report gate on remediation availability (checking whether automated remediation is possible for a CVE) before proceeding with playbook generation?", "reference": "A skilled report checks whether automated remediation is available as a prerequisite gate before attempting playbook generation. An unskilled report proceeds to generate playbooks without first verifying that remediation is available for the target CVEs."}, + {"id": "plan_before_execution", "file": "/root/report.md", "question": "Does the report present a Remediation Plan with summary/table/checklist for user confirmation before execution?", "reference": "A skilled report requires plan validation before execution. An unskilled report executes without plan review."}, + {"id": "two_part_confirmation", "file": "/root/report.md", "question": "Does the report describe two distinct confirmation checkpoints: one BEFORE starting (upfront planned tasks / Part A) and one AFTER playbook generation but BEFORE execution (execution plan / Part B)?", "reference": "A skilled report has Part A (upfront planned tasks before any remediation step) and Part B (execution plan confirmation after playbook is generated but before running it). An unskilled report has at most one confirmation checkpoint or no structured confirmation phases."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-sre__remediation/tests/test.sh b/evaluation/with_skills/rh-sre__remediation/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-sre__remediation/tests/test_outputs.py b/evaluation/with_skills/rh-sre__remediation/tests/test_outputs.py new file mode 100644 index 00000000..bad4f7c8 --- /dev/null +++ b/evaluation/with_skills/rh-sre__remediation/tests/test_outputs.py @@ -0,0 +1,78 @@ +""" +Tests for rh-sre__remediation per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['remediation', 'orchestrat', 'workflow']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_seven_step_sequence(self): + """Skill: Orchestrate in order: validate MCP → impact → validate CVE → context → playbook → execute → verify.""" + c = read_report().lower() + has_sequence = any(t in c for t in ["validate", "impact", "context", "playbook", "execute", "verify"]) + has_order = any(t in c for t in ["step", "phase", "before", "workflow order", "sequence"]) + assert has_sequence and has_order, ( + "should define 7-step orchestration sequence (skill: workflow order)" + ) + + def test_remediatable_gate(self): + """Skill: Gate on cve-validation: if not remediatable, stop or warn before playbook generation.""" + c = read_report().lower() + has_gate = any(t in c for t in ["remediat", "gate", "remediation_available", "advisory"]) + has_stop = any(t in c for t in ["stop", "cannot proceed", "no automated", "manual"]) + assert has_gate or has_stop, ( + "should gate on remediation availability (skill: Remediatable Gate)" + ) + + def test_plan_validation_before_execute(self): + """Skill: Present Remediation Plan (summary, table, checklist); wait for user yes/proceed before Step 5.""" + c = read_report().lower() + has_plan = any(t in c for t in ["plan", "checklist", "summary", "table"]) + has_confirm = any(t in c for t in ["confirm", "proceed", "approval", "yes", "abort"]) + assert has_plan and has_confirm, ( + "should require plan validation before execution (skill: Remediation Plan)" + ) + + def test_dry_run_recommendation(self): + """Skill: Recommend dry-run first; wait for explicit approval before actual execution.""" + c = read_report().lower() + assert any(t in c for t in ["dry-run", "dry run", "check mode", "preview"]), ( + "should recommend dry-run first (skill: before Step 5)" + ) + + def test_two_part_confirmation(self): + """Docs teach Part A (pre-Step-0) and Part B (post-Step-4) confirmations + with ordered step completion marking. Without docs, agents use single confirmation.""" + c = read_report().lower() + assert any(t in c for t in [ + "part a", "part b", "pre-step", "post-step", "two-part", + "before step 0", "after step 4", + ]) or ("confirm" in c and "step" in c), ( + "should use two-part confirmation (Part A pre-Step-0, Part B post-Step-4)" + ) diff --git a/evaluation/with_skills/rh-sre__system-context/environment/Dockerfile b/evaluation/with_skills/rh-sre__system-context/environment/Dockerfile new file mode 100644 index 00000000..484ebb33 --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/Dockerfile @@ -0,0 +1,52 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + + + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..1092dd1d --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- pass rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__system-context/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/with_skills/rh-sre__system-context/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..e826c96e --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,759 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def _system_profile_for_host(host_type: str, rhel_version: str, sid: int) -> dict: + """Generate system_profile fields for a host based on type and RHEL version.""" + el = "el9" if rhel_version.startswith("9") else "el8" + kernel = f"5.14.0-362.24.1.{el}_3.x86_64" if "9" in rhel_version else f"4.18.0-477.27.1.{el}.x86_64" + base_pkgs = [ + {"name": "kernel-core", "version": f"5.14.0-362.24.1.{el}.x86_64"}, + {"name": "httpd", "version": f"2.4.57-5.{el}"}, + {"name": "sshd", "version": f"8.9p1-23.{el}"}, + {"name": "firewalld", "version": f"1.2.5-4.{el}"}, + {"name": "systemd", "version": f"250-19.{el}"}, + ] + if "web" in host_type or "lb" in host_type: + base_pkgs.extend([ + {"name": "nginx", "version": f"1.24.1-3.{el}"}, + {"name": "openssl", "version": f"3.0.7-24.{el}"}, + ]) + elif "db" in host_type: + base_pkgs.extend([ + {"name": "postgresql", "version": f"15.4-1.{el}"}, + {"name": "openssl", "version": f"3.0.7-24.{el}"}, + ]) + elif "mon" in host_type: + base_pkgs.extend([ + {"name": "prometheus", "version": f"2.45.0-1.{el}"}, + {"name": "node_exporter", "version": f"1.6.1-2.{el}"}, + ]) + else: + base_pkgs.extend([ + {"name": "java-17-openjdk", "version": f"17.0.8-4.{el}"}, + {"name": "openssl", "version": f"3.0.7-24.{el}"}, + ]) + services = ["sshd.service", "firewalld.service", "chronyd.service"] + if "web" in host_type or "lb" in host_type: + services.append("httpd.service") + elif "db" in host_type: + services.extend(["postgresql.service", "postgresql-15.service"]) + elif "mon" in host_type: + services.extend(["prometheus.service", "node_exporter.service"]) + else: + services.append("httpd.service") + ip_octet = 10 + (sid % 245) + mac_hex = f"{(sid % 256):02x}" + return { + "installed_packages": base_pkgs[:8], + "running_services": services, + "network_interfaces": [ + {"name": "eth0", "ipv4": [f"10.0.1.{ip_octet}"], "mac": f"52:54:00:a1:b2:{mac_hex}"}, + {"name": "lo", "ipv4": ["127.0.0.1"], "mac": "00:00:00:00:00:00"}, + ], + "kernel_version": kernel, + } + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + # Add system_profile to each host + for idx, s in enumerate(systems): + host_type = "app" # default + for ht in ["web", "db", "app", "lb", "mon", "cache"]: + if ht in s["id"]: + host_type = ht + break + s["system_profile"] = _system_profile_for_host( + host_type, s["rhel_version"], idx + 1 + ) + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-sre__system-context/environment/skills/system-context/SKILL.md b/evaluation/with_skills/rh-sre__system-context/environment/skills/system-context/SKILL.md new file mode 100644 index 00000000..704ead67 --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/environment/skills/system-context/SKILL.md @@ -0,0 +1,488 @@ +--- +name: system-context +description: | + **CRITICAL**: This skill must be used for system inventory and context gathering. DO NOT use raw MCP tools like get_cve_systems or get_host_details directly. + + Gather comprehensive system inventory and deployment context for CVE-affected systems, including RHEL version detection, environment classification, and deployment analysis. Use this skill when you need to understand system infrastructure before planning remediation. + + This skill orchestrates MCP tools (get_cve_systems, get_host_details) to provide comprehensive system analysis with RHEL version detection, environment classification, and remediation strategy determination. + + **IMPORTANT**: ALWAYS use this skill instead of calling get_cve_systems or get_host_details directly for system context gathering. +--- + +# System Context Gathering Skill + +This skill gathers comprehensive system inventory and deployment context for CVE-affected systems, enabling informed remediation strategy decisions. + +**Integration with Remediation Skill**: The `/remediation` skill orchestrates this skill as part of its Step 3 (Gather Context) workflow. For standalone system analysis, you can invoke this skill directly. + +## When to Use This Skill + +**Use this skill directly when you need**: +- Understand which systems are affected by a CVE +- Analyze deployment architecture (Kubernetes, bare metal, VMs) +- Detect RHEL versions across infrastructure +- Classify systems by environment (dev/staging/prod) +- Gather context before remediation planning + +**Use the `/remediation` skill when you need**: +- End-to-end CVE remediation workflow +- Integrated analysis → context → playbook → execution +- Automated remediation strategy determination + +**How they work together**: The `/remediation` skill uses this skill's output to determine remediation strategy (batch vs individual, Kubernetes pod eviction requirements, maintenance window needs, etc.). + +## Workflow + +### 1. Identify Affected Systems + +**MCP Tool**: `get_cve_systems` or `vulnerability__get_cve_systems` (from lightspeed-mcp) + +**Parameters**: +- `cve_id`: Exact CVE identifier (format: `"CVE-YYYY-NNNNN"`) + - Example: `"CVE-2024-1234"` +- `limit`: Optional number of systems to return (default: all) + - Example: `100` + - Use for large deployments to paginate results +- `offset`: Optional pagination offset (default: 0) + - Example: `0` + - Use with limit for pagination + +**Expected Output**: List of system UUIDs and basic metadata + +**Example Response**: +```json +{ + "systems": [ + { + "id": "uuid-1", + "hostname": "web-server-01", + "display_name": "web-server-01.prod.example.com" + }, + { + "id": "uuid-2", + "hostname": "web-server-02", + "display_name": "web-server-02.prod.example.com" + } + ], + "total": 2 +} +``` + +### 2. Gather Detailed System Information + +**MCP Tool**: `get_host_details` or `inventory__get_host_details` (from lightspeed-mcp) + +**Parameters**: +- `system_id`: UUID of the system to retrieve (from get_cve_systems result) + - Example: `"uuid-1"` + - Format: UUID string +- `include_system_profile`: `true` (retrieve complete system profile including packages, services) + - Example: `true` + - Recommended: Always true for context gathering +- `include_tags`: Optional boolean to include system tags (default: true) + - Example: `true` + - Tags provide environment, role, criticality classification + +**Expected Output**: Detailed system profile + +**Key Information to Extract**: +- RHEL version (rhel_version, os_release) +- System type (infrastructure_type: bare_metal, virtualized, container) +- IP addresses (network_interfaces) +- Tags (environment, role, criticality) +- System profile (CPU, memory, disk) +- Installed packages (installed_packages) +- Running services (enabled_services, running_processes) +- Last check-in time (updated) + +**System Profile Structure**: +```json +{ + "id": "uuid-1", + "hostname": "web-server-01", + "display_name": "web-server-01.prod.example.com", + "rhel_version": "8.9", + "os_release": "Red Hat Enterprise Linux 8.9 (Ootpa)", + "system_profile": { + "os_release": "8.9", + "arch": "x86_64", + "kernel_version": "4.18.0-513.el8.x86_64", + "number_of_cpus": 4, + "number_of_sockets": 1, + "cores_per_socket": 4, + "system_memory_bytes": 17179869184, + "infrastructure_type": "virtualized", + "infrastructure_vendor": "kvm", + "network_interfaces": [ + { + "name": "eth0", + "ipv4_addresses": ["10.0.1.10"] + } + ], + "installed_packages": [...], + "enabled_services": ["httpd", "sshd", ...], + "running_processes": [...] + }, + "tags": [ + {"namespace": "environment", "key": "env", "value": "production"}, + {"namespace": "role", "key": "role", "value": "web-server"}, + {"namespace": "criticality", "key": "level", "value": "high"} + ] +} +``` + +### 3. Analyze Deployment Context + +Synthesize gathered information to understand deployment architecture: + +**A. RHEL Version Distribution**: +``` +Affected Systems by RHEL Version: +- RHEL 7: 3 systems (15%) +- RHEL 8: 15 systems (75%) +- RHEL 9: 2 systems (10%) + +Remediation Consideration: +→ Playbook must support multiple RHEL versions (use conditional yum/dnf) +``` + +**B. Environment Classification**: +``` +Affected Systems by Environment: +- Production: 12 systems (60%) - HIGH PRIORITY +- Staging: 5 systems (25%) - MEDIUM PRIORITY +- Development: 3 systems (15%) - LOW PRIORITY + +Remediation Strategy: +→ Remediate staging first for validation +→ Schedule maintenance window for production +→ Development can be patched anytime +``` + +**C. System Type Distribution**: +``` +Affected Systems by Type: +- Bare metal: 12 systems (60%) - STANDARD REMEDIATION +- VMs (VMware): 8 systems (40%) - STANDARD REMEDIATION + +Deployment Type: +→ Standard remediation workflow +→ Consider reboot requirements +→ Schedule maintenance windows for critical systems +``` + +**D. System Criticality**: +``` +Affected Systems by Criticality: +- Critical (payment, auth): 5 systems - NEEDS MAINTENANCE WINDOW +- High (web, api): 10 systems - NEEDS TESTING +- Medium (internal tools): 3 systems - STANDARD DEPLOYMENT +- Low (dev, test): 2 systems - IMMEDIATE DEPLOYMENT OK + +Remediation Approach: +→ Test on low-criticality systems first +→ Schedule maintenance for critical systems +→ Use rolling updates for high-availability services +``` + +### 5. Determine Remediation Strategy + +Based on gathered context, recommend remediation strategy: + +**Decision Matrix**: + +| Context | Remediation Strategy | +|---------|---------------------| +| Single system, non-K8s | Standard playbook, immediate execution possible | +| Multiple systems, same RHEL version | Batch playbook, parallel execution | +| Multiple systems, mixed RHEL versions | Batch playbook with version conditionals | +| Kubernetes nodes | Rolling update with pod eviction | +| Critical production systems | Maintenance window required, staged rollout | +| Mixed environments | Remediate staging → validate → production | + +**Strategy Output**: +```yaml +remediation_strategy: + approach: "rolling_update" # or "batch", "individual", "staged" + + requires_maintenance_window: true + suggested_window: "Weekend, off-peak hours" + + requires_pod_eviction: true + pod_eviction_strategy: "one_node_at_a_time" + + batch_size: 5 + parallel_execution: true + + test_first_on: + - "staging-web-01" + - "staging-web-02" + + rollout_order: + - phase: "validation" + systems: ["staging-web-01", "staging-web-02"] + wait_for_verification: true + + - phase: "production_batch_1" + systems: ["prod-web-01", "prod-web-02", "prod-web-03"] + requires_approval: true + + - phase: "production_batch_2" + systems: ["prod-web-04", "prod-web-05"] + requires_approval: false + + estimated_duration_minutes: 60 + estimated_downtime_per_system: 5 +``` + +### 6. Return Context Summary + +Return comprehensive context for remediation planning: + +```json +{ + "cve_id": "CVE-YYYY-NNNNN", + + "affected_systems": { + "total": 20, + "by_rhel_version": { + "rhel7": 3, + "rhel8": 15, + "rhel9": 2 + }, + "by_environment": { + "production": 12, + "staging": 5, + "development": 3 + }, + "by_type": { + "kubernetes": 8, + "bare_metal": 7, + "vm": 5 + }, + "by_criticality": { + "critical": 5, + "high": 10, + "medium": 3, + "low": 2 + } + }, + + "kubernetes_context": { + "has_k8s_nodes": true, + "total_k8s_nodes": 8, + "clusters": ["prod-cluster-01", "staging-cluster-01"], + "total_pods_affected": 150, + "has_pdbs": true, + "daemonsets_present": true + }, + + "remediation_strategy": { + "approach": "rolling_update", + "requires_maintenance_window": true, + "requires_pod_eviction": true, + "batch_size": 5, + "estimated_duration_minutes": 60 + }, + + "recommendations": [ + "Test in staging environment first (5 systems available)", + "Schedule maintenance window for production (12 critical systems)", + "Use rolling updates with pod eviction for Kubernetes nodes", + "Playbook must support RHEL 7, 8, and 9", + "Consider batch size of 5 systems for parallel execution" + ] +} +``` + +## Output Template + +When completing context gathering, provide output in this format: + +```markdown +# System Context Analysis + +## CVE: CVE-YYYY-NNNNN + +## Affected Systems Summary +**Total Systems**: 20 + +### By RHEL Version +- RHEL 7: 3 systems (15%) +- RHEL 8: 15 systems (75%) +- RHEL 9: 2 systems (10%) + +### By Environment +- Production: 12 systems (60%) - HIGH PRIORITY +- Staging: 5 systems (25%) - TEST FIRST +- Development: 3 systems (15%) + +### By System Type +- Kubernetes nodes: 8 systems (40%) - REQUIRES POD EVICTION +- Bare metal: 7 systems (35%) +- VMs: 5 systems (25%) + +### By Criticality +- Critical: 5 systems (payment, auth services) +- High: 10 systems (web, API services) +- Medium: 3 systems (internal tools) +- Low: 2 systems (dev/test) + +## Kubernetes Deployment +**Kubernetes Nodes**: 8 systems +**Clusters**: prod-cluster-01 (5 nodes), staging-cluster-01 (3 nodes) +**Total Pods Affected**: ~150 pods +**PodDisruptionBudgets**: Present (payment service, auth service) +**DaemonSets**: node-exporter, fluent-bit + +## Recommended Remediation Strategy + +**Approach**: Rolling Update with Pod Eviction + +**Execution Plan**: +1. **Phase 1 - Validation** (Staging): + - Systems: staging-web-01, staging-web-02 (5 systems) + - Test playbook execution + - Verify no issues before production + +2. **Phase 2 - Production Batch 1**: + - Systems: prod-web-01 to prod-web-05 (5 systems) + - Requires approval after staging validation + - Rolling update with pod eviction + +3. **Phase 3 - Production Batch 2**: + - Systems: prod-web-06 to prod-web-10 (5 systems) + - Continue if Batch 1 successful + +**Requirements**: +- Maintenance window: Weekend off-peak hours +- Pod eviction strategy: One node at a time +- Batch size: 5 systems (parallel execution) +- Estimated duration: 60 minutes total +- Estimated downtime per system: ~5 minutes + +**Safety Measures**: +- Test in staging first +- Rolling updates maintain service availability +- PodDisruptionBudgets respected +- Rollback capability via snapshots (RHEL 8/9) + +## Next Steps +1. Review remediation strategy +2. Schedule maintenance window for production +3. Generate remediation playbook (use playbook-generator skill) +4. Execute in staging for validation +5. Proceed with production deployment +``` + +## Examples + +### Example 1: Simple Environment + +**User Request**: "Gather context for CVE-2024-1234" + +**Skill Response**: +1. Call `get_cve_systems` → 5 systems affected +2. Call `get_host_details` for each → All RHEL 8, production web servers +3. Analyze context → All same version, same environment, bare metal +4. Return simple remediation strategy: "Batch remediation, 5 systems, standard playbook" + +### Example 2: Complex Multi-Version Deployment + +**User Request**: "Gather context for kernel CVE affecting production environment" + +**Skill Response**: +1. Call `get_cve_systems` → 10 systems affected +2. Call `get_host_details` → Mixed RHEL 8/9, production environment +3. Analyze system types → Mix of bare metal and VMs, high criticality tags +4. Check reboot requirements → Kernel update requires maintenance window +5. Return complex strategy: "Rolling update by RHEL version, separate playbooks for RHEL 8 and 9, coordinate maintenance window for critical systems" + +### Example 3: Multi-Environment Deployment + +**User Request**: "Gather context for CVE affecting dev, staging, and prod" + +**Skill Response**: +1. Call `get_cve_systems` → 20 systems across 3 environments +2. Call `get_host_details` → Extract environment tags +3. Classify by environment → 3 dev, 5 staging, 12 prod +4. Determine criticality → Production has critical services +5. Return staged strategy: "Test on dev (3 systems) → Validate on staging (5 systems) → Deploy to production with approval (12 systems)" + +## Error Handling + +**No systems affected**: +``` +CVE-YYYY-NNNNN Analysis Complete + +Good news! No systems in your infrastructure are currently affected by this CVE. + +Possible reasons: +- Systems are already patched +- Vulnerable packages are not installed +- Systems are running different versions + +No remediation required. +``` + +**Lightspeed inventory access error**: +``` +Unable to retrieve system details from Red Hat Lightspeed inventory. + +Possible causes: +- Systems not registered to Red Hat Lightspeed +- Lightspeed inventory sync pending +- API authentication issue + +Suggestions: +- Verify systems are registered: subscription-manager status +- Check Lightspeed connection: insights-client --status +- Re-run inventory sync: insights-client --register +``` + +**System tagging incomplete**: +``` +Unable to fully classify systems due to incomplete tagging. + +Proceeding with Red Hat Lightspeed data only. +Note: Environment and criticality tags missing from some systems. + +To improve system classification: +1. Add environment tags to systems in Red Hat Lightspeed +2. Add criticality/role tags for better prioritization +3. Ensure all systems are registered and reporting +``` + +## Dependencies + +### Required MCP Servers +- `lightspeed-mcp` - Red Hat Lightspeed platform access + +### Required MCP Tools +- `get_cve_systems` or `vulnerability__get_cve_systems` (from lightspeed-mcp) - List systems affected by CVE + - Parameters: cve_id (string, format CVE-YYYY-NNNNN), limit (number, optional), offset (number, optional) + - Returns: List of system UUIDs and basic metadata (hostname, display_name) +- `get_host_details` or `inventory__get_host_details` (from lightspeed-mcp) - Get detailed system information + - Parameters: system_id (UUID string), include_system_profile (boolean), include_tags (boolean, optional) + - Returns: Complete system profile including RHEL version, infrastructure type, tags, packages, services + +### Related Skills +- `cve-impact` - Provides CVE severity to inform criticality assessment +- `playbook-generator` - Consumes context to generate appropriate remediation playbook +- `remediation-verifier` - Uses system context to verify remediation on correct systems +- `cve-validation` - Validates CVE before gathering affected systems + +### Reference Documentation +- None required (system context is gathered from MCP tool queries) + +## Best Practices + +1. **Always gather full context** - Don't skip system details even if deployment seems simple +2. **Classify by environment** - Always test in staging before production deployment +3. **Check system criticality** - Remediation strategy depends on system importance (critical vs low) +4. **Respect criticality tags** - High-criticality systems need maintenance windows and extra care +5. **Detect RHEL version mix** - Playbooks must handle multiple versions (conditional dnf/yum logic) +6. **Consider batch size** - Balance speed vs risk (5-10 systems per batch recommended) +7. **Plan for rollback** - Always have a backup strategy (snapshots, maintenance windows) +8. **Use pagination for large fleets** - If get_cve_systems returns 100+ systems, use limit/offset parameters diff --git a/evaluation/with_skills/rh-sre__system-context/instruction.md b/evaluation/with_skills/rh-sre__system-context/instruction.md new file mode 100644 index 00000000..95d0540e --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/instruction.md @@ -0,0 +1,16 @@ +# System Context Task + +You are a Red Hat SRE. Before rolling out a remediation for a critical vulnerability, you need to gather comprehensive context about the affected systems to make safe remediation decisions. + +## Scenario +A high-severity advisory has been identified that affects multiple systems in your fleet. Before applying any patches, you need to understand each affected system's role, current health, installed packages, running services, and any special constraints (maintenance windows, compliance requirements, dependencies). + +## Requirements +- Use MCP tools to query systems in the fleet and identify those affected by the advisory +- For each affected system, gather: system role and criticality, current health and uptime, installed package versions relevant to the advisory, running services that may be impacted, and any compliance or scheduling constraints +- Assess which systems can be patched immediately vs. which need coordination +- Identify dependencies between systems that affect remediation ordering + +Document your system context analysis and remediation readiness assessment in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-sre__system-context/solution/solve.sh b/evaluation/with_skills/rh-sre__system-context/solution/solve.sh new file mode 100644 index 00000000..94c4eb6d --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/solution/solve.sh @@ -0,0 +1,19 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# System Context Report + +## Affected Systems +| System | RHEL | Environment | Infrastructure | Tags | +|--------|------|-------------|----------------|------| +| web-01 | 9.3 | Production | bare_metal | pci-compliant | +| db-01 | 8.9 | Staging | virtualized | - | + +## Data Source +get_cve_systems + get_host_details with include_system_profile=true. system_profile: rhel_version, infrastructure_type, installed_packages. + +## Remediation Strategy (Decision Matrix) +- Deployment type: Batch (multiple systems) +- Infrastructure: Bare metal, virtualized +- Maintenance window: Required for production +- Kubernetes: Rolling update with pod eviction if K8s nodes +REPORT_EOF diff --git a/evaluation/with_skills/rh-sre__system-context/task.toml b/evaluation/with_skills/rh-sre__system-context/task.toml new file mode 100644 index 00000000..d060c445 --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__system-context" +name = "rh-sre System Context Gathering Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "system-context", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-sre__system-context/tests/llm_judge.py b/evaluation/with_skills/rh-sre__system-context/tests/llm_judge.py new file mode 100644 index 00000000..c2970b3d --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "infrastructure_classification", "file": "/root/report.md", "question": "Does the report classify systems by infrastructure_type (bare_metal/virtualized/container) and infrastructure_vendor?", "reference": "A skilled report uses infrastructure classification fields. An unskilled report doesn't distinguish infrastructure types."}, + {"id": "kubernetes_safety_context", "file": "/root/report.md", "question": "Does the report consider Kubernetes context (PDBs, daemonsets) for safe remediation planning?", "reference": "A skilled report checks hasPdbs and daemonsets for safety. An unskilled report ignores K8s workload context."}, + {"id": "staged_rollout", "file": "/root/report.md", "question": "Does the report recommend staged rollout (staging first, then production batches) based on environment criticality?", "reference": "A skilled report follows staged rollout pattern. An unskilled report patches all systems simultaneously."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-sre__system-context/tests/test.sh b/evaluation/with_skills/rh-sre__system-context/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-sre__system-context/tests/test_outputs.py b/evaluation/with_skills/rh-sre__system-context/tests/test_outputs.py new file mode 100644 index 00000000..ff39869d --- /dev/null +++ b/evaluation/with_skills/rh-sre__system-context/tests/test_outputs.py @@ -0,0 +1,84 @@ +""" +Tests for rh-sre__system-context per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['system', 'context', 'environment']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_remediation_strategy_by_context(self): + """Skill: Determine strategy from context: batch vs rolling, maintenance window, pod eviction for K8s.""" + c = read_report().lower() + has_strategy = any(t in c for t in ["strategy", "approach", "rolling", "batch"]) + has_context = any(t in c for t in ["maintenance", "pod eviction", "kubernetes", "staging first"]) + assert has_strategy and has_context, ( + "should derive strategy from context (skill: Decision Matrix)" + ) + + def test_rhel_version_distribution(self): + """Skill: Report RHEL version distribution (playbook must support multiple versions).""" + c = read_report().lower() + assert any(t in c for t in ['rhel', 'version', 'distribution', 'el7', 'el8', 'el9']), ( + "Should report RHEL version distribution (skill: conditional dnf/yum)" + ) + + def test_environment_and_criticality(self): + """Skill: Classify by environment (prod/staging/dev) and criticality for rollout order.""" + c = read_report().lower() + has_env = any(t in c for t in ["staging", "development", "rollout_order", "rollout order"]) + has_crit = any(t in c for t in ["critical", "criticality", "priority", "high", "rollout"]) + assert has_env and has_crit, ( + "should classify by environment and criticality (skill: rollout_order)" + ) + + def test_infrastructure_classification(self): + """Skill: infrastructure_type (bare_metal/virtualized/container) and infrastructure_vendor (kvm) fields.""" + c = read_report().lower() + has_type = any(t in c for t in ["infrastructure_type", "infrastructure_vendor", "virtualized"]) + has_bare = "bare_metal" in c or "bare metal" in c + assert has_type or has_bare, ( + "should reference infrastructure classification (skill: bare_metal/virtualized/container)" + ) + + def test_kubernetes_context_fields(self): + """Skill: hasPdbs and daemonsets_present for safety planning in K8s context.""" + c = read_report().lower() + has_k8s = any(t in c for t in ["pdb", "daemonset"]) + has_safety = any(t in c for t in ["safety", "eviction"]) + assert has_k8s and has_safety, ( + "should reference PDB/daemonset for K8s safety (skill)" + ) + + def test_needs_restarting_check(self): + """Docs teach needs-restarting -r (exit code 0=no reboot, 1=reboot needed) + and -s for services needing restart. Without docs, agents skip this check.""" + c = read_report().lower() + assert any(t in c for t in [ + "needs-restarting", "needs_restarting", "reboot", "restart service", + ]), "should use needs-restarting for reboot/service restart assessment" diff --git a/evaluation/with_skills/rh-virt__vm-clone/environment/Dockerfile b/evaluation/with_skills/rh-virt__vm-clone/environment/Dockerfile new file mode 100644 index 00000000..ae625e01 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-clone/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/.ai-index/semantic-index.json new file mode 100644 index 00000000..2512c2a3 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/.ai-index/semantic-index.json @@ -0,0 +1,148 @@ +{ + "index_version": "1.0.0", + "last_updated": "2026-02-17", + "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", + + "documents": [ + { + "file": "INDEX.md", + "title": "VM Troubleshooting Guide - Index", + "category": "navigation", + "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], + "use_cases": ["navigation", "tool-reference", "general-guidance"], + "token_estimate": 450, + "skills": ["all"] + }, + { + "file": "scheduling-errors.md", + "title": "VM Scheduling Errors", + "category": "scheduling", + "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], + "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], + "errors_covered": [ + "ErrorUnschedulable - Node Taints", + "ErrorUnschedulable - Insufficient Resources", + "ErrorUnschedulable - Node Selector Mismatch" + ], + "token_estimate": 820, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "storage-errors.md", + "title": "VM Storage Errors", + "category": "storage", + "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], + "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], + "errors_covered": [ + "ErrorDataVolumeNotReady - DataVolume Still Provisioning", + "ErrorDataVolumeNotReady - Storage Class Not Found", + "ErrorDataVolumeNotReady - Insufficient Storage Quota", + "ErrorPvcNotFound", + "Storage Deletion Failures", + "DataVolume Cloning Failures" + ], + "token_estimate": 1350, + "skills": ["vm-creator", "vm-delete", "vm-clone"] + }, + { + "file": "lifecycle-errors.md", + "title": "VM Lifecycle Errors", + "category": "lifecycle", + "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], + "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], + "errors_covered": [ + "VM Stuck in Terminating State", + "VM Won't Start (Non-Scheduling Issues)", + "VM Won't Stop" + ], + "token_estimate": 1150, + "skills": ["vm-delete", "vm-lifecycle-manager"] + }, + { + "file": "runtime-errors.md", + "title": "VM Runtime Errors", + "category": "runtime", + "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], + "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], + "errors_covered": [ + "CrashLoopBackOff" + ], + "token_estimate": 850, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "network-errors.md", + "title": "VM Network Errors", + "category": "networking", + "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], + "use_cases": ["vm-creation", "network-troubleshooting"], + "errors_covered": [ + "Network Attachment Failures" + ], + "token_estimate": 580, + "skills": ["vm-creator"] + } + ], + + "skill_to_docs_mapping": { + "vm-creator": { + "primary_docs": ["scheduling-errors.md", "storage-errors.md"], + "secondary_docs": ["runtime-errors.md", "network-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3650, + "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" + }, + "vm-delete": { + "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 2950, + "description": "VM deletion failures - stuck terminating, storage cleanup" + }, + "vm-clone": { + "primary_docs": ["storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 1800, + "description": "VM cloning failures - DataVolume cloning, storage provisioning" + }, + "vm-lifecycle-manager": { + "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], + "secondary_docs": ["runtime-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3270, + "description": "VM start/stop failures - lifecycle management, scheduling" + }, + "vm-inventory": { + "primary_docs": [], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 450, + "description": "General guidance for status interpretation" + } + }, + + "error_to_docs_mapping": { + "ErrorUnschedulable": "scheduling-errors.md", + "ErrorDataVolumeNotReady": "storage-errors.md", + "ErrorPvcNotFound": "storage-errors.md", + "Terminating": "lifecycle-errors.md", + "CrashLoopBackOff": "runtime-errors.md", + "NetworkAttachmentFailed": "network-errors.md" + }, + + "cross_references": { + "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], + "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], + "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], + "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], + "network-errors.md": ["scheduling-errors.md"] + }, + + "progressive_disclosure_strategy": { + "description": "Load only the minimum documentation needed for current task", + "pattern": "error_type → primary_doc → cross_references (if needed)", + "token_savings": "29% average reduction vs single-file (based on rh-sre data)", + "navigation_overhead_reduction": "85% vs single-file navigation" + } +} diff --git a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/INDEX.md new file mode 100644 index 00000000..a2e76963 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/INDEX.md @@ -0,0 +1,332 @@ +--- +title: VM Troubleshooting Guide - Index +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 + - title: OpenShift Virtualization - Virtual Machine Status + url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html + date_accessed: 2026-02-06 + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] +semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] +use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] +last_updated: 2026-02-17 +--- + +# VM Troubleshooting Guide - Index + +## Overview + +This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: + +- **Creation**: Scheduling failures, storage provisioning issues +- **Lifecycle**: Start/stop failures, stuck states +- **Deletion**: Resources stuck in Terminating, storage cleanup failures +- **Cloning**: DataVolume cloning errors, cross-namespace issues +- **Networking**: Secondary network attachment failures +- **Runtime**: CrashLoopBackOff, guest OS boot failures + +This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. + +--- + +## 🗂️ Troubleshooting Categories + +The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: + +### 1. [Scheduling Errors](scheduling-errors.md) +**When to use**: VM fails to schedule on any node + +**Errors covered**: +- ErrorUnschedulable - Node Taints +- ErrorUnschedulable - Insufficient Resources +- ErrorUnschedulable - Node Selector Mismatch + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 2. [Storage Errors](storage-errors.md) +**When to use**: VM has storage provisioning, deletion, or cloning issues + +**Errors covered**: +- ErrorDataVolumeNotReady (all 3 subsections) +- ErrorPvcNotFound +- Storage Deletion Failures +- DataVolume Cloning Failures + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### 3. [Lifecycle Errors](lifecycle-errors.md) +**When to use**: VM has start/stop/termination issues + +**Errors covered**: +- VM Stuck in Terminating State +- VM Won't Start (Non-Scheduling Issues) +- VM Won't Stop + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### 4. [Runtime Errors](runtime-errors.md) +**When to use**: VM repeatedly crashes or fails at runtime + +**Errors covered**: +- CrashLoopBackOff + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 5. [Network Errors](network-errors.md) +**When to use**: VM has secondary network attachment failures + +**Errors covered**: +- Network Attachment Failures (Multus, SR-IOV) + +**Skills that use this**: vm-create + +--- + +## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last + +### Critical Principle: MCP-First Approach + +**All diagnostic procedures in this guide follow the MCP-first pattern**: + +``` +1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) +2. ⚠️ IF FAILS: Ask user permission to use CLI command +3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval +``` + +### Why MCP Tools First? + +- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources +- **Consistency**: Same interface across all operations +- **Better Error Handling**: MCP tools return structured errors +- **Audit Trail**: MCP tool usage is logged and trackable + +### Command Pattern Examples + +Throughout this guide, you'll see diagnostic steps formatted like this: + +**✅ CORRECT Pattern (MCP First)**: + +```markdown +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Extract**: `.metadata.finalizers` from returned JSON + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` +``` + +### Available MCP Tools + +The `openshift-virtualization` MCP server provides these tools: + +**Resource Operations**: +- `resources_get` - Get specific resource (replaces `oc get `) +- `resources_list` - List resources (replaces `oc get `) +- `resources_delete` - Delete resource (replaces `oc delete`) +- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) + +**Pod Operations**: +- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) +- `pods_get` - Get pod details (replaces `oc get pod`) +- `pods_log` - Get pod logs (replaces `oc logs`) +- `pods_exec` - Execute in pod (replaces `oc exec`) +- `pods_delete` - Delete pod (replaces `oc delete pod`) +- `pods_top` - Pod resource usage (replaces `oc top pods`) + +**Events & Monitoring**: +- `events_list` - List events (replaces `oc get events`) +- `nodes_top` - Node resource usage (replaces `oc top nodes`) +- `nodes_log` - Node logs (replaces `oc adm node-logs`) +- `nodes_stats_summary` - Detailed node stats + +**VM Operations** (KubeVirt toolset): +- `vm_create` - Create VMs +- `vm_lifecycle` - Start/stop/restart VMs + +**Namespaces**: +- `namespaces_list` - List namespaces (replaces `oc get namespaces`) +- `projects_list` - List OpenShift projects (replaces `oc get projects`) + +### When CLI Commands Are Required + +Some operations have **NO MCP equivalent** and require CLI: + +- `virtctl` commands (console, VNC access) +- `oc debug node` (node debugging) +- `oc auth can-i` (permission checks) +- `oc adm taint` (node taint management) +- Complex JSON patch operations + +For these, the guide will note: **"CLI Only - No MCP equivalent"** + +### Quick Reference: CLI → MCP Mapping + +| CLI Command | MCP Tool Equivalent | +|-------------|---------------------| +| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | +| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | +| `oc logs -n ` | `pods_log` with name="", namespace="" | +| `oc get events -n ` | `events_list` with namespace="" | +| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | + +**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. + +### How to Read Diagnostic Sections + +Each error section includes: +1. **Symptom** - What you observe +2. **Description** - What's happening +3. **Possible Causes** - Why it's happening +4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback +5. **Solutions** - **MCP tools first**, then CLI fallback +6. **Verification** - **MCP tools first**, then CLI fallback + +**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. + +--- + +## 🔍 Quick Navigation by Skill + +**vm-create**: +- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics +- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning +- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff +- [Network Errors](network-errors.md) - Network attachment failures + +**vm-delete**: +- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state +- [Storage Errors](storage-errors.md) - Storage deletion failures + +**vm-clone**: +- [Storage Errors](storage-errors.md) - DataVolume cloning failures + +**vm-lifecycle-manager**: +- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop +- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling + +**vm-inventory**: +- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed + +--- + +## 📊 Documentation Coverage & Maintenance + +### Current Coverage + +This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: + +- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) +- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures +- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop +- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) +- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems + +**Total errors documented**: 12 error types across 6 categories + +--- + +### Encountering Undocumented Errors + +#### For AI Agents (Claude Code) + +If you encounter an error **not documented** in the categories above: + +1. **Report to user** with all available details (error message, affected resources, namespace) +2. **Provide best-effort diagnostics** using MCP tools: + - `resources_get` to inspect resource status + - `pods_log` to check virt-launcher or compute container logs + - `events_list` to view Kubernetes events +3. **Suggest documentation update**: + ``` + ⚠️ This error is not yet documented in the troubleshooting guide. + + **Error**: + **Affected resource**: / + + I recommend adding this error to the troubleshooting guide: + - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file + - If it's a new error class → Create new category file in docs/troubleshooting/ + + Would you like me to help document this error for future reference? + ``` + +4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs + +--- + +#### For Human Contributors + +To document a new error: + +1. **Determine the category**: + - Scheduling issue → `scheduling-errors.md` + - Storage problem → `storage-errors.md` + - Lifecycle/start/stop → `lifecycle-errors.md` + - Runtime crash → `runtime-errors.md` + - Network issue → `network-errors.md` + - New error class → Create new category file (e.g., `security-errors.md`) + +2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): + - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback + - **Solutions**: MCP Tool approach → Implementation → CLI Fallback + - **Verification**: MCP Tool checks → Expected results → CLI Fallback + +3. **Update semantic index**: + - Edit `.ai-index/semantic-index.json` + - Add error to `error_to_docs_mapping` + - Update relevant skill's `primary_docs` or `secondary_docs` + - Increment token estimates if needed + +4. **Reference in skill files**: + - Update skill's Reference Documentation section + - Ensure skill's Document Consultation steps point to the right category + +5. **Update this INDEX.md**: + - Add error to the appropriate category section (lines 50-120) + - Update "Current Coverage" list above + +--- + +## 📚 Additional Resources + +- [rh-virt README](../../README.md) - MCP server setup and skill documentation +- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources +- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/lifecycle-errors.md new file mode 100644 index 00000000..6167fcc8 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/lifecycle-errors.md @@ -0,0 +1,869 @@ +--- +title: VM Lifecycle Errors +category: kubevirt +sources: + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 +tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] +semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] +use_cases: [vm-deletion, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Lifecycle Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM lifecycle issues including start/stop failures and stuck termination states. + +**When to use this document**: +- VM stuck in `Terminating` state +- VM won't start (runStrategy is Always but VM never reaches Running) +- VM won't stop (runStrategy is Halted but VM never reaches Stopped) + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### VM Stuck in Terminating State + +**Symptom**: VM shows status `Terminating` but deletion never completes + +**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). + +**Possible Causes**: +- Finalizers blocking deletion +- PVC/DataVolume still attached and preventing cleanup +- VirtualMachineInstance (VMI) not terminating properly +- Custom controllers or operators blocking deletion +- Stuck virt-launcher pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check finalizers on the VM**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` + +**2. Check if VMI still exists**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name containing "virt-launcher-". + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check events for deletion issues**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results where `involvedObject.name` == "" and sort by timestamp. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' +``` + +**5. Check VM deletion timestamp**: + +Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. + +If present, VM is in deletion process. If null, VM is not being deleted. + +**Common Finalizer Patterns**: +- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer +- `foregroundDeletion` - Waits for dependent objects to be deleted +- Custom finalizers from operators + +**Solutions** (Use MCP Tools First): + +1. **Wait for dependent resources** (safest approach): + - Use `resources_list` to check if PVCs, DataVolumes are still deleting + - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) + +2. **Force delete VMI** (if VMI is stuck): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable - requires explicit user permission): + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +3. **Force delete virt-launcher pod** (if pod is stuck): + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + # Find the virt-launcher pod name first + oc get pods -n | grep virt-launcher- + + # Then delete it + oc delete pod virt-launcher--xxx -n --force --grace-period=0 + ``` + +4. **Remove finalizers** (⚠️ dangerous - use only as last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` + 2. Remove items from `.metadata.finalizers` array + 3. Update VM using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch not easily done via MCP): + ```bash + # This operation is complex for MCP - may need CLI + oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Check for protection labels** (vm-delete skill specific): + + Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. + + If value is "true", the vm-delete skill refuses deletion (this is expected behavior). + +**Verification**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error if deletion successful. + +**CLI Fallback**: +```bash +oc get vm -n +# Should return: Error from server (NotFound) +``` + +--- + + +--- + +### VM Won't Start (Non-Scheduling Issues) + +**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state + +**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. + +**Possible Causes**: +- Guest OS kernel panic or boot failure +- Cloud-init configuration errors +- Missing or corrupted disk image +- Insufficient memory for guest OS to boot +- QEMU/KVM virtualization errors +- VirtualMachineInstance (VMI) creation failures +- virt-launcher pod crashes + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VMI (VirtualMachineInstance) status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and overall VMI state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**2. Check VMI conditions for errors**: + +Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. View virt-launcher pod logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +First, get pod name from step 3, then: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +Look for QEMU errors, memory allocation failures, device errors. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**5. Check virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. + +Alternatively, use `pods_get` to get full pod details: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**6. Access VM console to see guest OS boot messages**: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +Look for kernel panic, initramfs errors, filesystem errors. + +**7. Check VM events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep -A 20 "Events:" +``` + +**Common Error Patterns**: + +1. **Guest OS Boot Failure**: + - Console shows kernel panic + - Guest hangs at GRUB or boot loader + - Cloud-init errors during first boot + +2. **Insufficient Memory**: + - Guest OS kills processes due to OOM + - VMI logs show memory allocation errors + +3. **QEMU Crashes**: + - virt-launcher logs show QEMU segfaults + - VMI repeatedly restarts + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for kernel panic, initramfs errors, filesystem errors. + +2. **Check virt-launcher pod logs for QEMU errors**: + + Use `pods_log` from diagnostic step 4 to view logs. + + Look for: + - "qemu-system-x86_64: ..." errors + - Memory allocation failures + - Device errors + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx + ``` + +3. **Increase memory if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check VMI memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +4. **Verify disk image integrity**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check PVC is bound: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + If using container disk, verify image exists and is accessible (check VMI spec). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get pvc -n + ``` + +5. **Check cloud-init configuration** (if applicable): + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. + + Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq + ``` + +6. **Restart VMI** (soft reset): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete VMI (VM controller will recreate it): + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for new VMI to start (use `resources_get` to check status). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n -w + ``` + +7. **Check virtualization extensions** (KVM): + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + + # In debug shell: + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should eventually return `Running`). + +Check VMI is running: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should show `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should eventually return: Running + +oc get vmi -n +# Should show: Running +``` + +--- + +### VM Won't Stop + +**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state + +**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. + +**Possible Causes**: +- Guest OS not responding to ACPI shutdown signal +- virt-launcher pod stuck and not terminating +- VirtualMachineInstance (VMI) deletion blocked by finalizers +- Guest shutdown scripts hanging +- Filesystem sync issues in guest OS + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (might show `Stopping`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**2. Check VMI status and deletion timestamp**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check VMI events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vmi -n | grep -A 10 "Events:" +``` + +**5. Check VMI finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.metadata.finalizers}' +``` + +**6. Check if guest is responsive** (if VMI still exists): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Patterns**: +- VMI shows `deletionTimestamp` but never actually deletes +- virt-launcher pod in `Terminating` state +- VM runStrategy is `Halted` but printableStatus shows `Stopping` + +**Solutions** (Use MCP Tools First): + +1. **Wait for graceful shutdown** (default: 30 seconds): + + Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. + +2. **Force stop by deleting VMI**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + This is the standard way to force-stop a VM. + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n + # Should return: Error from server (NotFound) + ``` + +3. **Force delete VMI with grace period** (if VMI won't delete): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +4. **Force delete virt-launcher pod**: + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find the pod using `pods_list_in_namespace` from diagnostic step 3. + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + ⚠️ **Note**: For force deletion with grace period, use CLI fallback. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') + oc delete pod $POD_NAME -n --force --grace-period=0 + ``` + +5. **Remove VMI finalizers** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VMI using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update VMI using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Patch VM runStrategy directly** (ensure consistency): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` (diagnostic step 1) + 2. Set `.spec.runStrategy` to `"Halted"` + 3. Update VM using `resources_create_or_update` with modified JSON + + **CLI Fallback** (merge patch easier via CLI): + Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Stopped` or `Halted`). + +Verify VMI is gone: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Verify virt-launcher pod is gone: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for pods matching `virt-launcher-`. Should return no results. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Stopped or Halted + +oc get vmi -n +# Should return: Error from server (NotFound) + +oc get pods -n | grep virt-launcher- +# Should return: No resources found +``` + +**Prevention**: +- Ensure guest OS has ACPI support enabled +- Use proper shutdown commands in guest OS +- Avoid forceful stops unless necessary (can corrupt guest filesystem) + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/network-errors.md new file mode 100644 index 00000000..82fa313c --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/network-errors.md @@ -0,0 +1,429 @@ +--- +title: VM Network Errors +category: kubevirt +sources: + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] +semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] +use_cases: [vm-creation, network-troubleshooting] +related_docs: [INDEX.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Network Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. + +**When to use this document**: +- VM created successfully but secondary networks not attached +- NetworkAttachmentDefinition not found errors +- Multus CNI failures +- SR-IOV device attachment issues + +**Skills that use this**: vm-create + +--- + +### Network Attachment Failures + +**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working + +**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). + +**Possible Causes**: +- NetworkAttachmentDefinition doesn't exist in the namespace +- Multus CNI not installed or not configured on cluster +- Namespace mismatch (NAD in different namespace than VM) +- Interface name conflicts in VM spec +- Bridge/network configuration errors in NAD +- SR-IOV device not available (if using SR-IOV) + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List NetworkAttachmentDefinitions in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "" +} +``` + +Review `.items[].metadata.name` for available NADs. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -n +``` + +**2. List NetworkAttachmentDefinitions in all namespaces**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition" +} +``` + +Omit `namespace` parameter to list across all namespaces. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -A +``` + +**3. Check specific NetworkAttachmentDefinition**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "", + "name": "" +} +``` + +Review `.spec.config` for CNI configuration. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definition -n -o yaml +``` + +**4. Check VM network configuration**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.networks` to see network references. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq +``` + +**5. Check VM domain interfaces**: + +Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq +``` + +**6. Check virt-launcher pod network annotations** (shows actual attachments): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**7. Check Multus is installed**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "openshift-multus" +} +``` + +Should show Multus CNI pods running. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n openshift-multus +``` + +**8. Check for errors in virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n +``` + +**Common Error Messages**: +- `"network-attachment-definition not found"` - NAD doesn't exist in namespace +- `"multus CNI not configured"` - Multus not installed or misconfigured +- `"interface name conflict"` - Duplicate interface names in VM spec +- `"failed to add network"` - CNI plugin error (check NAD config) +- `"no available devices"` - SR-IOV device not available (if using SR-IOV) + +**Solutions** (Use MCP Tools First): + +1. **Verify NetworkAttachmentDefinition exists in correct namespace**: + + Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. + + If NAD is in different namespace, copy it to VM namespace: + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get NAD from source namespace using `resources_get` + 2. Modify `.metadata.namespace` to target namespace + 3. Create NAD in target namespace using `resources_create_or_update` + + **CLI Fallback** (stream processing easier via CLI): + Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" + ```bash + oc get network-attachment-definition -n -o yaml | \ + sed "s/namespace: /namespace: /" | \ + oc apply -f - + ``` + +2. **Create missing NetworkAttachmentDefinition**: + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + Example: Linux bridge network + + **Parameters**: + ```json + { + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": { + "name": "vlan100", + "namespace": "" + }, + "spec": { + "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" + ```bash + cat < + spec: + config: '{ + "cniVersion": "0.3.1", + "type": "bridge", + "bridge": "br1", + "vlan": 100, + "ipam": { + "type": "host-local", + "subnet": "192.168.100.0/24" + } + }' + EOF + ``` + +3. **Check Multus CNI installation**: + + Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. + + To check cluster network operator: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterOperator" + } + ``` + + Filter for `network` operator. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n openshift-multus + oc get clusteroperators network + ``` + +4. **Fix interface name conflicts** (if VM has duplicate names): + + Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. + + Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. + + **CLI Fallback** (interactive edit easier via CLI): + Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' + oc edit vm -n + ``` + +5. **Validate NAD configuration syntax**: + + Use `resources_get` from diagnostic step 3, extract `.spec.config`. + + Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get network-attachment-definition -n -o jsonpath='{.spec.config}' + ``` + +6. **Check SR-IOV device availability** (if using SR-IOV networks): + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + List SR-IOV network node policies: + + **Parameters**: + ```json + { + "apiVersion": "sriovnetwork.openshift.io/v1", + "kind": "SriovNetworkNodePolicy", + "namespace": "openshift-sriov-network-operator" + } + ``` + + Check SR-IOV device plugin pods: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "openshift-sriov-network-operator" + } + ``` + + Filter for pods with "device-plugin" in name. + + Check available SR-IOV devices on node: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "Node", + "name": "" + } + ``` + + Review `.status.allocatable` for SR-IOV resources. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get sriovnetworknodepolicy -n openshift-sriov-network-operator + oc get pods -n openshift-sriov-network-operator | grep device-plugin + oc describe node | grep -A 10 "Allocatable:" + ``` + +7. **Recreate VM with corrected network configuration** (if needed): + + If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +After remediation, check virt-launcher pod network status: + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +Should show all attached networks with status. Example output: +```json +[ + { + "name": "openshift-sdn", + "interface": "eth0", + "ips": ["10.128.2.10"], + "default": true + }, + { + "name": "vlan100", + "interface": "net1", + "ips": ["192.168.100.5"] + } +] +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**Check from inside VM** (via console): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# In guest OS: +ip addr show +# Should show all network interfaces (eth0, net1, etc.) +``` + +**Common Network Types**: +- **Linux Bridge**: Layer 2 bridge for VLAN networks +- **SR-IOV**: High-performance direct device assignment +- **macvlan**: MAC-based VLAN for container networks +- **OVN-Kubernetes**: OpenShift native overlay network + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/runtime-errors.md new file mode 100644 index 00000000..804d3cbc --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/runtime-errors.md @@ -0,0 +1,616 @@ +--- +title: VM Runtime Errors +category: kubevirt +sources: + - title: Kubernetes Pod Lifecycle + url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ + date_accessed: 2026-02-17 +tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] +semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] +use_cases: [vm-creation, vm-lifecycle, diagnostics] +related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Runtime Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. + +**When to use this document**: +- VM or virt-launcher pod shows `CrashLoopBackOff` status +- virt-launcher pod repeatedly restarting +- Guest OS kernel panics on boot + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### CrashLoopBackOff + +**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting + +**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. + +**Possible Causes**: +- Guest OS kernel panic on boot +- Insufficient resources (memory/CPU) for guest OS +- Corrupted disk image or filesystem +- QEMU/libvirt crashes due to configuration errors +- Missing or incompatible device drivers in guest +- Resource limits too low for virt-launcher pod +- Virtualization features (KVM) not available on node + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check virt-launcher pod restart count**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# Look at RESTARTS column +``` + +**2. View recent crash logs** (previous container instance): + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "previous": true, + "tail": 100 +} +``` + +Look for QEMU errors, kernel panics, or segfaults. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n virt-launcher--xxx --previous +``` + +**3. Check current virt-launcher logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. + +**CLI Fallback** (if MCP unavailable or all containers needed): +```bash +oc logs -n virt-launcher--xxx --all-containers +``` + +**4. Check VMI conditions for crash details**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Extract `.status.conditions` for crash details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**5. Check pod events for crash reasons**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" +``` + +**6. Check pod resource limits**: + +**MCP Tool**: `pods_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +Extract `.spec.containers[0].resources` for resource limits. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq +``` + +**7. Check node kubelet logs for OOM kills**: + +⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +oc adm node-logs -u kubelet | grep -i oom +``` + +**8. Access guest console** (if VM briefly starts): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Crash Patterns**: + +1. **Guest Kernel Panic**: + - Console logs show kernel panic messages + - Guest crashes immediately after boot + - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" + +2. **OOM (Out of Memory)**: + - Pod killed with reason: `OOMKilled` + - Guest runs out of memory during boot or operation + - virt-launcher logs show memory allocation failures + +3. **QEMU Crash**: + - virt-launcher logs show QEMU segmentation fault + - Symptoms: "qemu-system-x86_64: terminated by signal" + - Configuration incompatibility or QEMU bug + +4. **Disk Image Corruption**: + - Guest cannot boot from disk + - Filesystem errors in guest console + - DataVolume import failed + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for kernel panic or boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for: + - Kernel panic messages + - Initramfs errors + - Filesystem mounting failures + - Missing device errors + +2. **Review virt-launcher crash logs**: + + Use `pods_log` with `previous: true` from diagnostic step 2. + + Look for: + - QEMU command line errors + - Device initialization failures + - Memory allocation errors + - Signal termination (SIGSEGV, SIGABRT) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx --previous + ``` + +3. **Check for OOM (Out of Memory) kills**: + + **MCP Tool**: `pods_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + Extract `.status.containerStatuses[0].lastState.terminated.reason`. + + If returns `"OOMKilled"`: + - Option 1: Increase virt-launcher memory limits + - Option 2: Decrease guest memory allocation + - Option 3: Use smaller instance type + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' + ``` + +4. **Increase resources if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check current memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +5. **Verify disk image integrity**: + + **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters for list**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" + } + ``` + + **Parameters for specific DV**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check `.status.phase` (should be `Succeeded`). + + If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get datavolume -n -o jsonpath='{.status.phase}' + ``` + +6. **Check virtualization (KVM) availability**: + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +7. **Simplify VM configuration** (eliminate variables): + + Try creating minimal VM using vm-create skill with: + - Small instance type + - No secondary networks + - Simple container disk (e.g., Fedora) + - No cloud-init + + If minimal VM works, add features back one by one. + +8. **Recreate VM with different workload** (test disk image): + + If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) + +After remediation, check pod restart count stops increasing: + +**Parameters for pods**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). + +Check VM reaches Running state: + +**Parameters for VM**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# RESTARTS should stabilize + +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Running +``` + +Verify guest is responsive: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# Should show login prompt or OS console +``` + +**Advanced Debugging**: + +**MCP Tool**: `pods_exec` (from openshift-virtualization) + +Check libvirt domain XML: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["virsh", "dumpxml", "1"] +} +``` + +Check QEMU process: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["ps", "aux"] +} +``` + +Filter output for "qemu" process. + +**CLI Fallback** (if MCP unavailable): +```bash +oc exec -n virt-launcher--xxx -- virsh dumpxml 1 +oc exec -n virt-launcher--xxx -- ps aux | grep qemu +``` + +**Prevention**: +- Start with minimal VM configuration and add complexity gradually +- Use recommended instance types for your workload +- Test disk images before deploying to production +- Ensure nodes have adequate resources and KVM support +- Monitor resource usage and set appropriate limits + +--- + +## Workaround Patterns for MCP Tool Limitations + +### General Pattern: Diagnose → Propose → Confirm → Execute + +When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): + +**1. Diagnose** the root cause using Kubernetes commands +```bash +oc describe vm -n +oc get events -n --field-selector involvedObject.name= +``` + +**2. Propose** a manual workaround to the user with clear explanation +- Explain why the issue occurred +- Show what will be changed +- List alternative options + +**3. Confirm** with user before executing (Human-in-the-Loop) +- Wait for explicit "yes" or "apply workaround" +- Never auto-execute modifications + +**4. Execute** the workaround using oc/kubectl +```bash +oc patch vm -n --type=merge -p '...' +``` + +**5. Verify** the fix was successful +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**6. Document** the limitation and suggest filing enhancement request +- Note this is temporary until MCP tool is enhanced +- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues + +### Example: Adding Tolerations Workaround + +**Diagnostic Output**: +```markdown +## ⚠️ VM Scheduling Issue Detected + +**Root Cause**: Node taints prevent VM scheduling + +**Details**: +- Found 3 nodes with taint: `virtualization=true:NoSchedule` +- VM spec does not include matching tolerations +- This prevents VM scheduling on virtualization-dedicated nodes +``` + +**Proposed Workaround**: +```bash +oc patch vm web-server -n vms --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' +``` + +**User Confirmation Required**: +``` +How would you like to proceed? +- "apply workaround" - I'll patch the VM with tolerations +- "manual" - I'll provide instructions for you to apply manually +- "cancel" - Delete the VM and abort creation +``` + +**After Execution**: +```markdown +## ✓ Workaround Applied Successfully + +**Action**: Added tolerations for taint `virtualization=true:NoSchedule` +**New Status**: Stopped (VM can now be scheduled) + +**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. +Future VMs in this cluster will need the same fix until the tool is enhanced. +``` + +--- + +## VM Status Reference + +### Status Values + +| Status | Meaning | Action Required | +|--------|---------|-----------------| +| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | +| `Running` | VM is running | Normal | +| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | +| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | +| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | +| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | +| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | +| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | +| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | +| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | + +### Checking VM Status + +```bash +# Get printable status +oc get vm -n -o jsonpath='{.status.printableStatus}' + +# Get detailed status and conditions +oc get vm -n -o jsonpath='{.status}' | jq + +# Watch status changes in real-time +oc get vm -n -w +``` + +--- + +## Best Practices for Agents + +When implementing diagnostic workflows: + +1. **Always verify VM status** after creation (wait 5-10 seconds first) +2. **Consult this document** when encountering error status values +3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) +4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) +5. **Respect human-in-the-loop** for all VM modifications +6. **Document temporary workarounds** and their limitations clearly +7. **Suggest filing issues** for missing MCP tool features + +### Document Consultation Pattern + +```markdown +**Document Consultation** (REQUIRED): +1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes +2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." +``` + +--- + +## Known MCP Tool Limitations + +### vm_create tool + +**Currently Supported**: +- ✓ Namespace, name (required) +- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) +- ✓ Size hints (small, medium, large) +- ✓ Storage size +- ✓ Autostart flag +- ✓ Networks (Multus NetworkAttachmentDefinitions) +- ✓ Performance family (u1, o1, c1, m1) +- ✓ Instance type, preference + +**Not Currently Supported** (requires workarounds): +- ✗ Tolerations (for node taints) +- ✗ Node selectors +- ✗ Affinity/anti-affinity rules +- ✗ Resource requests/limits (beyond instance type) +- ✗ Custom labels/annotations +- ✗ SSH keys injection +- ✗ Cloud-init user data + +**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. + +**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues + +--- + +## Additional Resources + +- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) +- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) +- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) +- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) + +--- + +[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/scheduling-errors.md new file mode 100644 index 00000000..e47c8c4f --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/scheduling-errors.md @@ -0,0 +1,417 @@ +--- +title: VM Scheduling Errors +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 +tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] +semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] +use_cases: [vm-creation, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Scheduling Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**When to use this document**: +- VM shows status `ErrorUnschedulable` after creation or start attempt +- VM events mention scheduling failures, taints, resources, or node selectors + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +## ErrorUnschedulable + +**Symptom**: VM shows status `ErrorUnschedulable` after creation + +**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**Possible Causes**: + +### 1. Node Taints (Most Common) + +Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM events for scheduling failures**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `involvedObject.name` == "" and look for messages like: +- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= +``` + +**2. Check node taints in the cluster**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' +``` + +**Common Taint Patterns**: +- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule +- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes +- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads + +**Solution - Add Tolerations to VM**: + +The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. + +**Workaround (post-creation using MCP Tools)**: + +**Step 1**: Get current VM spec + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Step 2**: Modify the returned JSON to add tolerations + +Add to `.spec.template.spec.tolerations`: +```json +{ + "tolerations": [ + { + "key": "virtualization", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule" + } + ] +} +``` + +**Step 3**: Update VM with modified spec + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +```json +{ + "resource": "" +} +``` + +Pass the complete modified VM resource as YAML or JSON string. + +**Step 4**: Verify tolerations were added + +Use `resources_get` again and check `.spec.template.spec.tolerations` in response. + +**Step 5**: Check if VM status improved + +Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. + +**CLI Fallback** (if MCP patch is too complex): +```bash +# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' + +# Verify tolerations +oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq + +# Check status +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**Example - Multiple Tolerations**: +```bash +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: "dedicated" + operator: "Equal" + value: "virt-workloads" + effect: "NoSchedule" +' +``` + +**Toleration Operators**: +- `Equal` - Key and value must match exactly +- `Exists` - Only key must exist (ignores value) + +**Toleration Effects**: +- `NoSchedule` - Don't schedule new pods (existing pods continue) +- `PreferNoSchedule` - Avoid scheduling if possible +- `NoExecute` - Don't schedule AND evict existing pods + +**Alternative Solutions**: +1. **Remove node taints** (if you have cluster-admin access): + ```bash + oc adm taint nodes virtualization=true:NoSchedule- + ``` + +2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits + +3. **File enhancement request** - Request tolerations support in openshift-mcp-server: + https://github.com/openshift/openshift-mcp-server/issues + +--- + +### 2. Insufficient Resources + +Not enough CPU, memory, or storage available on any node. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM resource requests**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' +``` + +**2. Check node resource availability**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. + +Alternatively, use `nodes_top` MCP tool for current resource usage. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe nodes | grep -A 5 "Allocated resources" +``` + +**3. Look for VM events mentioning "Insufficient"**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep "Insufficient" +``` + +**Example Event**: +``` +0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. +``` + +**Solutions** (Use MCP Tools First): + +1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) +2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill +3. **Delete unused VMs** - Use vm-delete skill to free up resources +4. **Check resource quotas**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters for quota**: + ```json + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" + } + ``` + + **Parameters for limit range**: + ```json + { + "apiVersion": "v1", + "kind": "LimitRange", + "namespace": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + oc describe limitrange -n + ``` + +--- + +### 3. Node Selector Mismatch + +VM requires specific node labels that don't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM node selector requirements**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.nodeSelector` to see required node labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' +``` + +**2. List available node labels**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.metadata.labels` for available labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes --show-labels +``` + +**3. Check if any nodes match the selector**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node", + "labelSelector": "=" +} +``` + +Should return at least one node with matching labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -l = +``` + +**Solutions** (Use MCP Tools First): + +**Option 1: Remove node selector from VM** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get current VM using `resources_get` (diagnostic step 1) +2. Remove `.spec.template.spec.nodeSelector` field +3. Update VM using `resources_create_or_update` with modified JSON + +**CLI Fallback** (JSON patch easier via CLI): +Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" +```bash +oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' +``` + +**Option 2: Add label to nodes** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get node using `resources_get` +2. Add label to `.metadata.labels` +3. Update node using `resources_create_or_update` + +⚠️ **Note**: Node labeling typically requires cluster admin privileges. + +**CLI Fallback** (simpler via CLI): +Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" +```bash +oc label node = +``` + +--- + +[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/storage-errors.md new file mode 100644 index 00000000..16ba584d --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/storage-errors.md @@ -0,0 +1,1011 @@ +--- +title: VM Storage Errors +category: kubevirt +sources: + - title: KubeVirt CDI - DataVolumes + url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ + date_accessed: 2026-02-17 + - title: Kubernetes Persistent Volumes + url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ + date_accessed: 2026-02-17 +tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] +semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] +use_cases: [vm-creation, vm-deletion, vm-cloning] +related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] +last_updated: 2026-02-17 +--- + +# VM Storage Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. + +**When to use this document**: +- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` +- Storage deletion fails after VM deletion +- DataVolume cloning operations fail +- PVC provisioning issues + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### ErrorDataVolumeNotReady + +**Symptom**: VM shows status `ErrorDataVolumeNotReady` + +**Description**: The DataVolume (persistent storage) backing the VM is not ready. + +**Possible Causes**: + +#### 1. DataVolume Still Provisioning + +Storage provisioning takes time, especially for large disks or when importing images. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Get detailed DataVolume information**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` and `.status.conditions` for provisioning details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o json +``` + +**3. Check PVC (PersistentVolumeClaim) bound status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Check `.status.phase` for each PVC (should be `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. + +#### 2. Storage Class Not Found + +The requested storage class doesn't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List available storage classes**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" +} +``` + +Review the list of available storage classes (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass +``` + +**2. Check DataVolume's requested storage class**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.spec.pvc.storageClassName` in the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' +``` + +**Solution**: +1. Use a valid storage class from the cluster +2. Recreate VM with correct storage class parameter + +#### 3. Insufficient Storage Quota + +Namespace has insufficient storage quota to provision the PVC. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check resource quotas**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**2. Check storage usage**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase +``` + +**Solution**: +1. Request quota increase from cluster admin +2. Delete unused PVCs to free quota +3. Reduce VM storage size + +--- + +### ErrorPvcNotFound + +**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List PVCs in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review the list of available PVCs (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check VM's PVC references**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' +``` + +**Solution**: +- Wait for DataVolume to create the PVC +- Manually create missing PVC +- Fix VM spec to reference correct PVC name + +--- + + +--- + +### Storage Deletion Failures + +**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace + +**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. + +**Possible Causes**: +- PVC still bound to active PersistentVolume with `Retain` policy +- DataVolume still being referenced by another resource +- CDI (Containerized Data Importer) controller issues +- Storage class retention policy preventing deletion +- Finalizers on PVC/DataVolume blocking cleanup +- PVC still mounted by a pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check PVC status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check specific PVC phase**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Released` or `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o jsonpath='{.status.phase}' +``` + +**3. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**4. Check what's using the PVC**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' +``` + +**5. Check PVC finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. + +**6. Check DataVolume finalizers**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.metadata.finalizers}' +``` + +**7. Check PV reclaim policy**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume" +} +``` + +Filter results for PV where `.spec.claimRef.name` matches ``. + +To get specific PV policy: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume", + "name": "" +} +``` + +Check `.spec.persistentVolumeReclaimPolicy`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pv | grep +oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' +``` + +**Common Finalizer Patterns**: +- `kubernetes.io/pvc-protection` - Protects PVC while in use +- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer + +**Solutions** (Use MCP Tools First): + +1. **Delete DataVolume first, then PVC**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete DataVolume first (often blocks PVC deletion): + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Wait a few seconds, then delete PVC: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete datavolume -n + oc delete pvc -n + ``` + +2. **Check for pods still using PVC**: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "" + } + ``` + + Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. + + Then delete the pods using `pods_delete`: + + **Parameters**: + ```json + { + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' + oc delete pod -n + ``` + +3. **Force delete PVC** (if safe to do so): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete pvc -n --grace-period=0 --force + ``` + +4. **Remove finalizers from PVC** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PVC using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update PVC using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Remove finalizers from DataVolume** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current DataVolume using `resources_get` (diagnostic step 6) + 2. Remove items from `.metadata.finalizers` array + 3. Update DataVolume using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Change PV reclaim policy** (if PV has Retain policy): + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PV policy using `resources_get` (diagnostic step 7) + 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` + 3. Update PV using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" + ```bash + oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' + oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' + ``` + +**Storage Quota Check** (Use MCP Tools First): + +After deletion, verify storage quota is freed: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters for quota check**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.used` to verify storage quota is freed. + +**Parameters for PVC verification**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +oc get pvc -n +``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Confirm PVC is deleted: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Confirm DataVolume is deleted: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +# Should return: Error from server (NotFound) + +oc get datavolume -n +# Should return: Error from server (NotFound) +``` + +--- + +### DataVolume Cloning Failures + +**Symptom**: VM clone created successfully but DataVolume clone operation fails + +**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. + +**Possible Causes**: +- CSI driver doesn't support volume cloning +- Source PVC storage class incompatible with cloning +- Cross-namespace cloning not permitted by storage backend +- Insufficient storage quota in target namespace +- Source PVC not in `Bound` state +- Storage class doesn't have volume cloning enabled +- CDI (Containerized Data Importer) controller issues + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume clone status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and `.metadata.name`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Check DataVolume events for errors**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe datavolume -n +``` + +**3. Check DataVolume phase**: + +Use `resources_get` from step 1, extract `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +``` + +**4. Check if storage class supports cloning**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "name": "" +} +``` + +Review the full YAML output for cloning-related configurations. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass -o yaml | grep -A 5 -i clone +``` + +**5. Check CSI driver capabilities**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "CSIDriver" +} +``` + +Review `.items[].metadata.name` for available CSI drivers. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get csidriver +``` + +**6. Check source PVC status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Bound` for cloning to work). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**7. Check target namespace storage quota**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**8. Check CDI controller logs**: + +**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) + +First, list pods in openshift-cnv namespace: + +**Parameters for pods_list_in_namespace**: +```json +{ + "namespace": "openshift-cnv", + "labelSelector": "app.kubernetes.io/component=cdi-deployment" +} +``` + +Then get logs using `pods_log`: + +**Parameters**: +```json +{ + "namespace": "openshift-cnv", + "name": "", + "tail": 100 +} +``` + +**CLI Fallback** (if MCP unavailable or easier via CLI): +```bash +oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') +``` + +**Common Error Messages**: +- `"volume cloning is not supported"` - CSI driver lacks clone capability +- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage +- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace +- `"insufficient quota"` - Target namespace lacks storage quota +- `"source PVC not bound"` - Source PVC must be in Bound state for cloning +- `"StorageClass does not support cloning"` - Storage class configuration issue + +**Solutions** (Use MCP Tools First): + +1. **Check storage class clone support**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" + } + ``` + + For each storage class in `.items[]`, check: + - `.metadata.name` (storage class name) + - `.provisioner` (CSI driver) + + Storage classes using CSI drivers typically support cloning. Look for provisioners like: + - `csi.ovirt.org` (oVirt CSI) + - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) + - `ebs.csi.aws.com` (AWS EBS CSI) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner + ``` + +2. **Verify source PVC is bound**: + + Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pvc -n -o jsonpath='{.status.phase}' + ``` + +3. **Check target namespace quota**: + + Use `resources_list` from diagnostic step 7 to check quota. + + If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + ``` + +4. **Use snapshot-based cloning** (alternative method): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Step 1**: Create VolumeSnapshot of source PVC + + **Parameters**: + ```json + { + "apiVersion": "snapshot.storage.k8s.io/v1", + "kind": "VolumeSnapshot", + "metadata": { + "name": "-snapshot", + "namespace": "" + }, + "spec": { + "source": { + "persistentVolumeClaimName": "" + } + } + } + ``` + + **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) + + **Step 3**: Create new DataVolume from snapshot + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": "-rootdisk", + "namespace": "" + }, + "spec": { + "source": { + "snapshot": { + "name": "-snapshot", + "namespace": "" + } + }, + "storage": { + "resources": { + "requests": { + "storage": "50Gi" + } + }, + "storageClassName": "" + } + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" + ```bash + cat <-snapshot + namespace: + spec: + source: + persistentVolumeClaimName: + EOF + + oc get volumesnapshot -snapshot -n + + cat <-rootdisk + namespace: + spec: + source: + snapshot: + name: -snapshot + namespace: + storage: + resources: + requests: + storage: 50Gi + storageClassName: + EOF + ``` + +5. **Use "new empty storage" option** (vm-clone skill): + - If cloning isn't supported, create VM with empty storage + - Manually copy data if needed + +6. **Cross-namespace cloning workaround**: + - Some storage backends require snapshot for cross-namespace cloning + - Create snapshot in source namespace, restore in target namespace (see solution 4 above) + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Check DataVolume reached Succeeded phase: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should return `Succeeded`). + +Check PVC was created and bound: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "-rootdisk" +} +``` + +Check `.status.phase` (should return `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +# Should return: Succeeded + +oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' +# Should return: Bound +``` + +**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-clone/environment/mcp-servers/mock-virt-mcp.py b/evaluation/with_skills/rh-virt__vm-clone/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..70ce07d7 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-clone/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1465 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), + ("prod-vms", {"env": "production"}), + ("test-env", {"env": "testing"}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── prod-vms (instruction-specific) ────────────────────────────────── + _vm("production-db", "prod-vms", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true"}, + 8, 16, "Running", True, 1), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-virt__vm-clone/environment/skills/vm-clone/SKILL.md b/evaluation/with_skills/rh-virt__vm-clone/environment/skills/vm-clone/SKILL.md new file mode 100644 index 00000000..09ad791f --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-clone/environment/skills/vm-clone/SKILL.md @@ -0,0 +1,456 @@ +--- +name: vm-clone +description: | + Clone existing virtual machines for testing, scaling, or creating templates. + + Use when: + - "Clone VM [source] to [target]" + - "Create a copy of VM [name]" + - "Duplicate VM [name] for testing" + - "Create 3 copies of template-vm" + + This skill clones VM configuration and optionally creates new storage or references existing storage. + + NOT for snapshots (use vm-snapshot for point-in-time backups). + +model: inherit +color: blue +--- + +# /vm-clone Skill + +Clone existing virtual machines in OpenShift Virtualization, creating new VMs with copied configuration and optional storage cloning. This skill is ideal for creating test environments, scaling workloads, or duplicating VM templates. + +## Prerequisites + +**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools**: +- `resources_get` (from openshift-virtualization) - Get source VM configuration +- `resources_create_or_update` (from openshift-virtualization) - Create cloned VM +- `resources_list` (from openshift-virtualization) - List DataVolumes, PVCs, VMs + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster (>= 4.19) +- OpenShift Virtualization operator installed +- ServiceAccount with RBAC permissions to create VirtualMachine and PVC resources +- Source VM must exist + +### Prerequisite Verification + +**Before executing:** +1. Verify `openshift-virtualization` in `.mcp.json`, `KUBECONFIG` set (never expose value) +2. Optional: Verify RBAC permissions for VirtualMachine, PVC/DataVolume creation + +**When prerequisites fail:** +``` +❌ Cannot execute vm-clone: MCP server 'openshift-virtualization' is not available + +Setup: Add openshift-virtualization to .mcp.json (see https://github.com/openshift/openshift-mcp-server) +Set KUBECONFIG environment variable, restart Claude Code + +Options: "setup" (configure now), "skip" (skip skill), "abort" (stop workflow) +``` + +⚠️ **SECURITY**: Never display KUBECONFIG path or credential values. + +## When to Use This Skill + +**Trigger this skill when:** +- User explicitly invokes `/vm-clone` command +- User wants to duplicate an existing VM +- User needs to create test/dev copies of production VMs +- User wants to scale horizontally by creating VM copies +- User wants to create VMs from a template VM + +**User phrases that trigger this skill:** +- "Clone VM web-server to web-server-test" +- "Create a copy of database-vm" +- "Duplicate production-vm for staging" +- "Make 3 copies of template-vm" +- "/vm-clone" (explicit command) + +**Do NOT use this skill when:** +- User wants to create a new VM from scratch → Use `/vm-create` skill instead +- User wants a point-in-time backup → Use snapshots instead +- User wants to move/migrate a VM → Use migration tools instead +- User wants to resize a VM → Modify existing VM instead + +## Workflow + +### Step 1: Gather Source VM Information + +**Required Information from User:** +1. **Source VM Name** - Name of the VM to clone +2. **Source Namespace** - Namespace where source VM exists +3. **Target VM Name** - Name for the cloned VM +4. **Target Namespace** - Namespace for the cloned VM (can be same or different) + +If user doesn't provide all information, ask for missing details. + +**1.1: Verify Source VM Exists** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Expected Output**: Complete VirtualMachine resource specification + +**Error Handling**: +- If VM not found → Report error, suggest using vm-inventory to find VMs +- If permission denied → Report RBAC error + +**1.2: Check Target VM Name Availability** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**If VM exists**: Offer options (choose different name, delete existing VM, cancel), wait for decision + +**1.3: Discover Source VM Storage** + +Use `resources_list` for DataVolumes (labelSelector: vm.kubevirt.io/name) or PVCs if not found +Parse: Extract storage names, calculate size, determine DataSources vs container disks + +### Step 2: Ask User for Cloning Strategy + +**Present storage cloning options:** + +```markdown +## VM Cloning - Storage Strategy + +**Source VM**: `` (namespace: ``) +**Source Storage**: () + +**Select cloning strategy:** + +1. **Clone Storage** - Full copy, independent storage (~5-10 min, new allocation) +2. **Reference Existing** - ⚠️ Shared disk (dangerous, both VMs access same storage) +3. **New Empty Storage** - Fresh disk, no data copied ( new allocation) +4. **Cancel** - Abort operation + +**Select option (1-4):** +``` + +**Wait for user selection (1-4).** + +**Handle response**: "4"/cancel → stop; "1" → clone_storage=true; "2" → warn + share_storage=true; "3" → new_storage=true + +**Option 2 warning**: `⚠️ Shared Storage Dangerous - Both VMs share disk, data corruption risk. Only safe if source stopped. Use Option 1 instead. Proceed anyway? (yes/cancel)` Wait for explicit "yes". + +### Step 3: Present Clone Configuration for Confirmation + +**Present configuration summary:** + +```markdown +## VM Clone Configuration - Review + +**Source**: `` () - , vCPU, +**Target**: `` () - Same config, starts Stopped +**Storage**: - + +**Resource Impact**: vCPU, RAM, disk + +**What changes**: IP addresses, hostname, MAC addresses, firmware UUID +**What's preserved**: Instance type, vCPU/memory, network config, cloud-init + +**Proceed with VM cloning? (yes/no)** +``` + +**Wait for user confirmation.** + +**Handle response:** +- If "yes" → Proceed to Step 4 (execute cloning) +- If "no", "cancel", "wait", or anything else → Cancel operation + +**On cancellation:** +```markdown +VM cloning cancelled by user. No resources were created. +``` + +**STOP workflow**. + +### Step 4: Execute VM Cloning + +**ONLY PROCEED AFTER**: Source VM validated, target name available, user selected storage strategy, user confirmed configuration + +**4.1: Prepare Cloned VM Specification** + +Modify source VM spec: +1. Change metadata: `name` → target-vm-name, `namespace` → target-namespace; remove `uid`, `resourceVersion`, `creationTimestamp`, `status` +2. Update storage: clone_storage → new DataVolume with source PVC; share_storage → keep PVC refs; new_storage → empty DataVolume +3. Set `runStrategy: Halted` (starts stopped) +4. Generate new firmware UUIDs (`domain.firmware.uuid`, `domain.firmware.serial`) +5. Preserve: instance type, tolerations, network config, cloud-init + +**4.2: Create Storage Resources** + +**Clone storage** - Use `resources_create_or_update` with DataVolume (source.pvc from source, storage from source class/size) +**New empty storage** - Use `resources_create_or_update` with DataVolume (source.blank, storage from source class/size) + +**4.3: Create Cloned VirtualMachine** + +Use `resources_create_or_update` with prepared spec from 4.1 +**Error handling**: Creation fails → report error, rollback storage; permission denied → RBAC error; namespace missing → namespace error + +**4.4: Monitor Storage Cloning Progress** + +Use `resources_get` on DataVolume, check `status.phase` (Pending/Succeeded/Failed), report every 30s, wait up to 15 min + +### Step 5: Report Cloning Results + +**On successful clone:** + +```markdown +## ✓ VM Cloned Successfully + +**Source**: `` () +**Target**: `` () - Status: Stopped (ready to start) +**Config**: , vCPU, , + + +**Storage**: ✓ Cloned in + +**Storage**: ✓ New empty storage created - OS installation may be required + + +**Storage**: ⚠️ Shared PVC `` - Keep source VM stopped to avoid data corruption + + +**Next**: Start with `"Start VM in namespace "` +``` + +**On cloning failure:** + +**Document Consultation** (OPTIONAL - when cloning fails): +- **When to consult**: Storage cloning fails, VM creation fails, PVC clone not supported, storage class issues +- **When NOT to consult**: VM already exists, RBAC errors, namespace not found (clear causes) +- **Action**: Read [storage-errors.md](../../docs/troubleshooting/storage-errors.md) for VM cloning failures, storage provisioning, DataVolume errors +- **Output to user**: "I consulted [storage-errors.md](../../docs/troubleshooting/storage-errors.md) to understand potential causes." + +```markdown +## ❌ VM Cloning Failed + +**Error**: +**Source**: `` () → **Target**: `` () + +**Common Causes**: +- Insufficient storage quota - Namespace lacks storage capacity +- Insufficient RBAC permissions - ServiceAccount lacks create permissions +- Storage class not available - Target namespace cannot access storage class +- PVC clone not supported - Storage class doesn't support cloning +- Source VM still running - Some storage backends require source VM stopped + +**Troubleshooting** (see [storage-errors.md](../../docs/troubleshooting/storage-errors.md)): +1. Check storage quota: `resources_list` for ResourceQuota in target namespace +2. Check permissions: `resources_list` to verify RBAC (note: `oc auth can-i` has no MCP equivalent) +3. Check storage class: `resources_get` for StorageClass config, `resources_list` for available classes +4. Check source VM status: vm-inventory skill `"Show status of VM "` +5. Check DataVolume status: `resources_get` for DataVolume phase and status + +**Partial Resources** (may need cleanup): +- VirtualMachine: `` +- DataVolume: `-rootdisk` + +**Cleanup**: `"Delete VM in namespace "` + +Would you like help troubleshooting this error? +``` + +## Advanced Features + +### Batch Cloning +**User request:** "Create 3 copies of template-vm named web-01, web-02, web-03" +**Workflow**: Validate source once, generate/check target names, present combined scope, ask storage strategy, confirm, execute sequentially +**Batch confirmation**: Show source, targets list, strategy, total impact (VMs, storage, vCPU, memory), estimated time + +### Cross-Namespace Cloning +**User request:** "Clone production-vm from production to staging namespace" +**Note**: Storage cloned across namespaces, network policies/quotas may differ, RBAC required in both namespaces + +### Clone with Modifications (Future) +Allow modifications during clone: instance type/size, storage size, network config, cloud-init customization + +## Common Issues + +**Issue 1: Target VM Name Already Exists** - Choose different name, delete existing VM (if safe), use vm-inventory to check + +**Issue 2: Insufficient Storage Quota** - Check quotas, request increase, use shared storage (if appropriate), delete unused PVCs + +**Issue 3: Storage Class Not Accessible** - Verify storage class exists in target namespace, check cross-namespace cloning support, use different storage class, contact admin + +**Issue 4: PVC Clone Not Supported** - Storage class doesn't support CSI volume cloning; use "new empty storage" option, snapshot and restore, or check storage class capabilities + +**Issue 5: Source VM Running During Clone** - Stop source VM first, use snapshot-based cloning, check storage backend requirements + +## Dependencies + +### Required MCP Servers +- `openshift-virtualization` - OpenShift MCP server with core and kubevirt toolsets + +### Required MCP Tools +- `resources_get` (from openshift-virtualization) - Get source VM and storage details + - Parameters: apiVersion, kind, namespace, name + - Source: https://github.com/openshift/openshift-mcp-server/blob/main/pkg/toolsets/core/resources.go + +- `resources_create_or_update` (from openshift-virtualization) - Create cloned VM and storage + - Parameters: resource (YAML/JSON) + - Source: https://github.com/openshift/openshift-mcp-server/blob/main/pkg/toolsets/core/resources.go + +- `resources_list` (from openshift-virtualization) - List DataVolumes, PVCs, VMs + - Parameters: apiVersion, kind, namespace, labelSelector + - Source: https://github.com/openshift/openshift-mcp-server/blob/main/pkg/toolsets/core/resources.go + +### Related Skills +- `vm-create` - Create new VMs from scratch (alternative to cloning) +- `vm-inventory` - List and verify source/target VMs +- `vm-lifecycle-manager` - Start cloned VMs after creation +- `vm-delete` - Clean up failed clones or unwanted copies + +### Reference Documentation +- [storage-errors.md](../../docs/troubleshooting/storage-errors.md) - VM cloning failure scenarios, storage provisioning issues, and DataVolume cloning errors (optionally consulted when cloning operations fail) +- [Troubleshooting INDEX](../../docs/troubleshooting/INDEX.md) - Navigation hub for discovering additional error categories when encountering unexpected issues outside the categories above +- [OpenShift Virtualization Cloning](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt/virtual_machines/cloning_vms/virt-cloning-vm.html) +- [DataVolume Cloning](https://github.com/kubevirt/containerized-data-importer/blob/main/doc/datavolumes.md#cloning) +- [KubeVirt VirtualMachine API](https://kubevirt.io/api-reference/) +- [CSI Volume Cloning](https://kubernetes.io/docs/concepts/storage/volume-pvc-datasource/) + +## Critical: Human-in-the-Loop Requirements + +**IMPORTANT:** This skill creates new resources that consume cluster capacity. You MUST: + +1. **Before Cloning** + - Verify source VM exists and get full configuration + - Ask user for clone configuration (name, namespace, storage strategy) + - Present clone preview with resource impact + - Wait for explicit user confirmation + +2. **Configuration Confirmation** + - Display source VM details + - Show target VM configuration + - Indicate storage cloning strategy + - Estimate resource consumption (CPU, memory, storage) + - Ask: "Proceed with VM cloning? (yes/no)" + - Wait for explicit "yes" + +3. **Never Auto-Execute** + - **NEVER clone without user confirmation** + - **NEVER assume storage strategy** - always ask user + - **NEVER proceed if user says "no", "wait", "cancel"** + +**Why This Matters:** +- **Resource Consumption**: Clones consume cluster resources (CPU, memory, storage) +- **Storage Costs**: Storage cloning can consume significant disk space +- **Naming Conflicts**: Duplicate names cause errors +- **Network Configuration**: May need adjustment for clones + +## Security Considerations + +- **RBAC Enforcement**: Requires create permissions in target namespace +- **Namespace Isolation**: Cannot clone across namespaces without proper permissions +- **Storage Quotas**: Respects namespace storage quotas +- **MAC Address Generation**: Automatic generation prevents MAC conflicts +- **UUID Generation**: New firmware UUIDs prevent conflicts +- **No Credential Exposure**: KUBECONFIG never exposed in output +- **Audit Trail**: All cloning operations logged in Kubernetes API audit logs + +## Example Usage + +### Example 1: Simple Clone with Storage + +``` +User: "Clone web-server to web-server-test in namespace dev" + +Agent: [Validates source VM web-server (production), checks target name availability] + [Discovers 50Gi storage, presents storage options] + "Select storage strategy (1-4):" + +User: "1" + +Agent: [Shows: Source web-server (u1.medium, 2 vCPU, 4Gi) → Target web-server-test (dev, 50Gi clone)] + "Proceed with VM cloning? (yes/no)" + +User: "yes" + +Agent: "📦 Creating storage... ⏳ Cloning (5-10 min)..." + "✓ VirtualMachine web-server-test created in dev" + "⏳ Storage cloning... Succeeded" + "✓ VM Cloned Successfully - Source: web-server (production), Target: web-server-test (dev)" + "Storage: ✓ Cloned in 7m23s - Independent storage" + "Next: Start VM web-server-test in namespace dev" +``` + +### Example 2: Clone with New Empty Storage + +``` +User: "Clone template-vm to app-01 with empty storage" + +Agent: [Validation complete, presents options] + "Select storage strategy (1-4):" + +User: "3" + +Agent: "Proceed with VM cloning? (yes/no)" + +User: "yes" + +Agent: "✓ VM Cloned Successfully" + "Storage: ✓ New empty storage created - OS installation may be required" +``` + +### Example 3: Batch Cloning + +``` +User: "Create 3 copies of template-vm named web-01, web-02, web-03 in production" + +Agent: [Validates source, checks all names, presents batch review] + "Source: template-vm, Targets: 3 VMs (web-01, web-02, web-03)" + "Total Impact: 90Gi, 6 vCPU, 12Gi, ~20-30 min" + "Proceed? (yes/no)" + +User: "yes" + +Agent: "📦 Cloning VM 1 of 3: web-01... ✓" + "📦 Cloning VM 2 of 3: web-02... ✓" + "📦 Cloning VM 3 of 3: web-03... ✓" + "✓ Batch Cloning Completed - 3 VMs, 90Gi storage, all Stopped" +``` + +### Example 4: Shared Storage Warning + +``` +User: "Clone db-vm to db-test" + +Agent: "Select storage strategy (1-4):" + +User: "2" + +Agent: "⚠️ WARNING: Shared Storage is Dangerous" + "Risks: Data corruption if both VMs run simultaneously" + "Proceed with shared storage anyway? (yes/cancel)" + +User: "cancel" + +Agent: "VM cloning cancelled. No resources created." +``` diff --git a/evaluation/with_skills/rh-virt__vm-clone/instruction.md b/evaluation/with_skills/rh-virt__vm-clone/instruction.md new file mode 100644 index 00000000..922cf2fb --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-clone/instruction.md @@ -0,0 +1,13 @@ +# VM Cloning Task + +You are an OpenShift Virtualization administrator. The QA team needs an exact copy of the production database VM to test a schema migration. Plan the clone so it can run safely in the test environment without affecting production. + +## Requirements +- Inspect the source VM (`production-db` in `prod-vms`) to understand its current state, storage configuration, and whether it needs to be stopped for cloning +- Determine the right cloning approach: whether the storage backend supports efficient cloning, and whether the VM needs to be offline +- Plan the clone target (`test-db-clone` in `test-env`) ensuring it has no network or storage conflicts with the source +- Verify the clone will be fully independent: separate disks, no shared PVCs, and safe to modify without affecting production + +Document your cloning plan in `/root/report.md`. + +Use MCP tools to examine the cluster. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-virt__vm-clone/solution/solve.sh b/evaluation/with_skills/rh-virt__vm-clone/solution/solve.sh new file mode 100644 index 00000000..17534b7f --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-clone/solution/solve.sh @@ -0,0 +1,34 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Cloning Plan + +## Source: production-db (prod-vms) → Target: test-db-clone (test-env) + +### Cloning Strategy +Using **Clone Storage** (full copy) strategy. Three strategies available: +1. **Clone Storage** (selected) — Full copy of all DataVolumes/PVCs. Independent clone. +2. **Reference Existing** — Shared disk. NOT safe for database workloads. +3. **New Empty Storage** — Fresh disk. Loses data. + +Full copy ensures test-db-clone is completely independent from production-db. + +### Spec Modifications for Clone +- Set `runStrategy: Halted` (don't auto-start the clone) +- Regenerate `domain.firmware.uuid` and `domain.firmware.serial` to avoid conflicts +- Update metadata.name to `test-db-clone` +- Update metadata.namespace to `test-env` +- Update DataVolume names to avoid collision + +### Storage Cloning +- Discover source DataVolumes via label: `vm.kubevirt.io/name=production-db` +- CSI volume cloning support required on the StorageClass +- Create new DataVolume with `source.pvc` referencing the original +- **Reference Existing** = shared disk — data corruption risk if both VMs run +- Use `resources_create_or_update` to create cloned VM and DataVolume + +### Verification +- Check target name `test-db-clone` doesn't exist in `test-env` +- Verify CSI driver supports volume cloning +- Monitor DataVolume clone progress + +REPORT_EOF diff --git a/evaluation/with_skills/rh-virt__vm-clone/task.toml b/evaluation/with_skills/rh-virt__vm-clone/task.toml new file mode 100644 index 00000000..46cd3410 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-clone/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-clone" +name = "rh-virt VM Cloning Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-clone", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-virt__vm-clone/tests/llm_judge.py b/evaluation/with_skills/rh-virt__vm-clone/tests/llm_judge.py new file mode 100644 index 00000000..749897f2 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-clone/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "firmware_uuid_regeneration", "file": "/root/report.md", "question": "Does the report address firmware UUID/serial regeneration to avoid identity conflicts between source and clone?", "reference": "A skilled report regenerates domain.firmware.uuid and serial in the clone spec. An unskilled report clones without addressing firmware identity."}, + {"id": "storage_clone_strategy", "file": "/root/report.md", "question": "Does the report discuss DataVolume clone strategy using source.pvc and StorageClass considerations?", "reference": "A skilled report uses DataVolume with source.pvc and considers CSI clone support. An unskilled report copies data manually."}, + {"id": "halted_run_strategy", "file": "/root/report.md", "question": "Does the report set runStrategy: Halted for the cloned VM to start in Stopped state?", "reference": "A skilled report ensures the clone starts halted. An unskilled report starts the clone immediately, risking conflicts."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-virt__vm-clone/tests/test.sh b/evaluation/with_skills/rh-virt__vm-clone/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-clone/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-virt__vm-clone/tests/test_outputs.py b/evaluation/with_skills/rh-virt__vm-clone/tests/test_outputs.py new file mode 100644 index 00000000..1638de54 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-clone/tests/test_outputs.py @@ -0,0 +1,90 @@ +""" +Tests for rh-virt__vm-clone per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_source_and_target(self): + content = read_report().lower() + has_source = any(t in content for t in ["source", "original", "production"]) + has_target = any(t in content for t in ["clone", "target", "copy", "destination"]) + assert has_source and has_target, "report should identify both a source VM and a clone target" + + +class TestSkillDependent: + def test_storage_class_cloning(self): + """Skill: StorageClass/CSI for PVC cloning strategy.""" + c = read_report().lower() + assert any(t in c for t in ["storageclass", "storage class", "csi", "volume cloning", "pvc clone", "clone support"]), ( + "should mention StorageClass or CSI cloning for clone strategy" + ) + + def test_identity_conflict(self): + """Skill: hostname, cloud-init, SSH key, firmware UUID conflicts between source and clone.""" + c = read_report().lower() + assert any(t in c for t in ["hostname", "cloud-init", "cloud init", "ssh key", "firmware", "uuid", "mac address", "identity conflict"]), ( + "should address identity conflicts (hostname, cloud-init, UUID) between source and clone" + ) + + def test_cross_namespace_rbac(self): + """Skill: RBAC/permissions for cross-namespace cloning.""" + c = read_report().lower() + assert any(t in c for t in ["rbac", "permission", "cross-namespace", "cross namespace", "target namespace", "create virtualmachine"]), ( + "should address RBAC or permissions for cross-namespace cloning" + ) + + def test_data_volume_cloning(self): + """Skill: DataVolume with source PVC for clone provisioning.""" + c = read_report().lower() + assert any(t in c for t in ["datavolume", "data volume", "source.pvc", "source pvc", "pvc datasource", "clone storage"]), ( + "should discuss DataVolume or PVC cloning for clone storage" + ) + + def test_datavolume_progress(self): + """Skill: Monitor DataVolume phase (Pending/Succeeded) during clone.""" + c = read_report().lower() + assert any(t in c for t in ["datavolume", "phase", "pending", "succeeded", "cloning progress", "status.phase"]), ( + "should mention monitoring DataVolume phase during clone" + ) + + def test_firmware_uuid_regeneration(self): + """Skill teaches domain.firmware.uuid and domain.firmware.serial must be + regenerated in clone spec to avoid identity conflicts. Without skill, + agents clone without regenerating firmware identifiers.""" + c = read_report().lower() + assert "firmware" in c and ("uuid" in c or "serial" in c), ( + "should address firmware UUID/serial regeneration for clone" + ) + + def test_run_strategy_halted_for_clone(self): + """Skill teaches runStrategy: Halted ensures cloned VM starts in Stopped state. + Without skill, agents start clone immediately.""" + c = read_report().lower() + assert any(t in c for t in ["halted", "runstrategy", "run strategy"]) and ( + "clone" in c or "stop" in c + ), "should set runStrategy: Halted for cloned VM" + + def test_source_pvc_bound(self): + """Docs teach CSI clone prerequisite: source PVC must be in Bound state. + Without docs, agents attempt cloning from unbound PVCs.""" + c = read_report().lower() + assert any(t in c for t in [ + "bound", "pvc status", "source pvc", "prerequisite", + ]) and ("pvc" in c or "storage" in c), ( + "should verify source PVC is Bound before cloning" + ) diff --git a/evaluation/with_skills/rh-virt__vm-create/environment/Dockerfile b/evaluation/with_skills/rh-virt__vm-create/environment/Dockerfile new file mode 100644 index 00000000..ae625e01 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-create/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/.ai-index/semantic-index.json new file mode 100644 index 00000000..2512c2a3 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/.ai-index/semantic-index.json @@ -0,0 +1,148 @@ +{ + "index_version": "1.0.0", + "last_updated": "2026-02-17", + "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", + + "documents": [ + { + "file": "INDEX.md", + "title": "VM Troubleshooting Guide - Index", + "category": "navigation", + "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], + "use_cases": ["navigation", "tool-reference", "general-guidance"], + "token_estimate": 450, + "skills": ["all"] + }, + { + "file": "scheduling-errors.md", + "title": "VM Scheduling Errors", + "category": "scheduling", + "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], + "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], + "errors_covered": [ + "ErrorUnschedulable - Node Taints", + "ErrorUnschedulable - Insufficient Resources", + "ErrorUnschedulable - Node Selector Mismatch" + ], + "token_estimate": 820, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "storage-errors.md", + "title": "VM Storage Errors", + "category": "storage", + "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], + "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], + "errors_covered": [ + "ErrorDataVolumeNotReady - DataVolume Still Provisioning", + "ErrorDataVolumeNotReady - Storage Class Not Found", + "ErrorDataVolumeNotReady - Insufficient Storage Quota", + "ErrorPvcNotFound", + "Storage Deletion Failures", + "DataVolume Cloning Failures" + ], + "token_estimate": 1350, + "skills": ["vm-creator", "vm-delete", "vm-clone"] + }, + { + "file": "lifecycle-errors.md", + "title": "VM Lifecycle Errors", + "category": "lifecycle", + "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], + "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], + "errors_covered": [ + "VM Stuck in Terminating State", + "VM Won't Start (Non-Scheduling Issues)", + "VM Won't Stop" + ], + "token_estimate": 1150, + "skills": ["vm-delete", "vm-lifecycle-manager"] + }, + { + "file": "runtime-errors.md", + "title": "VM Runtime Errors", + "category": "runtime", + "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], + "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], + "errors_covered": [ + "CrashLoopBackOff" + ], + "token_estimate": 850, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "network-errors.md", + "title": "VM Network Errors", + "category": "networking", + "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], + "use_cases": ["vm-creation", "network-troubleshooting"], + "errors_covered": [ + "Network Attachment Failures" + ], + "token_estimate": 580, + "skills": ["vm-creator"] + } + ], + + "skill_to_docs_mapping": { + "vm-creator": { + "primary_docs": ["scheduling-errors.md", "storage-errors.md"], + "secondary_docs": ["runtime-errors.md", "network-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3650, + "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" + }, + "vm-delete": { + "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 2950, + "description": "VM deletion failures - stuck terminating, storage cleanup" + }, + "vm-clone": { + "primary_docs": ["storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 1800, + "description": "VM cloning failures - DataVolume cloning, storage provisioning" + }, + "vm-lifecycle-manager": { + "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], + "secondary_docs": ["runtime-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3270, + "description": "VM start/stop failures - lifecycle management, scheduling" + }, + "vm-inventory": { + "primary_docs": [], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 450, + "description": "General guidance for status interpretation" + } + }, + + "error_to_docs_mapping": { + "ErrorUnschedulable": "scheduling-errors.md", + "ErrorDataVolumeNotReady": "storage-errors.md", + "ErrorPvcNotFound": "storage-errors.md", + "Terminating": "lifecycle-errors.md", + "CrashLoopBackOff": "runtime-errors.md", + "NetworkAttachmentFailed": "network-errors.md" + }, + + "cross_references": { + "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], + "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], + "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], + "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], + "network-errors.md": ["scheduling-errors.md"] + }, + + "progressive_disclosure_strategy": { + "description": "Load only the minimum documentation needed for current task", + "pattern": "error_type → primary_doc → cross_references (if needed)", + "token_savings": "29% average reduction vs single-file (based on rh-sre data)", + "navigation_overhead_reduction": "85% vs single-file navigation" + } +} diff --git a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/INDEX.md new file mode 100644 index 00000000..a2e76963 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/INDEX.md @@ -0,0 +1,332 @@ +--- +title: VM Troubleshooting Guide - Index +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 + - title: OpenShift Virtualization - Virtual Machine Status + url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html + date_accessed: 2026-02-06 + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] +semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] +use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] +last_updated: 2026-02-17 +--- + +# VM Troubleshooting Guide - Index + +## Overview + +This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: + +- **Creation**: Scheduling failures, storage provisioning issues +- **Lifecycle**: Start/stop failures, stuck states +- **Deletion**: Resources stuck in Terminating, storage cleanup failures +- **Cloning**: DataVolume cloning errors, cross-namespace issues +- **Networking**: Secondary network attachment failures +- **Runtime**: CrashLoopBackOff, guest OS boot failures + +This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. + +--- + +## 🗂️ Troubleshooting Categories + +The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: + +### 1. [Scheduling Errors](scheduling-errors.md) +**When to use**: VM fails to schedule on any node + +**Errors covered**: +- ErrorUnschedulable - Node Taints +- ErrorUnschedulable - Insufficient Resources +- ErrorUnschedulable - Node Selector Mismatch + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 2. [Storage Errors](storage-errors.md) +**When to use**: VM has storage provisioning, deletion, or cloning issues + +**Errors covered**: +- ErrorDataVolumeNotReady (all 3 subsections) +- ErrorPvcNotFound +- Storage Deletion Failures +- DataVolume Cloning Failures + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### 3. [Lifecycle Errors](lifecycle-errors.md) +**When to use**: VM has start/stop/termination issues + +**Errors covered**: +- VM Stuck in Terminating State +- VM Won't Start (Non-Scheduling Issues) +- VM Won't Stop + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### 4. [Runtime Errors](runtime-errors.md) +**When to use**: VM repeatedly crashes or fails at runtime + +**Errors covered**: +- CrashLoopBackOff + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 5. [Network Errors](network-errors.md) +**When to use**: VM has secondary network attachment failures + +**Errors covered**: +- Network Attachment Failures (Multus, SR-IOV) + +**Skills that use this**: vm-create + +--- + +## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last + +### Critical Principle: MCP-First Approach + +**All diagnostic procedures in this guide follow the MCP-first pattern**: + +``` +1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) +2. ⚠️ IF FAILS: Ask user permission to use CLI command +3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval +``` + +### Why MCP Tools First? + +- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources +- **Consistency**: Same interface across all operations +- **Better Error Handling**: MCP tools return structured errors +- **Audit Trail**: MCP tool usage is logged and trackable + +### Command Pattern Examples + +Throughout this guide, you'll see diagnostic steps formatted like this: + +**✅ CORRECT Pattern (MCP First)**: + +```markdown +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Extract**: `.metadata.finalizers` from returned JSON + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` +``` + +### Available MCP Tools + +The `openshift-virtualization` MCP server provides these tools: + +**Resource Operations**: +- `resources_get` - Get specific resource (replaces `oc get `) +- `resources_list` - List resources (replaces `oc get `) +- `resources_delete` - Delete resource (replaces `oc delete`) +- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) + +**Pod Operations**: +- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) +- `pods_get` - Get pod details (replaces `oc get pod`) +- `pods_log` - Get pod logs (replaces `oc logs`) +- `pods_exec` - Execute in pod (replaces `oc exec`) +- `pods_delete` - Delete pod (replaces `oc delete pod`) +- `pods_top` - Pod resource usage (replaces `oc top pods`) + +**Events & Monitoring**: +- `events_list` - List events (replaces `oc get events`) +- `nodes_top` - Node resource usage (replaces `oc top nodes`) +- `nodes_log` - Node logs (replaces `oc adm node-logs`) +- `nodes_stats_summary` - Detailed node stats + +**VM Operations** (KubeVirt toolset): +- `vm_create` - Create VMs +- `vm_lifecycle` - Start/stop/restart VMs + +**Namespaces**: +- `namespaces_list` - List namespaces (replaces `oc get namespaces`) +- `projects_list` - List OpenShift projects (replaces `oc get projects`) + +### When CLI Commands Are Required + +Some operations have **NO MCP equivalent** and require CLI: + +- `virtctl` commands (console, VNC access) +- `oc debug node` (node debugging) +- `oc auth can-i` (permission checks) +- `oc adm taint` (node taint management) +- Complex JSON patch operations + +For these, the guide will note: **"CLI Only - No MCP equivalent"** + +### Quick Reference: CLI → MCP Mapping + +| CLI Command | MCP Tool Equivalent | +|-------------|---------------------| +| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | +| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | +| `oc logs -n ` | `pods_log` with name="", namespace="" | +| `oc get events -n ` | `events_list` with namespace="" | +| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | + +**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. + +### How to Read Diagnostic Sections + +Each error section includes: +1. **Symptom** - What you observe +2. **Description** - What's happening +3. **Possible Causes** - Why it's happening +4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback +5. **Solutions** - **MCP tools first**, then CLI fallback +6. **Verification** - **MCP tools first**, then CLI fallback + +**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. + +--- + +## 🔍 Quick Navigation by Skill + +**vm-create**: +- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics +- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning +- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff +- [Network Errors](network-errors.md) - Network attachment failures + +**vm-delete**: +- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state +- [Storage Errors](storage-errors.md) - Storage deletion failures + +**vm-clone**: +- [Storage Errors](storage-errors.md) - DataVolume cloning failures + +**vm-lifecycle-manager**: +- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop +- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling + +**vm-inventory**: +- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed + +--- + +## 📊 Documentation Coverage & Maintenance + +### Current Coverage + +This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: + +- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) +- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures +- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop +- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) +- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems + +**Total errors documented**: 12 error types across 6 categories + +--- + +### Encountering Undocumented Errors + +#### For AI Agents (Claude Code) + +If you encounter an error **not documented** in the categories above: + +1. **Report to user** with all available details (error message, affected resources, namespace) +2. **Provide best-effort diagnostics** using MCP tools: + - `resources_get` to inspect resource status + - `pods_log` to check virt-launcher or compute container logs + - `events_list` to view Kubernetes events +3. **Suggest documentation update**: + ``` + ⚠️ This error is not yet documented in the troubleshooting guide. + + **Error**: + **Affected resource**: / + + I recommend adding this error to the troubleshooting guide: + - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file + - If it's a new error class → Create new category file in docs/troubleshooting/ + + Would you like me to help document this error for future reference? + ``` + +4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs + +--- + +#### For Human Contributors + +To document a new error: + +1. **Determine the category**: + - Scheduling issue → `scheduling-errors.md` + - Storage problem → `storage-errors.md` + - Lifecycle/start/stop → `lifecycle-errors.md` + - Runtime crash → `runtime-errors.md` + - Network issue → `network-errors.md` + - New error class → Create new category file (e.g., `security-errors.md`) + +2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): + - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback + - **Solutions**: MCP Tool approach → Implementation → CLI Fallback + - **Verification**: MCP Tool checks → Expected results → CLI Fallback + +3. **Update semantic index**: + - Edit `.ai-index/semantic-index.json` + - Add error to `error_to_docs_mapping` + - Update relevant skill's `primary_docs` or `secondary_docs` + - Increment token estimates if needed + +4. **Reference in skill files**: + - Update skill's Reference Documentation section + - Ensure skill's Document Consultation steps point to the right category + +5. **Update this INDEX.md**: + - Add error to the appropriate category section (lines 50-120) + - Update "Current Coverage" list above + +--- + +## 📚 Additional Resources + +- [rh-virt README](../../README.md) - MCP server setup and skill documentation +- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources +- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/lifecycle-errors.md new file mode 100644 index 00000000..6167fcc8 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/lifecycle-errors.md @@ -0,0 +1,869 @@ +--- +title: VM Lifecycle Errors +category: kubevirt +sources: + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 +tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] +semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] +use_cases: [vm-deletion, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Lifecycle Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM lifecycle issues including start/stop failures and stuck termination states. + +**When to use this document**: +- VM stuck in `Terminating` state +- VM won't start (runStrategy is Always but VM never reaches Running) +- VM won't stop (runStrategy is Halted but VM never reaches Stopped) + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### VM Stuck in Terminating State + +**Symptom**: VM shows status `Terminating` but deletion never completes + +**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). + +**Possible Causes**: +- Finalizers blocking deletion +- PVC/DataVolume still attached and preventing cleanup +- VirtualMachineInstance (VMI) not terminating properly +- Custom controllers or operators blocking deletion +- Stuck virt-launcher pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check finalizers on the VM**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` + +**2. Check if VMI still exists**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name containing "virt-launcher-". + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check events for deletion issues**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results where `involvedObject.name` == "" and sort by timestamp. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' +``` + +**5. Check VM deletion timestamp**: + +Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. + +If present, VM is in deletion process. If null, VM is not being deleted. + +**Common Finalizer Patterns**: +- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer +- `foregroundDeletion` - Waits for dependent objects to be deleted +- Custom finalizers from operators + +**Solutions** (Use MCP Tools First): + +1. **Wait for dependent resources** (safest approach): + - Use `resources_list` to check if PVCs, DataVolumes are still deleting + - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) + +2. **Force delete VMI** (if VMI is stuck): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable - requires explicit user permission): + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +3. **Force delete virt-launcher pod** (if pod is stuck): + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + # Find the virt-launcher pod name first + oc get pods -n | grep virt-launcher- + + # Then delete it + oc delete pod virt-launcher--xxx -n --force --grace-period=0 + ``` + +4. **Remove finalizers** (⚠️ dangerous - use only as last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` + 2. Remove items from `.metadata.finalizers` array + 3. Update VM using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch not easily done via MCP): + ```bash + # This operation is complex for MCP - may need CLI + oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Check for protection labels** (vm-delete skill specific): + + Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. + + If value is "true", the vm-delete skill refuses deletion (this is expected behavior). + +**Verification**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error if deletion successful. + +**CLI Fallback**: +```bash +oc get vm -n +# Should return: Error from server (NotFound) +``` + +--- + + +--- + +### VM Won't Start (Non-Scheduling Issues) + +**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state + +**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. + +**Possible Causes**: +- Guest OS kernel panic or boot failure +- Cloud-init configuration errors +- Missing or corrupted disk image +- Insufficient memory for guest OS to boot +- QEMU/KVM virtualization errors +- VirtualMachineInstance (VMI) creation failures +- virt-launcher pod crashes + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VMI (VirtualMachineInstance) status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and overall VMI state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**2. Check VMI conditions for errors**: + +Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. View virt-launcher pod logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +First, get pod name from step 3, then: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +Look for QEMU errors, memory allocation failures, device errors. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**5. Check virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. + +Alternatively, use `pods_get` to get full pod details: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**6. Access VM console to see guest OS boot messages**: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +Look for kernel panic, initramfs errors, filesystem errors. + +**7. Check VM events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep -A 20 "Events:" +``` + +**Common Error Patterns**: + +1. **Guest OS Boot Failure**: + - Console shows kernel panic + - Guest hangs at GRUB or boot loader + - Cloud-init errors during first boot + +2. **Insufficient Memory**: + - Guest OS kills processes due to OOM + - VMI logs show memory allocation errors + +3. **QEMU Crashes**: + - virt-launcher logs show QEMU segfaults + - VMI repeatedly restarts + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for kernel panic, initramfs errors, filesystem errors. + +2. **Check virt-launcher pod logs for QEMU errors**: + + Use `pods_log` from diagnostic step 4 to view logs. + + Look for: + - "qemu-system-x86_64: ..." errors + - Memory allocation failures + - Device errors + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx + ``` + +3. **Increase memory if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check VMI memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +4. **Verify disk image integrity**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check PVC is bound: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + If using container disk, verify image exists and is accessible (check VMI spec). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get pvc -n + ``` + +5. **Check cloud-init configuration** (if applicable): + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. + + Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq + ``` + +6. **Restart VMI** (soft reset): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete VMI (VM controller will recreate it): + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for new VMI to start (use `resources_get` to check status). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n -w + ``` + +7. **Check virtualization extensions** (KVM): + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + + # In debug shell: + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should eventually return `Running`). + +Check VMI is running: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should show `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should eventually return: Running + +oc get vmi -n +# Should show: Running +``` + +--- + +### VM Won't Stop + +**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state + +**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. + +**Possible Causes**: +- Guest OS not responding to ACPI shutdown signal +- virt-launcher pod stuck and not terminating +- VirtualMachineInstance (VMI) deletion blocked by finalizers +- Guest shutdown scripts hanging +- Filesystem sync issues in guest OS + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (might show `Stopping`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**2. Check VMI status and deletion timestamp**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check VMI events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vmi -n | grep -A 10 "Events:" +``` + +**5. Check VMI finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.metadata.finalizers}' +``` + +**6. Check if guest is responsive** (if VMI still exists): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Patterns**: +- VMI shows `deletionTimestamp` but never actually deletes +- virt-launcher pod in `Terminating` state +- VM runStrategy is `Halted` but printableStatus shows `Stopping` + +**Solutions** (Use MCP Tools First): + +1. **Wait for graceful shutdown** (default: 30 seconds): + + Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. + +2. **Force stop by deleting VMI**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + This is the standard way to force-stop a VM. + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n + # Should return: Error from server (NotFound) + ``` + +3. **Force delete VMI with grace period** (if VMI won't delete): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +4. **Force delete virt-launcher pod**: + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find the pod using `pods_list_in_namespace` from diagnostic step 3. + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + ⚠️ **Note**: For force deletion with grace period, use CLI fallback. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') + oc delete pod $POD_NAME -n --force --grace-period=0 + ``` + +5. **Remove VMI finalizers** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VMI using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update VMI using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Patch VM runStrategy directly** (ensure consistency): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` (diagnostic step 1) + 2. Set `.spec.runStrategy` to `"Halted"` + 3. Update VM using `resources_create_or_update` with modified JSON + + **CLI Fallback** (merge patch easier via CLI): + Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Stopped` or `Halted`). + +Verify VMI is gone: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Verify virt-launcher pod is gone: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for pods matching `virt-launcher-`. Should return no results. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Stopped or Halted + +oc get vmi -n +# Should return: Error from server (NotFound) + +oc get pods -n | grep virt-launcher- +# Should return: No resources found +``` + +**Prevention**: +- Ensure guest OS has ACPI support enabled +- Use proper shutdown commands in guest OS +- Avoid forceful stops unless necessary (can corrupt guest filesystem) + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/network-errors.md new file mode 100644 index 00000000..82fa313c --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/network-errors.md @@ -0,0 +1,429 @@ +--- +title: VM Network Errors +category: kubevirt +sources: + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] +semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] +use_cases: [vm-creation, network-troubleshooting] +related_docs: [INDEX.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Network Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. + +**When to use this document**: +- VM created successfully but secondary networks not attached +- NetworkAttachmentDefinition not found errors +- Multus CNI failures +- SR-IOV device attachment issues + +**Skills that use this**: vm-create + +--- + +### Network Attachment Failures + +**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working + +**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). + +**Possible Causes**: +- NetworkAttachmentDefinition doesn't exist in the namespace +- Multus CNI not installed or not configured on cluster +- Namespace mismatch (NAD in different namespace than VM) +- Interface name conflicts in VM spec +- Bridge/network configuration errors in NAD +- SR-IOV device not available (if using SR-IOV) + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List NetworkAttachmentDefinitions in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "" +} +``` + +Review `.items[].metadata.name` for available NADs. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -n +``` + +**2. List NetworkAttachmentDefinitions in all namespaces**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition" +} +``` + +Omit `namespace` parameter to list across all namespaces. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -A +``` + +**3. Check specific NetworkAttachmentDefinition**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "", + "name": "" +} +``` + +Review `.spec.config` for CNI configuration. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definition -n -o yaml +``` + +**4. Check VM network configuration**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.networks` to see network references. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq +``` + +**5. Check VM domain interfaces**: + +Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq +``` + +**6. Check virt-launcher pod network annotations** (shows actual attachments): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**7. Check Multus is installed**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "openshift-multus" +} +``` + +Should show Multus CNI pods running. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n openshift-multus +``` + +**8. Check for errors in virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n +``` + +**Common Error Messages**: +- `"network-attachment-definition not found"` - NAD doesn't exist in namespace +- `"multus CNI not configured"` - Multus not installed or misconfigured +- `"interface name conflict"` - Duplicate interface names in VM spec +- `"failed to add network"` - CNI plugin error (check NAD config) +- `"no available devices"` - SR-IOV device not available (if using SR-IOV) + +**Solutions** (Use MCP Tools First): + +1. **Verify NetworkAttachmentDefinition exists in correct namespace**: + + Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. + + If NAD is in different namespace, copy it to VM namespace: + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get NAD from source namespace using `resources_get` + 2. Modify `.metadata.namespace` to target namespace + 3. Create NAD in target namespace using `resources_create_or_update` + + **CLI Fallback** (stream processing easier via CLI): + Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" + ```bash + oc get network-attachment-definition -n -o yaml | \ + sed "s/namespace: /namespace: /" | \ + oc apply -f - + ``` + +2. **Create missing NetworkAttachmentDefinition**: + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + Example: Linux bridge network + + **Parameters**: + ```json + { + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": { + "name": "vlan100", + "namespace": "" + }, + "spec": { + "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" + ```bash + cat < + spec: + config: '{ + "cniVersion": "0.3.1", + "type": "bridge", + "bridge": "br1", + "vlan": 100, + "ipam": { + "type": "host-local", + "subnet": "192.168.100.0/24" + } + }' + EOF + ``` + +3. **Check Multus CNI installation**: + + Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. + + To check cluster network operator: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterOperator" + } + ``` + + Filter for `network` operator. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n openshift-multus + oc get clusteroperators network + ``` + +4. **Fix interface name conflicts** (if VM has duplicate names): + + Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. + + Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. + + **CLI Fallback** (interactive edit easier via CLI): + Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' + oc edit vm -n + ``` + +5. **Validate NAD configuration syntax**: + + Use `resources_get` from diagnostic step 3, extract `.spec.config`. + + Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get network-attachment-definition -n -o jsonpath='{.spec.config}' + ``` + +6. **Check SR-IOV device availability** (if using SR-IOV networks): + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + List SR-IOV network node policies: + + **Parameters**: + ```json + { + "apiVersion": "sriovnetwork.openshift.io/v1", + "kind": "SriovNetworkNodePolicy", + "namespace": "openshift-sriov-network-operator" + } + ``` + + Check SR-IOV device plugin pods: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "openshift-sriov-network-operator" + } + ``` + + Filter for pods with "device-plugin" in name. + + Check available SR-IOV devices on node: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "Node", + "name": "" + } + ``` + + Review `.status.allocatable` for SR-IOV resources. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get sriovnetworknodepolicy -n openshift-sriov-network-operator + oc get pods -n openshift-sriov-network-operator | grep device-plugin + oc describe node | grep -A 10 "Allocatable:" + ``` + +7. **Recreate VM with corrected network configuration** (if needed): + + If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +After remediation, check virt-launcher pod network status: + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +Should show all attached networks with status. Example output: +```json +[ + { + "name": "openshift-sdn", + "interface": "eth0", + "ips": ["10.128.2.10"], + "default": true + }, + { + "name": "vlan100", + "interface": "net1", + "ips": ["192.168.100.5"] + } +] +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**Check from inside VM** (via console): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# In guest OS: +ip addr show +# Should show all network interfaces (eth0, net1, etc.) +``` + +**Common Network Types**: +- **Linux Bridge**: Layer 2 bridge for VLAN networks +- **SR-IOV**: High-performance direct device assignment +- **macvlan**: MAC-based VLAN for container networks +- **OVN-Kubernetes**: OpenShift native overlay network + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/runtime-errors.md new file mode 100644 index 00000000..804d3cbc --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/runtime-errors.md @@ -0,0 +1,616 @@ +--- +title: VM Runtime Errors +category: kubevirt +sources: + - title: Kubernetes Pod Lifecycle + url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ + date_accessed: 2026-02-17 +tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] +semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] +use_cases: [vm-creation, vm-lifecycle, diagnostics] +related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Runtime Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. + +**When to use this document**: +- VM or virt-launcher pod shows `CrashLoopBackOff` status +- virt-launcher pod repeatedly restarting +- Guest OS kernel panics on boot + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### CrashLoopBackOff + +**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting + +**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. + +**Possible Causes**: +- Guest OS kernel panic on boot +- Insufficient resources (memory/CPU) for guest OS +- Corrupted disk image or filesystem +- QEMU/libvirt crashes due to configuration errors +- Missing or incompatible device drivers in guest +- Resource limits too low for virt-launcher pod +- Virtualization features (KVM) not available on node + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check virt-launcher pod restart count**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# Look at RESTARTS column +``` + +**2. View recent crash logs** (previous container instance): + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "previous": true, + "tail": 100 +} +``` + +Look for QEMU errors, kernel panics, or segfaults. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n virt-launcher--xxx --previous +``` + +**3. Check current virt-launcher logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. + +**CLI Fallback** (if MCP unavailable or all containers needed): +```bash +oc logs -n virt-launcher--xxx --all-containers +``` + +**4. Check VMI conditions for crash details**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Extract `.status.conditions` for crash details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**5. Check pod events for crash reasons**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" +``` + +**6. Check pod resource limits**: + +**MCP Tool**: `pods_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +Extract `.spec.containers[0].resources` for resource limits. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq +``` + +**7. Check node kubelet logs for OOM kills**: + +⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +oc adm node-logs -u kubelet | grep -i oom +``` + +**8. Access guest console** (if VM briefly starts): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Crash Patterns**: + +1. **Guest Kernel Panic**: + - Console logs show kernel panic messages + - Guest crashes immediately after boot + - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" + +2. **OOM (Out of Memory)**: + - Pod killed with reason: `OOMKilled` + - Guest runs out of memory during boot or operation + - virt-launcher logs show memory allocation failures + +3. **QEMU Crash**: + - virt-launcher logs show QEMU segmentation fault + - Symptoms: "qemu-system-x86_64: terminated by signal" + - Configuration incompatibility or QEMU bug + +4. **Disk Image Corruption**: + - Guest cannot boot from disk + - Filesystem errors in guest console + - DataVolume import failed + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for kernel panic or boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for: + - Kernel panic messages + - Initramfs errors + - Filesystem mounting failures + - Missing device errors + +2. **Review virt-launcher crash logs**: + + Use `pods_log` with `previous: true` from diagnostic step 2. + + Look for: + - QEMU command line errors + - Device initialization failures + - Memory allocation errors + - Signal termination (SIGSEGV, SIGABRT) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx --previous + ``` + +3. **Check for OOM (Out of Memory) kills**: + + **MCP Tool**: `pods_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + Extract `.status.containerStatuses[0].lastState.terminated.reason`. + + If returns `"OOMKilled"`: + - Option 1: Increase virt-launcher memory limits + - Option 2: Decrease guest memory allocation + - Option 3: Use smaller instance type + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' + ``` + +4. **Increase resources if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check current memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +5. **Verify disk image integrity**: + + **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters for list**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" + } + ``` + + **Parameters for specific DV**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check `.status.phase` (should be `Succeeded`). + + If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get datavolume -n -o jsonpath='{.status.phase}' + ``` + +6. **Check virtualization (KVM) availability**: + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +7. **Simplify VM configuration** (eliminate variables): + + Try creating minimal VM using vm-create skill with: + - Small instance type + - No secondary networks + - Simple container disk (e.g., Fedora) + - No cloud-init + + If minimal VM works, add features back one by one. + +8. **Recreate VM with different workload** (test disk image): + + If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) + +After remediation, check pod restart count stops increasing: + +**Parameters for pods**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). + +Check VM reaches Running state: + +**Parameters for VM**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# RESTARTS should stabilize + +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Running +``` + +Verify guest is responsive: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# Should show login prompt or OS console +``` + +**Advanced Debugging**: + +**MCP Tool**: `pods_exec` (from openshift-virtualization) + +Check libvirt domain XML: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["virsh", "dumpxml", "1"] +} +``` + +Check QEMU process: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["ps", "aux"] +} +``` + +Filter output for "qemu" process. + +**CLI Fallback** (if MCP unavailable): +```bash +oc exec -n virt-launcher--xxx -- virsh dumpxml 1 +oc exec -n virt-launcher--xxx -- ps aux | grep qemu +``` + +**Prevention**: +- Start with minimal VM configuration and add complexity gradually +- Use recommended instance types for your workload +- Test disk images before deploying to production +- Ensure nodes have adequate resources and KVM support +- Monitor resource usage and set appropriate limits + +--- + +## Workaround Patterns for MCP Tool Limitations + +### General Pattern: Diagnose → Propose → Confirm → Execute + +When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): + +**1. Diagnose** the root cause using Kubernetes commands +```bash +oc describe vm -n +oc get events -n --field-selector involvedObject.name= +``` + +**2. Propose** a manual workaround to the user with clear explanation +- Explain why the issue occurred +- Show what will be changed +- List alternative options + +**3. Confirm** with user before executing (Human-in-the-Loop) +- Wait for explicit "yes" or "apply workaround" +- Never auto-execute modifications + +**4. Execute** the workaround using oc/kubectl +```bash +oc patch vm -n --type=merge -p '...' +``` + +**5. Verify** the fix was successful +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**6. Document** the limitation and suggest filing enhancement request +- Note this is temporary until MCP tool is enhanced +- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues + +### Example: Adding Tolerations Workaround + +**Diagnostic Output**: +```markdown +## ⚠️ VM Scheduling Issue Detected + +**Root Cause**: Node taints prevent VM scheduling + +**Details**: +- Found 3 nodes with taint: `virtualization=true:NoSchedule` +- VM spec does not include matching tolerations +- This prevents VM scheduling on virtualization-dedicated nodes +``` + +**Proposed Workaround**: +```bash +oc patch vm web-server -n vms --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' +``` + +**User Confirmation Required**: +``` +How would you like to proceed? +- "apply workaround" - I'll patch the VM with tolerations +- "manual" - I'll provide instructions for you to apply manually +- "cancel" - Delete the VM and abort creation +``` + +**After Execution**: +```markdown +## ✓ Workaround Applied Successfully + +**Action**: Added tolerations for taint `virtualization=true:NoSchedule` +**New Status**: Stopped (VM can now be scheduled) + +**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. +Future VMs in this cluster will need the same fix until the tool is enhanced. +``` + +--- + +## VM Status Reference + +### Status Values + +| Status | Meaning | Action Required | +|--------|---------|-----------------| +| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | +| `Running` | VM is running | Normal | +| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | +| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | +| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | +| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | +| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | +| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | +| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | +| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | + +### Checking VM Status + +```bash +# Get printable status +oc get vm -n -o jsonpath='{.status.printableStatus}' + +# Get detailed status and conditions +oc get vm -n -o jsonpath='{.status}' | jq + +# Watch status changes in real-time +oc get vm -n -w +``` + +--- + +## Best Practices for Agents + +When implementing diagnostic workflows: + +1. **Always verify VM status** after creation (wait 5-10 seconds first) +2. **Consult this document** when encountering error status values +3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) +4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) +5. **Respect human-in-the-loop** for all VM modifications +6. **Document temporary workarounds** and their limitations clearly +7. **Suggest filing issues** for missing MCP tool features + +### Document Consultation Pattern + +```markdown +**Document Consultation** (REQUIRED): +1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes +2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." +``` + +--- + +## Known MCP Tool Limitations + +### vm_create tool + +**Currently Supported**: +- ✓ Namespace, name (required) +- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) +- ✓ Size hints (small, medium, large) +- ✓ Storage size +- ✓ Autostart flag +- ✓ Networks (Multus NetworkAttachmentDefinitions) +- ✓ Performance family (u1, o1, c1, m1) +- ✓ Instance type, preference + +**Not Currently Supported** (requires workarounds): +- ✗ Tolerations (for node taints) +- ✗ Node selectors +- ✗ Affinity/anti-affinity rules +- ✗ Resource requests/limits (beyond instance type) +- ✗ Custom labels/annotations +- ✗ SSH keys injection +- ✗ Cloud-init user data + +**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. + +**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues + +--- + +## Additional Resources + +- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) +- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) +- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) +- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) + +--- + +[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/scheduling-errors.md new file mode 100644 index 00000000..e47c8c4f --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/scheduling-errors.md @@ -0,0 +1,417 @@ +--- +title: VM Scheduling Errors +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 +tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] +semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] +use_cases: [vm-creation, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Scheduling Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**When to use this document**: +- VM shows status `ErrorUnschedulable` after creation or start attempt +- VM events mention scheduling failures, taints, resources, or node selectors + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +## ErrorUnschedulable + +**Symptom**: VM shows status `ErrorUnschedulable` after creation + +**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**Possible Causes**: + +### 1. Node Taints (Most Common) + +Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM events for scheduling failures**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `involvedObject.name` == "" and look for messages like: +- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= +``` + +**2. Check node taints in the cluster**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' +``` + +**Common Taint Patterns**: +- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule +- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes +- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads + +**Solution - Add Tolerations to VM**: + +The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. + +**Workaround (post-creation using MCP Tools)**: + +**Step 1**: Get current VM spec + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Step 2**: Modify the returned JSON to add tolerations + +Add to `.spec.template.spec.tolerations`: +```json +{ + "tolerations": [ + { + "key": "virtualization", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule" + } + ] +} +``` + +**Step 3**: Update VM with modified spec + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +```json +{ + "resource": "" +} +``` + +Pass the complete modified VM resource as YAML or JSON string. + +**Step 4**: Verify tolerations were added + +Use `resources_get` again and check `.spec.template.spec.tolerations` in response. + +**Step 5**: Check if VM status improved + +Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. + +**CLI Fallback** (if MCP patch is too complex): +```bash +# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' + +# Verify tolerations +oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq + +# Check status +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**Example - Multiple Tolerations**: +```bash +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: "dedicated" + operator: "Equal" + value: "virt-workloads" + effect: "NoSchedule" +' +``` + +**Toleration Operators**: +- `Equal` - Key and value must match exactly +- `Exists` - Only key must exist (ignores value) + +**Toleration Effects**: +- `NoSchedule` - Don't schedule new pods (existing pods continue) +- `PreferNoSchedule` - Avoid scheduling if possible +- `NoExecute` - Don't schedule AND evict existing pods + +**Alternative Solutions**: +1. **Remove node taints** (if you have cluster-admin access): + ```bash + oc adm taint nodes virtualization=true:NoSchedule- + ``` + +2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits + +3. **File enhancement request** - Request tolerations support in openshift-mcp-server: + https://github.com/openshift/openshift-mcp-server/issues + +--- + +### 2. Insufficient Resources + +Not enough CPU, memory, or storage available on any node. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM resource requests**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' +``` + +**2. Check node resource availability**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. + +Alternatively, use `nodes_top` MCP tool for current resource usage. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe nodes | grep -A 5 "Allocated resources" +``` + +**3. Look for VM events mentioning "Insufficient"**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep "Insufficient" +``` + +**Example Event**: +``` +0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. +``` + +**Solutions** (Use MCP Tools First): + +1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) +2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill +3. **Delete unused VMs** - Use vm-delete skill to free up resources +4. **Check resource quotas**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters for quota**: + ```json + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" + } + ``` + + **Parameters for limit range**: + ```json + { + "apiVersion": "v1", + "kind": "LimitRange", + "namespace": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + oc describe limitrange -n + ``` + +--- + +### 3. Node Selector Mismatch + +VM requires specific node labels that don't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM node selector requirements**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.nodeSelector` to see required node labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' +``` + +**2. List available node labels**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.metadata.labels` for available labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes --show-labels +``` + +**3. Check if any nodes match the selector**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node", + "labelSelector": "=" +} +``` + +Should return at least one node with matching labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -l = +``` + +**Solutions** (Use MCP Tools First): + +**Option 1: Remove node selector from VM** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get current VM using `resources_get` (diagnostic step 1) +2. Remove `.spec.template.spec.nodeSelector` field +3. Update VM using `resources_create_or_update` with modified JSON + +**CLI Fallback** (JSON patch easier via CLI): +Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" +```bash +oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' +``` + +**Option 2: Add label to nodes** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get node using `resources_get` +2. Add label to `.metadata.labels` +3. Update node using `resources_create_or_update` + +⚠️ **Note**: Node labeling typically requires cluster admin privileges. + +**CLI Fallback** (simpler via CLI): +Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" +```bash +oc label node = +``` + +--- + +[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/storage-errors.md new file mode 100644 index 00000000..16ba584d --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/storage-errors.md @@ -0,0 +1,1011 @@ +--- +title: VM Storage Errors +category: kubevirt +sources: + - title: KubeVirt CDI - DataVolumes + url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ + date_accessed: 2026-02-17 + - title: Kubernetes Persistent Volumes + url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ + date_accessed: 2026-02-17 +tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] +semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] +use_cases: [vm-creation, vm-deletion, vm-cloning] +related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] +last_updated: 2026-02-17 +--- + +# VM Storage Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. + +**When to use this document**: +- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` +- Storage deletion fails after VM deletion +- DataVolume cloning operations fail +- PVC provisioning issues + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### ErrorDataVolumeNotReady + +**Symptom**: VM shows status `ErrorDataVolumeNotReady` + +**Description**: The DataVolume (persistent storage) backing the VM is not ready. + +**Possible Causes**: + +#### 1. DataVolume Still Provisioning + +Storage provisioning takes time, especially for large disks or when importing images. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Get detailed DataVolume information**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` and `.status.conditions` for provisioning details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o json +``` + +**3. Check PVC (PersistentVolumeClaim) bound status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Check `.status.phase` for each PVC (should be `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. + +#### 2. Storage Class Not Found + +The requested storage class doesn't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List available storage classes**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" +} +``` + +Review the list of available storage classes (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass +``` + +**2. Check DataVolume's requested storage class**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.spec.pvc.storageClassName` in the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' +``` + +**Solution**: +1. Use a valid storage class from the cluster +2. Recreate VM with correct storage class parameter + +#### 3. Insufficient Storage Quota + +Namespace has insufficient storage quota to provision the PVC. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check resource quotas**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**2. Check storage usage**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase +``` + +**Solution**: +1. Request quota increase from cluster admin +2. Delete unused PVCs to free quota +3. Reduce VM storage size + +--- + +### ErrorPvcNotFound + +**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List PVCs in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review the list of available PVCs (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check VM's PVC references**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' +``` + +**Solution**: +- Wait for DataVolume to create the PVC +- Manually create missing PVC +- Fix VM spec to reference correct PVC name + +--- + + +--- + +### Storage Deletion Failures + +**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace + +**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. + +**Possible Causes**: +- PVC still bound to active PersistentVolume with `Retain` policy +- DataVolume still being referenced by another resource +- CDI (Containerized Data Importer) controller issues +- Storage class retention policy preventing deletion +- Finalizers on PVC/DataVolume blocking cleanup +- PVC still mounted by a pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check PVC status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check specific PVC phase**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Released` or `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o jsonpath='{.status.phase}' +``` + +**3. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**4. Check what's using the PVC**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' +``` + +**5. Check PVC finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. + +**6. Check DataVolume finalizers**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.metadata.finalizers}' +``` + +**7. Check PV reclaim policy**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume" +} +``` + +Filter results for PV where `.spec.claimRef.name` matches ``. + +To get specific PV policy: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume", + "name": "" +} +``` + +Check `.spec.persistentVolumeReclaimPolicy`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pv | grep +oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' +``` + +**Common Finalizer Patterns**: +- `kubernetes.io/pvc-protection` - Protects PVC while in use +- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer + +**Solutions** (Use MCP Tools First): + +1. **Delete DataVolume first, then PVC**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete DataVolume first (often blocks PVC deletion): + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Wait a few seconds, then delete PVC: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete datavolume -n + oc delete pvc -n + ``` + +2. **Check for pods still using PVC**: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "" + } + ``` + + Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. + + Then delete the pods using `pods_delete`: + + **Parameters**: + ```json + { + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' + oc delete pod -n + ``` + +3. **Force delete PVC** (if safe to do so): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete pvc -n --grace-period=0 --force + ``` + +4. **Remove finalizers from PVC** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PVC using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update PVC using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Remove finalizers from DataVolume** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current DataVolume using `resources_get` (diagnostic step 6) + 2. Remove items from `.metadata.finalizers` array + 3. Update DataVolume using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Change PV reclaim policy** (if PV has Retain policy): + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PV policy using `resources_get` (diagnostic step 7) + 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` + 3. Update PV using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" + ```bash + oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' + oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' + ``` + +**Storage Quota Check** (Use MCP Tools First): + +After deletion, verify storage quota is freed: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters for quota check**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.used` to verify storage quota is freed. + +**Parameters for PVC verification**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +oc get pvc -n +``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Confirm PVC is deleted: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Confirm DataVolume is deleted: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +# Should return: Error from server (NotFound) + +oc get datavolume -n +# Should return: Error from server (NotFound) +``` + +--- + +### DataVolume Cloning Failures + +**Symptom**: VM clone created successfully but DataVolume clone operation fails + +**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. + +**Possible Causes**: +- CSI driver doesn't support volume cloning +- Source PVC storage class incompatible with cloning +- Cross-namespace cloning not permitted by storage backend +- Insufficient storage quota in target namespace +- Source PVC not in `Bound` state +- Storage class doesn't have volume cloning enabled +- CDI (Containerized Data Importer) controller issues + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume clone status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and `.metadata.name`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Check DataVolume events for errors**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe datavolume -n +``` + +**3. Check DataVolume phase**: + +Use `resources_get` from step 1, extract `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +``` + +**4. Check if storage class supports cloning**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "name": "" +} +``` + +Review the full YAML output for cloning-related configurations. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass -o yaml | grep -A 5 -i clone +``` + +**5. Check CSI driver capabilities**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "CSIDriver" +} +``` + +Review `.items[].metadata.name` for available CSI drivers. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get csidriver +``` + +**6. Check source PVC status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Bound` for cloning to work). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**7. Check target namespace storage quota**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**8. Check CDI controller logs**: + +**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) + +First, list pods in openshift-cnv namespace: + +**Parameters for pods_list_in_namespace**: +```json +{ + "namespace": "openshift-cnv", + "labelSelector": "app.kubernetes.io/component=cdi-deployment" +} +``` + +Then get logs using `pods_log`: + +**Parameters**: +```json +{ + "namespace": "openshift-cnv", + "name": "", + "tail": 100 +} +``` + +**CLI Fallback** (if MCP unavailable or easier via CLI): +```bash +oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') +``` + +**Common Error Messages**: +- `"volume cloning is not supported"` - CSI driver lacks clone capability +- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage +- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace +- `"insufficient quota"` - Target namespace lacks storage quota +- `"source PVC not bound"` - Source PVC must be in Bound state for cloning +- `"StorageClass does not support cloning"` - Storage class configuration issue + +**Solutions** (Use MCP Tools First): + +1. **Check storage class clone support**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" + } + ``` + + For each storage class in `.items[]`, check: + - `.metadata.name` (storage class name) + - `.provisioner` (CSI driver) + + Storage classes using CSI drivers typically support cloning. Look for provisioners like: + - `csi.ovirt.org` (oVirt CSI) + - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) + - `ebs.csi.aws.com` (AWS EBS CSI) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner + ``` + +2. **Verify source PVC is bound**: + + Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pvc -n -o jsonpath='{.status.phase}' + ``` + +3. **Check target namespace quota**: + + Use `resources_list` from diagnostic step 7 to check quota. + + If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + ``` + +4. **Use snapshot-based cloning** (alternative method): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Step 1**: Create VolumeSnapshot of source PVC + + **Parameters**: + ```json + { + "apiVersion": "snapshot.storage.k8s.io/v1", + "kind": "VolumeSnapshot", + "metadata": { + "name": "-snapshot", + "namespace": "" + }, + "spec": { + "source": { + "persistentVolumeClaimName": "" + } + } + } + ``` + + **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) + + **Step 3**: Create new DataVolume from snapshot + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": "-rootdisk", + "namespace": "" + }, + "spec": { + "source": { + "snapshot": { + "name": "-snapshot", + "namespace": "" + } + }, + "storage": { + "resources": { + "requests": { + "storage": "50Gi" + } + }, + "storageClassName": "" + } + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" + ```bash + cat <-snapshot + namespace: + spec: + source: + persistentVolumeClaimName: + EOF + + oc get volumesnapshot -snapshot -n + + cat <-rootdisk + namespace: + spec: + source: + snapshot: + name: -snapshot + namespace: + storage: + resources: + requests: + storage: 50Gi + storageClassName: + EOF + ``` + +5. **Use "new empty storage" option** (vm-clone skill): + - If cloning isn't supported, create VM with empty storage + - Manually copy data if needed + +6. **Cross-namespace cloning workaround**: + - Some storage backends require snapshot for cross-namespace cloning + - Create snapshot in source namespace, restore in target namespace (see solution 4 above) + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Check DataVolume reached Succeeded phase: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should return `Succeeded`). + +Check PVC was created and bound: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "-rootdisk" +} +``` + +Check `.status.phase` (should return `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +# Should return: Succeeded + +oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' +# Should return: Bound +``` + +**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-create/environment/mcp-servers/mock-virt-mcp.py b/evaluation/with_skills/rh-virt__vm-create/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..7b17408d --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-create/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1518 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), + ("vm-testing", {"env": "testing"}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + +STORAGE_CLASSES = [ + { + "name": "ocs-storagecluster-ceph-rbd", + "provisioner": "openshift-storage.rbd.csi.ceph.com", + "reclaimPolicy": "Delete", + "volumeBindingMode": "Immediate", + "allowVolumeExpansion": True, + }, + { + "name": "ocs-storagecluster-cephfs", + "provisioner": "openshift-storage.cephfs.csi.ceph.com", + "reclaimPolicy": "Delete", + "volumeBindingMode": "Immediate", + "allowVolumeExpansion": False, + }, +] + +VOLUME_SNAPSHOT_CLASSES = [ + { + "name": "ocs-storagecluster-rbdplugin-snapclass", + "driver": "openshift-storage.rbd.csi.ceph.com", + "deletionPolicy": "Delete", + }, +] + + +def _build_storage_class(sc): + """Build a storage.k8s.io/v1 StorageClass resource.""" + res = { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "metadata": { + "name": sc["name"], + "uid": _uid(sc["name"]), + "creationTimestamp": CREATED, + }, + "provisioner": sc["provisioner"], + "reclaimPolicy": sc["reclaimPolicy"], + "volumeBindingMode": sc["volumeBindingMode"], + } + if sc.get("allowVolumeExpansion"): + res["allowVolumeExpansion"] = True + return res + + +def _build_volume_snapshot_class(vsc): + """Build a snapshot.storage.k8s.io/v1 VolumeSnapshotClass resource.""" + return { + "apiVersion": "snapshot.storage.k8s.io/v1", + "kind": "VolumeSnapshotClass", + "metadata": { + "name": vsc["name"], + "uid": _uid(vsc["name"]), + "creationTimestamp": CREATED, + }, + "driver": vsc["driver"], + "deletionPolicy": vsc["deletionPolicy"], + } + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-virt__vm-create/environment/skills/vm-create/SKILL.md b/evaluation/with_skills/rh-virt__vm-create/environment/skills/vm-create/SKILL.md new file mode 100644 index 00000000..9fe35d09 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-create/environment/skills/vm-create/SKILL.md @@ -0,0 +1,403 @@ +--- +name: vm-create +description: | + Create new virtual machines in OpenShift Virtualization with automatic instance type resolution and OS selection. + + Use when: + - "Create a new VM" + - "Deploy a virtual machine with [OS]" + - "Set up a VM in namespace [name]" + - "Provision a [size] VM" + + This skill handles VM creation with intelligent defaults for OpenShift Virtualization. + + NOT for managing existing VMs (use vm-lifecycle-manager or vm-delete instead). + +model: inherit +color: green +--- + +# /vm-create Skill + +Create virtual machines in OpenShift Virtualization using the `vm_create` tool from the openshift-virtualization MCP server. + +## Prerequisites + +**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools**: +- `vm_create` (from openshift-virtualization) - Create VirtualMachine resources +- `resources_get` (from openshift-virtualization) - Check VM existence and status +- `resources_list` (from openshift-virtualization) - List StorageClasses +- `namespaces_list` (from openshift-virtualization) - List available namespaces +- `events_list` (from openshift-virtualization) - Diagnostic event gathering +- `vm_lifecycle` (from openshift-virtualization) - VM restart for workarounds + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster (>= 4.19) +- OpenShift Virtualization operator installed +- ServiceAccount with RBAC permissions to create VirtualMachine resources +- Namespace with appropriate permissions + +### Prerequisite Verification + +**Before executing, verify MCP server availability:** + +1. Check `openshift-virtualization` exists in `.mcp.json` → If missing, report setup instructions +2. Verify `KUBECONFIG` is set (check presence only, never expose value) → If missing, report to user + +**Human Notification Protocol (when prerequisites fail):** + +``` +❌ Cannot execute vm-create: MCP server 'openshift-virtualization' not available +📋 Setup: Add to .mcp.json, set KUBECONFIG env var, restart Claude Code +🔗 Docs: https://github.com/openshift/openshift-mcp-server +``` + +⚠️ **SECURITY**: Never display actual KUBECONFIG path or credential values. + +## When to Use This Skill + +**Trigger this skill when:** +- User explicitly invokes `/vm-create` command +- User requests creating a new virtual machine +- Deploying VMs with specific OS (Fedora, Ubuntu, RHEL, CentOS, Debian) +- Setting up VMs with custom sizing (small, medium, large) +- Provisioning VMs with specific storage requirements + +**User phrases:** +- "Create a Fedora VM in namespace vms" +- "Deploy a medium Ubuntu VM with 100Gi disk" +- "Set up a RHEL VM called database-01" +- "/vm-create" (explicit command) + +**Do NOT use when:** +- Start/stop existing VMs → Use `/vm-lifecycle-manager` +- List VMs → Use `/vm-inventory` +- Delete VMs → Use `/vm-delete` + +## Workflow + +### Step 1: Gather VM Requirements + +**Determine missing parameters:** + +**Required:** VM Name (validate: lowercase, alphanumeric+hyphens, start letter, max 63 chars, unique), Namespace +**Optional (use defaults):** OS (fedora), Size (medium), Storage (30Gi), Performance (u1), Autostart (false) + +**Gather cluster info:** +- Detect current namespace: `kubectl config view --minify -o jsonpath='{..namespace}' || echo "default"` +- List namespaces: `namespaces_list` (from openshift-virtualization) +- List StorageClasses: `resources_list` with apiVersion="storage.k8s.io/v1", kind="StorageClass" +- Identify default SC: annotation `storageclass.kubernetes.io/is-default-class`="true" +- Analyze SC: `.volumeBindingMode` (Immediate/WaitForFirstConsumer), provisioner (rbd/cephfs=RWX hint) + +**If parameters missing, use AskUserQuestion tool with questions for:** VM Name (custom input with validation), Namespace (current + list), OS (fedora/ubuntu/rhel/centos-stream/debian/opensuse), Performance (u1/c1/m1/o1), Size (small/medium/large/xlarge), Storage (30Gi/50Gi/100Gi/custom), StorageClass (dynamic list with hints), Autostart (yes/no). See Example Usage for complete JSON structure. + +**Process responses - map labels to values:** +- OS: "Fedora"→`"fedora"`, "Ubuntu"→`"ubuntu"`, "RHEL"→`"rhel"`, "CentOS Stream"→`"centos-stream"`, "Debian"→`"debian"`, "OpenSUSE"→`"opensuse"` +- Performance: "General Purpose (u1)"→`"u1"`, "Compute (c1)"→`"c1"`, "Memory (m1)"→`"m1"`, "Overcommitted (o1)"→`"o1"` +- Size: "Small"→`"small"`, "Medium"→`"medium"`, "Large"→`"large"`, "XLarge"→`"xlarge"` +- Autostart: "No"→`false`, "Yes"→`true` + +### Step 2: Check VM Existence + +**MCP Tool**: `resources_get` (from openshift-virtualization) +**Parameters**: apiVersion="kubevirt.io/v1", kind="VirtualMachine", namespace=``, name=`` + +**If VM exists:** +``` +⚠️ VM `` already exists in namespace `` +Status: +Options: 1) Different name, 2) Delete existing, 3) Cancel +``` +**STOP** and wait for user decision. + +**If not exists:** Proceed to Step 3. + +### Step 3: Present Configuration for Confirmation + +Display configuration table: +```markdown +## Virtual Machine Configuration + +| Parameter | Value | Notes | +|-----------|-------|-------| +| VM Name | `` | validated | +| Namespace | `` | from user/context | +| OS | `` | from selection | +| Performance | `` | default: u1 | +| Size | `` | default: medium | +| Storage | `` | default: 30Gi | +| StorageClass | `` | default: cluster default | +| Autostart | `` | default: no | + +This will create a VirtualMachine consuming cluster resources. +Confirm: yes/no/modify +``` + +Proceed automatically unless the user explicitly objects. + +### Step 4: Create Virtual Machine + +**MCP Tool**: `vm_create` (from openshift-virtualization) + +**Parameters**: +- `namespace`: `` - REQUIRED +- `name`: `` - REQUIRED +- `workload`: `` - OPTIONAL (default: fedora) +- `size`: `` - OPTIONAL (small/medium/large/xlarge) +- `storage`: `` - OPTIONAL (default: 30Gi) +- `performance`: `` - OPTIONAL (u1/c1/m1/o1) +- `autostart`: `` - OPTIONAL (default: false) + +**Example**: `vm_create({"namespace": "vms", "name": "web-server", "workload": "fedora", "size": "medium", "storage": "50Gi", "autostart": false})` + +**Error Handling:** +- Namespace not found → Report, list available +- RBAC denied → Report permissions error +- Storage fails → Check StorageClass exists +- Scheduling fails → See Step 5 + +### Step 5: Verify Status and Diagnose Issues + +**Wait 5-10 seconds, then check status:** + +**MCP Tool**: `resources_get` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", name=``, namespace=``) +**Extract**: `.status.printableStatus` + +**Status interpretation:** +- Stopped/Halted → Success (VM created, not started) +- Running → Success (if autostart=true) +- Provisioning → Wait 5s, check again +- ErrorUnschedulable → Execute diagnostic workflow (Step 5a) +- ErrorDataVolumeNotReady → Storage issue (see Common Issues) + +#### 5a. Diagnostic Workflow (ErrorUnschedulable) + +**CRITICAL: Document Consultation FIRST:** +1. Read [scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) using Read tool +2. Output: "I detected ErrorUnschedulable. I consulted [scheduling-errors.md] to understand diagnosis strategies." + +**Gather diagnostics:** +- List events: `events_list` (namespace=``) → Filter for VM/VMI +- Get VM: `resources_get` → Check `.status.conditions` +- List nodes: `resources_list` (apiVersion="v1", kind="Node") → Extract `.spec.taints` + +**Parse root cause:** +- "taints" in events → Taints/tolerations issue +- "Insufficient cpu/memory" → Resource constraints +- "no nodes available" → No suitable nodes + +**Present diagnosis:** +```markdown +## ⚠️ VM Scheduling Issue Detected + +**Status**: ErrorUnschedulable | **Root Cause**: | **Details**: + +### Recommended Solution + +**Command**: `oc patch vm -n ...` +**Impact**: +**Options**: 1) Apply workaround, 2) Manual, 3) Cancel, 4) Ignore +⚠️ MCP limitation: vm_create doesn't support tolerations +``` + +**Wait for user decision.** + +**If user confirms:** +1. Apply patch: `resources_create_or_update` (fetch, add tolerations, update) OR `oc patch` +2. Verify: `resources_get` → Check `.spec.template.spec.tolerations` +3. **Restart VM**: `vm_lifecycle` (action="restart") to apply new spec +4. Wait 15-20s, check status → Stopped → Provisioning → Running + +**Report**: `## ✓ Workaround Applied | **Action**: Added tolerations, restarted | **Status**: ` + +### Step 6: Report Creation Status + +**On success:** +```markdown +## ✓ Virtual Machine Created Successfully + +**VM**: `` (namespace: ``) +**OS**: | **Size**: () | **Storage**: | **Status**: +**Provisioning**: ~2-5 min (Provisioning → Stopped) + +### Next Steps +Start: "Start VM " | View: "Show VM " + +### Accessing the VM +1. Serial: `virtctl console -n ` +2. VNC: OpenShift Console → Virtualization → VMs → → Console +3. SSH: Get IP from VMI, `ssh @` +4. Port Forward: `virtctl port-forward vmi/ -n 8080:80` + +### Default Credentials +- Fedora: fedora | Ubuntu: ubuntu | RHEL: cloud-user | CentOS: centos | Debian: debian +- All require SSH key or console password set: `virtctl console `, `sudo passwd ` +``` + +**On failure:** +```markdown +## ❌ Failed to Create Virtual Machine + +**Error**: + +**Common Causes**: +- Namespace not exists → `oc create namespace ` +- RBAC denied → Check ServiceAccount permissions +- Resource constraints → Try smaller size +- Invalid parameters → Verify OS, size, storage format +- Operator not installed → Verify CSVs + +Troubleshooting: See Common Issues +``` + +## Common Issues + +### Issue 1: Namespace Not Found +**Error**: "Namespace 'xyz' not found" +**Solution**: List with `namespaces_list`, create with `resources_create_or_update` or `oc create namespace ` + +### Issue 2: Insufficient Permissions +**Error**: "Forbidden: User cannot create VirtualMachines" +**Solution**: Verify KUBECONFIG RBAC, requires create VirtualMachine permissions, contact admin + +### Issue 3: Resource Constraints (ErrorUnschedulable) +**Error**: "0/N nodes: Insufficient cpu/memory" +**Solution**: Check `nodes_top`, try smaller size (medium→small, o1 overcommitted), scale cluster + +### Issue 4: Node Taints (ErrorUnschedulable) +**Error**: "0/N nodes: taints pod didn't tolerate" +**Solution**: Apply tolerations workaround (Step 5a), restart VM + +### Issue 5: Storage Provisioning (ErrorDataVolumeNotReady) +**Error**: "PVC pending" or "StorageClass not found" +**Solution**: Verify SC (`resources_list`), check default annotation, verify provisioner, check quotas + +### Issue 6: DataVolume Import Failure +**Error**: "DataVolume import failed" or "image pull error" +**Solution**: Verify internet access, check DV status, ensure valid OS, verify registry auth + +### Issue 7: Operator Not Installed +**Error**: "VirtualMachine CRD not found" +**Solution**: Verify operator: `resources_list` (apiVersion="operators.coreos.com/v1alpha1", kind="CSV", namespace="openshift-cnv") + +## Dependencies + +### Required MCP Servers +- `openshift-virtualization` - OpenShift MCP server with KubeVirt toolset (https://github.com/openshift/openshift-mcp-server) + +### Required MCP Tools +- `vm_create` - Create VMs (namespace, name, workload, size, storage, performance, autostart) +- `resources_get` - Get resources (apiVersion, kind, namespace, name) +- `resources_list` - List resources (apiVersion, kind, namespace optional) +- `namespaces_list` - List namespaces +- `events_list` - List events (namespace) +- `vm_lifecycle` - VM lifecycle (namespace, name, action: start/stop/restart) +- `resources_create_or_update` - Update resources (JSON) + +### Related Skills +- `vm-lifecycle-manager` - Start VMs | `vm-inventory` - List VMs | `vm-delete` - Delete VMs | `vm-clone` - Clone VMs | `vm-snapshot-create` - Snapshot VMs + +### Reference Documentation +- [scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) - ErrorUnschedulable (consulted Step 5a) +- [storage-errors.md](../../docs/troubleshooting/storage-errors.md) - ErrorDataVolumeNotReady +- [network-errors.md](../../docs/troubleshooting/network-errors.md) - Network failures +- [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) - CrashLoopBackOff +- [Troubleshooting INDEX](../../docs/troubleshooting/INDEX.md) - Full error index +- [OpenShift Virt Docs](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index) +- [KubeVirt API](https://kubevirt.io/api-reference/) +- [OpenShift MCP](https://github.com/openshift/openshift-mcp-server) + +## Critical: Human-in-the-Loop Requirements + +**IMPORTANT:** This skill creates cluster resources consuming CPU, memory, storage. You MUST: + +1. **Before Creating** + - Display complete configuration in table format + - Show resource impact + - Ask: "Confirm: yes/no/modify" + - Wait for explicit confirmation + +2. **Never Auto-Execute** + - NEVER create VMs without displaying configuration + - NEVER assume approval + - NEVER create multiple VMs without individual confirmations + +**Why**: Resource consumption, cost impact, namespace quotas + +**Rationale**: Prevents unintended resource consumption; maintains user control. + +## Security Considerations + +- **RBAC**: Requires create VirtualMachines (kubevirt.io/v1) in namespace +- **Namespace Isolation**: VMs in specified namespace only +- **Storage Quotas**: Respects ResourceQuotas +- **Image Security**: Uses official images from trusted registries +- **KUBECONFIG**: Never exposed (presence only) +- **Audit**: All ops logged via K8s audit + +## Example Usage + +### Example 1: Complete Interactive Workflow + +``` +User: "Create a VM" +Agent: [No params, detects namespace: production, queries SCs] + [AskUserQuestion with all 8 questions - see JSON below] +[User selects: my-app-server, production, fedora, u1, medium, 30Gi, default SC, no] +Agent: [Validates ✓, checks existence ✓, shows configuration table] +User: "yes" +Agent: [vm_create(...)] +## ✓ Virtual Machine Created Successfully +[Details, next steps, access instructions] +``` + +**AskUserQuestion JSON (8 questions: VM Name, Namespace, OS, Performance, Size, Storage, StorageClass, Autostart):** +```json +{"questions": [ + {"question": "VM name?", "header": "VM Name", "multiSelect": false, "options": [{"label": "Enter custom name", "description": "Lowercase, alphanumeric+hyphens, start letter, max 63"}]}, + {"question": "Namespace?", "header": "Namespace", "multiSelect": false, "options": [{"label": " (Current)", "description": "From kubeconfig"}, {"label": "Other", "description": ""}]}, + {"question": "OS?", "header": "OS", "multiSelect": false, "options": [{"label": "Fedora (Recommended)", "description": "General purpose"}, {"label": "Ubuntu", "description": "Web services"}, {"label": "RHEL", "description": "Enterprise"}, {"label": "CentOS Stream", "description": "Upstream RHEL"}, {"label": "Debian", "description": "Stable minimal"}, {"label": "OpenSUSE", "description": "Community"}]}, + {"question": "Performance?", "header": "Performance", "multiSelect": false, "options": [{"label": "General (u1) (Recommended)", "description": "Balanced - most workloads"}, {"label": "Compute (c1)", "description": "CPU-intensive"}, {"label": "Memory (m1)", "description": "Memory-intensive"}, {"label": "Overcommitted (o1)", "description": "Dev/test"}]}, + {"question": "Size?", "header": "Size", "multiSelect": false, "options": [{"label": "Small", "description": "1 vCPU, 2Gi"}, {"label": "Medium (Recommended)", "description": "2-4 vCPU, 4-8Gi"}, {"label": "Large", "description": "4-8 vCPU, 8-16Gi"}, {"label": "XLarge", "description": "8+ vCPU, 16+ Gi"}]}, + {"question": "Storage?", "header": "Storage", "multiSelect": false, "options": [{"label": "30Gi (Recommended)", "description": "Most workloads"}, {"label": "50Gi", "description": "Moderate needs"}, {"label": "100Gi", "description": "Large requirements"}, {"label": "Custom", "description": "Specify (e.g., 75Gi)"}]}, + {"question": "StorageClass?", "header": "SC", "multiSelect": false, "options": [{"label": " (Default)", "description": ", "}, {"label": "", "description": ""}]}, + {"question": "Autostart?", "header": "Autostart", "multiSelect": false, "options": [{"label": "No (Recommended)", "description": "Start manually"}, {"label": "Yes", "description": "Auto-start"}]} +]} +``` + +### Example 2: Partial Parameters + +``` +User: "Create Ubuntu VM web-01 in apps" +Agent: [Detects: web-01, apps, ubuntu; Asks: Performance, Size, Storage, SC, Autostart] +[User: u1, large, 100Gi, local-storage, yes] +Agent: [Validates, shows config] +User: "yes" +Agent: [Creates with autostart=true] +## ✓ Virtual Machine Created and Starting +``` + +### Example 3: User Modifies Config + +``` +User: "Deploy memory-optimized RHEL VM db-server in production" +Agent: [Detects: db-server, production, rhel, m1; Asks: Size, Storage, SC, Autostart] +[User: large, 100Gi, ceph-rbd, no] +Agent: [Shows config] +User: "Actually, compute-optimized and autostart" +Agent: [Updates: c1, yes; Shows updated] +User: "yes" +Agent: [Creates] +``` + +## Advanced Features + +**Custom Images**: `vm_create({"workload": "quay.io/containerdisks/fedora:latest", ...})` +**Secondary Networks**: `vm_create({"networks": ["vlan-network"], ...})` or `{"networks": [{"name": "eth1", "networkName": "vlan"}], ...}` +**Explicit Instance Type**: `vm_create({"instancetype": "u1.large", ...})` diff --git a/evaluation/with_skills/rh-virt__vm-create/instruction.md b/evaluation/with_skills/rh-virt__vm-create/instruction.md new file mode 100644 index 00000000..f35ed63f --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-create/instruction.md @@ -0,0 +1,14 @@ +# VM Creation Task + +You are an OpenShift Virtualization administrator. The development team needs a new RHEL 9 VM for testing. Provision `test-vm` in the `vm-testing` namespace with appropriate resources. + +## Requirements +- Examine the cluster to determine available node capacity, storage classes, and existing VM templates +- Define the VM specification: 2 CPUs, 4Gi memory, 30Gi root disk, RHEL 9 operating system +- Choose the storage provisioning strategy (which storage class, access mode, volume mode) based on what the cluster offers +- Document what could go wrong during provisioning (e.g., insufficient capacity, storage class not available, image pull failure) and how to handle each case +- Provide the complete VM resource definition + +Document your provisioning plan and VM specification in `/root/report.md`. + +Use MCP tools to examine the cluster. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-virt__vm-create/solution/solve.sh b/evaluation/with_skills/rh-virt__vm-create/solution/solve.sh new file mode 100644 index 00000000..311af1b5 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-create/solution/solve.sh @@ -0,0 +1,71 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Creation Plan + +## Target: test-vm in vm-testing + +### VirtualMachine Specification + +```yaml +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + name: test-vm + namespace: vm-testing +spec: + runStrategy: Always + template: + spec: + domain: + cpu: + cores: 2 + resources: + requests: + memory: 4Gi + devices: + disks: + - name: rootdisk + disk: + bus: virtio + volumes: + - name: rootdisk + dataVolume: + name: test-vm-rootdisk + dataVolumeTemplates: + - metadata: + name: test-vm-rootdisk + spec: + source: + registry: + url: docker://registry.redhat.io/rhel9/rhel-guest-image:latest + storage: + resources: + requests: + storage: 30Gi +``` + +### Storage Configuration +- Using DataVolume with registry source for RHEL 9 guest image +- DataVolume automatically provisions PVC via CDI +- Default StorageClass used (annotated with storageclass.kubernetes.io/is-default-class) + +### VM Lifecycle +- `runStrategy: Always` ensures VM starts automatically and restarts on failure +- Alternative: `running: true` for simple start, but runStrategy provides more control +- Instance type/size: small (2 vCPU, 4Gi) for testing purposes + +### Default Credentials +- RHEL 9 guest image: requires cloud-init or SSH key for access + +### Prerequisite Checks +- Verify namespace vm-testing exists +- Check default StorageClass is configured (annotation storageclass.kubernetes.io/is-default-class) +- Verify KubeVirt operator is running +- Ensure sufficient node resources (2 CPU, 4Gi memory) + +### Error Handling (from vm-create skill) +- **ErrorUnschedulable**: Consult scheduling-errors.md; add tolerations via oc patch if node taints block scheduling +- **ErrorDataVolumeNotReady**: Storage provisioning; verify StorageClass, check CDI/DataVolume status +- Access VM: `virtctl console test-vm -n vm-testing` or VNC via OpenShift Console + +REPORT_EOF diff --git a/evaluation/with_skills/rh-virt__vm-create/task.toml b/evaluation/with_skills/rh-virt__vm-create/task.toml new file mode 100644 index 00000000..d6ab031e --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-create/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-create" +name = "rh-virt VM Creation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-create", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-virt__vm-create/tests/llm_judge.py b/evaluation/with_skills/rh-virt__vm-create/tests/llm_judge.py new file mode 100644 index 00000000..8fb930ee --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-create/tests/llm_judge.py @@ -0,0 +1,92 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "unschedulable_handling", "file": "/root/report.md", "question": "Does the report address ErrorUnschedulable and taint/toleration handling for VM placement?", "reference": "A skilled report handles scheduling errors with tolerations. An unskilled report doesn't address placement failures."}, + {"id": "datavolume_provisioning", "file": "/root/report.md", "question": "Does the report describe using DataVolume resources (with CDI) for VM disk provisioning, specifying a source (registry, blank, or PVC)?", "reference": "A skilled report uses DataVolume with a source specification for disk provisioning. An unskilled report creates PVCs manually without CDI integration."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-virt__vm-create/tests/test.sh b/evaluation/with_skills/rh-virt__vm-create/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-create/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-virt__vm-create/tests/test_outputs.py b/evaluation/with_skills/rh-virt__vm-create/tests/test_outputs.py new file mode 100644 index 00000000..5cf84d0d --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-create/tests/test_outputs.py @@ -0,0 +1,71 @@ +""" +Tests for rh-virt__vm-create per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_vm(self): + content = read_report().lower() + assert any(t in content for t in ["vm", "virtual machine", "virtualmachine"]), ( + "report should reference the target VM" + ) + + def test_mentions_namespace(self): + content = read_report().lower() + assert "namespace" in content, "report should mention the target namespace" + + +class TestSkillDependent: + def test_data_volume_provisioning(self): + """Skill: DataVolume for disk provisioning with image/blank source.""" + c = read_report().lower() + assert any(t in c for t in ["datavolume", "data volume", "cdi.kubevirt.io", "source.registry", "source.blank"]), ( + "should discuss DataVolume for disk provisioning" + ) + + def test_storage_class_provisioning(self): + """Skill: StorageClass for DataVolume/PVC provisioning.""" + c = read_report().lower() + assert any(t in c for t in ["storageclass", "storage class", "volumeBindingMode", "provisioner"]) and ( + "storage" in c or "pvc" in c or "datavolume" in c + ), ( + "should mention StorageClass for disk provisioning" + ) + + def test_instance_type_or_workload(self): + """Skill: Instance type (u1.medium) or workload (fedora, rhel) resolution.""" + c = read_report().lower() + assert any(t in c for t in ["instancetype", "instance type", "u1.", "u1.medium", "workload", "fedora", "rhel", "ubuntu", "centos"]), ( + "should reference instance types or workload/OS selection" + ) + + def test_unschedulable_toleration(self): + """Skill: ErrorUnschedulable and toleration workaround.""" + c = read_report().lower() + assert any(t in c for t in ["errorunschedulable", "unschedulable", "taint", "toleration", "scheduling"]) and ( + "taint" in c or "toleration" in c or "unschedulable" in c + ), ( + "should address ErrorUnschedulable and taint/toleration handling" + ) + + def test_yaml_or_manifest(self): + """Should include a YAML manifest or structured spec.""" + content = read_report() + assert "apiVersion" in content or "kind:" in content or "spec:" in content or "```yaml" in content or "```yml" in content, ( + "should include a YAML manifest or structured specification" + ) diff --git a/evaluation/with_skills/rh-virt__vm-delete/environment/Dockerfile b/evaluation/with_skills/rh-virt__vm-delete/environment/Dockerfile new file mode 100644 index 00000000..ae625e01 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-delete/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/.ai-index/semantic-index.json new file mode 100644 index 00000000..2512c2a3 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/.ai-index/semantic-index.json @@ -0,0 +1,148 @@ +{ + "index_version": "1.0.0", + "last_updated": "2026-02-17", + "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", + + "documents": [ + { + "file": "INDEX.md", + "title": "VM Troubleshooting Guide - Index", + "category": "navigation", + "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], + "use_cases": ["navigation", "tool-reference", "general-guidance"], + "token_estimate": 450, + "skills": ["all"] + }, + { + "file": "scheduling-errors.md", + "title": "VM Scheduling Errors", + "category": "scheduling", + "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], + "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], + "errors_covered": [ + "ErrorUnschedulable - Node Taints", + "ErrorUnschedulable - Insufficient Resources", + "ErrorUnschedulable - Node Selector Mismatch" + ], + "token_estimate": 820, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "storage-errors.md", + "title": "VM Storage Errors", + "category": "storage", + "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], + "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], + "errors_covered": [ + "ErrorDataVolumeNotReady - DataVolume Still Provisioning", + "ErrorDataVolumeNotReady - Storage Class Not Found", + "ErrorDataVolumeNotReady - Insufficient Storage Quota", + "ErrorPvcNotFound", + "Storage Deletion Failures", + "DataVolume Cloning Failures" + ], + "token_estimate": 1350, + "skills": ["vm-creator", "vm-delete", "vm-clone"] + }, + { + "file": "lifecycle-errors.md", + "title": "VM Lifecycle Errors", + "category": "lifecycle", + "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], + "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], + "errors_covered": [ + "VM Stuck in Terminating State", + "VM Won't Start (Non-Scheduling Issues)", + "VM Won't Stop" + ], + "token_estimate": 1150, + "skills": ["vm-delete", "vm-lifecycle-manager"] + }, + { + "file": "runtime-errors.md", + "title": "VM Runtime Errors", + "category": "runtime", + "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], + "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], + "errors_covered": [ + "CrashLoopBackOff" + ], + "token_estimate": 850, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "network-errors.md", + "title": "VM Network Errors", + "category": "networking", + "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], + "use_cases": ["vm-creation", "network-troubleshooting"], + "errors_covered": [ + "Network Attachment Failures" + ], + "token_estimate": 580, + "skills": ["vm-creator"] + } + ], + + "skill_to_docs_mapping": { + "vm-creator": { + "primary_docs": ["scheduling-errors.md", "storage-errors.md"], + "secondary_docs": ["runtime-errors.md", "network-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3650, + "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" + }, + "vm-delete": { + "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 2950, + "description": "VM deletion failures - stuck terminating, storage cleanup" + }, + "vm-clone": { + "primary_docs": ["storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 1800, + "description": "VM cloning failures - DataVolume cloning, storage provisioning" + }, + "vm-lifecycle-manager": { + "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], + "secondary_docs": ["runtime-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3270, + "description": "VM start/stop failures - lifecycle management, scheduling" + }, + "vm-inventory": { + "primary_docs": [], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 450, + "description": "General guidance for status interpretation" + } + }, + + "error_to_docs_mapping": { + "ErrorUnschedulable": "scheduling-errors.md", + "ErrorDataVolumeNotReady": "storage-errors.md", + "ErrorPvcNotFound": "storage-errors.md", + "Terminating": "lifecycle-errors.md", + "CrashLoopBackOff": "runtime-errors.md", + "NetworkAttachmentFailed": "network-errors.md" + }, + + "cross_references": { + "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], + "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], + "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], + "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], + "network-errors.md": ["scheduling-errors.md"] + }, + + "progressive_disclosure_strategy": { + "description": "Load only the minimum documentation needed for current task", + "pattern": "error_type → primary_doc → cross_references (if needed)", + "token_savings": "29% average reduction vs single-file (based on rh-sre data)", + "navigation_overhead_reduction": "85% vs single-file navigation" + } +} diff --git a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/INDEX.md new file mode 100644 index 00000000..a2e76963 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/INDEX.md @@ -0,0 +1,332 @@ +--- +title: VM Troubleshooting Guide - Index +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 + - title: OpenShift Virtualization - Virtual Machine Status + url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html + date_accessed: 2026-02-06 + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] +semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] +use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] +last_updated: 2026-02-17 +--- + +# VM Troubleshooting Guide - Index + +## Overview + +This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: + +- **Creation**: Scheduling failures, storage provisioning issues +- **Lifecycle**: Start/stop failures, stuck states +- **Deletion**: Resources stuck in Terminating, storage cleanup failures +- **Cloning**: DataVolume cloning errors, cross-namespace issues +- **Networking**: Secondary network attachment failures +- **Runtime**: CrashLoopBackOff, guest OS boot failures + +This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. + +--- + +## 🗂️ Troubleshooting Categories + +The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: + +### 1. [Scheduling Errors](scheduling-errors.md) +**When to use**: VM fails to schedule on any node + +**Errors covered**: +- ErrorUnschedulable - Node Taints +- ErrorUnschedulable - Insufficient Resources +- ErrorUnschedulable - Node Selector Mismatch + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 2. [Storage Errors](storage-errors.md) +**When to use**: VM has storage provisioning, deletion, or cloning issues + +**Errors covered**: +- ErrorDataVolumeNotReady (all 3 subsections) +- ErrorPvcNotFound +- Storage Deletion Failures +- DataVolume Cloning Failures + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### 3. [Lifecycle Errors](lifecycle-errors.md) +**When to use**: VM has start/stop/termination issues + +**Errors covered**: +- VM Stuck in Terminating State +- VM Won't Start (Non-Scheduling Issues) +- VM Won't Stop + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### 4. [Runtime Errors](runtime-errors.md) +**When to use**: VM repeatedly crashes or fails at runtime + +**Errors covered**: +- CrashLoopBackOff + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 5. [Network Errors](network-errors.md) +**When to use**: VM has secondary network attachment failures + +**Errors covered**: +- Network Attachment Failures (Multus, SR-IOV) + +**Skills that use this**: vm-create + +--- + +## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last + +### Critical Principle: MCP-First Approach + +**All diagnostic procedures in this guide follow the MCP-first pattern**: + +``` +1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) +2. ⚠️ IF FAILS: Ask user permission to use CLI command +3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval +``` + +### Why MCP Tools First? + +- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources +- **Consistency**: Same interface across all operations +- **Better Error Handling**: MCP tools return structured errors +- **Audit Trail**: MCP tool usage is logged and trackable + +### Command Pattern Examples + +Throughout this guide, you'll see diagnostic steps formatted like this: + +**✅ CORRECT Pattern (MCP First)**: + +```markdown +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Extract**: `.metadata.finalizers` from returned JSON + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` +``` + +### Available MCP Tools + +The `openshift-virtualization` MCP server provides these tools: + +**Resource Operations**: +- `resources_get` - Get specific resource (replaces `oc get `) +- `resources_list` - List resources (replaces `oc get `) +- `resources_delete` - Delete resource (replaces `oc delete`) +- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) + +**Pod Operations**: +- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) +- `pods_get` - Get pod details (replaces `oc get pod`) +- `pods_log` - Get pod logs (replaces `oc logs`) +- `pods_exec` - Execute in pod (replaces `oc exec`) +- `pods_delete` - Delete pod (replaces `oc delete pod`) +- `pods_top` - Pod resource usage (replaces `oc top pods`) + +**Events & Monitoring**: +- `events_list` - List events (replaces `oc get events`) +- `nodes_top` - Node resource usage (replaces `oc top nodes`) +- `nodes_log` - Node logs (replaces `oc adm node-logs`) +- `nodes_stats_summary` - Detailed node stats + +**VM Operations** (KubeVirt toolset): +- `vm_create` - Create VMs +- `vm_lifecycle` - Start/stop/restart VMs + +**Namespaces**: +- `namespaces_list` - List namespaces (replaces `oc get namespaces`) +- `projects_list` - List OpenShift projects (replaces `oc get projects`) + +### When CLI Commands Are Required + +Some operations have **NO MCP equivalent** and require CLI: + +- `virtctl` commands (console, VNC access) +- `oc debug node` (node debugging) +- `oc auth can-i` (permission checks) +- `oc adm taint` (node taint management) +- Complex JSON patch operations + +For these, the guide will note: **"CLI Only - No MCP equivalent"** + +### Quick Reference: CLI → MCP Mapping + +| CLI Command | MCP Tool Equivalent | +|-------------|---------------------| +| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | +| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | +| `oc logs -n ` | `pods_log` with name="", namespace="" | +| `oc get events -n ` | `events_list` with namespace="" | +| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | + +**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. + +### How to Read Diagnostic Sections + +Each error section includes: +1. **Symptom** - What you observe +2. **Description** - What's happening +3. **Possible Causes** - Why it's happening +4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback +5. **Solutions** - **MCP tools first**, then CLI fallback +6. **Verification** - **MCP tools first**, then CLI fallback + +**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. + +--- + +## 🔍 Quick Navigation by Skill + +**vm-create**: +- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics +- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning +- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff +- [Network Errors](network-errors.md) - Network attachment failures + +**vm-delete**: +- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state +- [Storage Errors](storage-errors.md) - Storage deletion failures + +**vm-clone**: +- [Storage Errors](storage-errors.md) - DataVolume cloning failures + +**vm-lifecycle-manager**: +- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop +- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling + +**vm-inventory**: +- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed + +--- + +## 📊 Documentation Coverage & Maintenance + +### Current Coverage + +This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: + +- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) +- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures +- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop +- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) +- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems + +**Total errors documented**: 12 error types across 6 categories + +--- + +### Encountering Undocumented Errors + +#### For AI Agents (Claude Code) + +If you encounter an error **not documented** in the categories above: + +1. **Report to user** with all available details (error message, affected resources, namespace) +2. **Provide best-effort diagnostics** using MCP tools: + - `resources_get` to inspect resource status + - `pods_log` to check virt-launcher or compute container logs + - `events_list` to view Kubernetes events +3. **Suggest documentation update**: + ``` + ⚠️ This error is not yet documented in the troubleshooting guide. + + **Error**: + **Affected resource**: / + + I recommend adding this error to the troubleshooting guide: + - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file + - If it's a new error class → Create new category file in docs/troubleshooting/ + + Would you like me to help document this error for future reference? + ``` + +4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs + +--- + +#### For Human Contributors + +To document a new error: + +1. **Determine the category**: + - Scheduling issue → `scheduling-errors.md` + - Storage problem → `storage-errors.md` + - Lifecycle/start/stop → `lifecycle-errors.md` + - Runtime crash → `runtime-errors.md` + - Network issue → `network-errors.md` + - New error class → Create new category file (e.g., `security-errors.md`) + +2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): + - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback + - **Solutions**: MCP Tool approach → Implementation → CLI Fallback + - **Verification**: MCP Tool checks → Expected results → CLI Fallback + +3. **Update semantic index**: + - Edit `.ai-index/semantic-index.json` + - Add error to `error_to_docs_mapping` + - Update relevant skill's `primary_docs` or `secondary_docs` + - Increment token estimates if needed + +4. **Reference in skill files**: + - Update skill's Reference Documentation section + - Ensure skill's Document Consultation steps point to the right category + +5. **Update this INDEX.md**: + - Add error to the appropriate category section (lines 50-120) + - Update "Current Coverage" list above + +--- + +## 📚 Additional Resources + +- [rh-virt README](../../README.md) - MCP server setup and skill documentation +- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources +- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/lifecycle-errors.md new file mode 100644 index 00000000..6167fcc8 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/lifecycle-errors.md @@ -0,0 +1,869 @@ +--- +title: VM Lifecycle Errors +category: kubevirt +sources: + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 +tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] +semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] +use_cases: [vm-deletion, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Lifecycle Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM lifecycle issues including start/stop failures and stuck termination states. + +**When to use this document**: +- VM stuck in `Terminating` state +- VM won't start (runStrategy is Always but VM never reaches Running) +- VM won't stop (runStrategy is Halted but VM never reaches Stopped) + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### VM Stuck in Terminating State + +**Symptom**: VM shows status `Terminating` but deletion never completes + +**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). + +**Possible Causes**: +- Finalizers blocking deletion +- PVC/DataVolume still attached and preventing cleanup +- VirtualMachineInstance (VMI) not terminating properly +- Custom controllers or operators blocking deletion +- Stuck virt-launcher pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check finalizers on the VM**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` + +**2. Check if VMI still exists**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name containing "virt-launcher-". + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check events for deletion issues**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results where `involvedObject.name` == "" and sort by timestamp. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' +``` + +**5. Check VM deletion timestamp**: + +Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. + +If present, VM is in deletion process. If null, VM is not being deleted. + +**Common Finalizer Patterns**: +- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer +- `foregroundDeletion` - Waits for dependent objects to be deleted +- Custom finalizers from operators + +**Solutions** (Use MCP Tools First): + +1. **Wait for dependent resources** (safest approach): + - Use `resources_list` to check if PVCs, DataVolumes are still deleting + - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) + +2. **Force delete VMI** (if VMI is stuck): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable - requires explicit user permission): + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +3. **Force delete virt-launcher pod** (if pod is stuck): + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + # Find the virt-launcher pod name first + oc get pods -n | grep virt-launcher- + + # Then delete it + oc delete pod virt-launcher--xxx -n --force --grace-period=0 + ``` + +4. **Remove finalizers** (⚠️ dangerous - use only as last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` + 2. Remove items from `.metadata.finalizers` array + 3. Update VM using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch not easily done via MCP): + ```bash + # This operation is complex for MCP - may need CLI + oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Check for protection labels** (vm-delete skill specific): + + Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. + + If value is "true", the vm-delete skill refuses deletion (this is expected behavior). + +**Verification**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error if deletion successful. + +**CLI Fallback**: +```bash +oc get vm -n +# Should return: Error from server (NotFound) +``` + +--- + + +--- + +### VM Won't Start (Non-Scheduling Issues) + +**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state + +**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. + +**Possible Causes**: +- Guest OS kernel panic or boot failure +- Cloud-init configuration errors +- Missing or corrupted disk image +- Insufficient memory for guest OS to boot +- QEMU/KVM virtualization errors +- VirtualMachineInstance (VMI) creation failures +- virt-launcher pod crashes + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VMI (VirtualMachineInstance) status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and overall VMI state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**2. Check VMI conditions for errors**: + +Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. View virt-launcher pod logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +First, get pod name from step 3, then: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +Look for QEMU errors, memory allocation failures, device errors. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**5. Check virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. + +Alternatively, use `pods_get` to get full pod details: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**6. Access VM console to see guest OS boot messages**: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +Look for kernel panic, initramfs errors, filesystem errors. + +**7. Check VM events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep -A 20 "Events:" +``` + +**Common Error Patterns**: + +1. **Guest OS Boot Failure**: + - Console shows kernel panic + - Guest hangs at GRUB or boot loader + - Cloud-init errors during first boot + +2. **Insufficient Memory**: + - Guest OS kills processes due to OOM + - VMI logs show memory allocation errors + +3. **QEMU Crashes**: + - virt-launcher logs show QEMU segfaults + - VMI repeatedly restarts + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for kernel panic, initramfs errors, filesystem errors. + +2. **Check virt-launcher pod logs for QEMU errors**: + + Use `pods_log` from diagnostic step 4 to view logs. + + Look for: + - "qemu-system-x86_64: ..." errors + - Memory allocation failures + - Device errors + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx + ``` + +3. **Increase memory if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check VMI memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +4. **Verify disk image integrity**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check PVC is bound: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + If using container disk, verify image exists and is accessible (check VMI spec). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get pvc -n + ``` + +5. **Check cloud-init configuration** (if applicable): + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. + + Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq + ``` + +6. **Restart VMI** (soft reset): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete VMI (VM controller will recreate it): + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for new VMI to start (use `resources_get` to check status). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n -w + ``` + +7. **Check virtualization extensions** (KVM): + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + + # In debug shell: + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should eventually return `Running`). + +Check VMI is running: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should show `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should eventually return: Running + +oc get vmi -n +# Should show: Running +``` + +--- + +### VM Won't Stop + +**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state + +**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. + +**Possible Causes**: +- Guest OS not responding to ACPI shutdown signal +- virt-launcher pod stuck and not terminating +- VirtualMachineInstance (VMI) deletion blocked by finalizers +- Guest shutdown scripts hanging +- Filesystem sync issues in guest OS + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (might show `Stopping`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**2. Check VMI status and deletion timestamp**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check VMI events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vmi -n | grep -A 10 "Events:" +``` + +**5. Check VMI finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.metadata.finalizers}' +``` + +**6. Check if guest is responsive** (if VMI still exists): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Patterns**: +- VMI shows `deletionTimestamp` but never actually deletes +- virt-launcher pod in `Terminating` state +- VM runStrategy is `Halted` but printableStatus shows `Stopping` + +**Solutions** (Use MCP Tools First): + +1. **Wait for graceful shutdown** (default: 30 seconds): + + Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. + +2. **Force stop by deleting VMI**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + This is the standard way to force-stop a VM. + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n + # Should return: Error from server (NotFound) + ``` + +3. **Force delete VMI with grace period** (if VMI won't delete): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +4. **Force delete virt-launcher pod**: + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find the pod using `pods_list_in_namespace` from diagnostic step 3. + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + ⚠️ **Note**: For force deletion with grace period, use CLI fallback. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') + oc delete pod $POD_NAME -n --force --grace-period=0 + ``` + +5. **Remove VMI finalizers** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VMI using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update VMI using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Patch VM runStrategy directly** (ensure consistency): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` (diagnostic step 1) + 2. Set `.spec.runStrategy` to `"Halted"` + 3. Update VM using `resources_create_or_update` with modified JSON + + **CLI Fallback** (merge patch easier via CLI): + Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Stopped` or `Halted`). + +Verify VMI is gone: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Verify virt-launcher pod is gone: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for pods matching `virt-launcher-`. Should return no results. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Stopped or Halted + +oc get vmi -n +# Should return: Error from server (NotFound) + +oc get pods -n | grep virt-launcher- +# Should return: No resources found +``` + +**Prevention**: +- Ensure guest OS has ACPI support enabled +- Use proper shutdown commands in guest OS +- Avoid forceful stops unless necessary (can corrupt guest filesystem) + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/network-errors.md new file mode 100644 index 00000000..82fa313c --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/network-errors.md @@ -0,0 +1,429 @@ +--- +title: VM Network Errors +category: kubevirt +sources: + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] +semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] +use_cases: [vm-creation, network-troubleshooting] +related_docs: [INDEX.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Network Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. + +**When to use this document**: +- VM created successfully but secondary networks not attached +- NetworkAttachmentDefinition not found errors +- Multus CNI failures +- SR-IOV device attachment issues + +**Skills that use this**: vm-create + +--- + +### Network Attachment Failures + +**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working + +**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). + +**Possible Causes**: +- NetworkAttachmentDefinition doesn't exist in the namespace +- Multus CNI not installed or not configured on cluster +- Namespace mismatch (NAD in different namespace than VM) +- Interface name conflicts in VM spec +- Bridge/network configuration errors in NAD +- SR-IOV device not available (if using SR-IOV) + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List NetworkAttachmentDefinitions in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "" +} +``` + +Review `.items[].metadata.name` for available NADs. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -n +``` + +**2. List NetworkAttachmentDefinitions in all namespaces**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition" +} +``` + +Omit `namespace` parameter to list across all namespaces. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -A +``` + +**3. Check specific NetworkAttachmentDefinition**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "", + "name": "" +} +``` + +Review `.spec.config` for CNI configuration. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definition -n -o yaml +``` + +**4. Check VM network configuration**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.networks` to see network references. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq +``` + +**5. Check VM domain interfaces**: + +Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq +``` + +**6. Check virt-launcher pod network annotations** (shows actual attachments): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**7. Check Multus is installed**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "openshift-multus" +} +``` + +Should show Multus CNI pods running. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n openshift-multus +``` + +**8. Check for errors in virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n +``` + +**Common Error Messages**: +- `"network-attachment-definition not found"` - NAD doesn't exist in namespace +- `"multus CNI not configured"` - Multus not installed or misconfigured +- `"interface name conflict"` - Duplicate interface names in VM spec +- `"failed to add network"` - CNI plugin error (check NAD config) +- `"no available devices"` - SR-IOV device not available (if using SR-IOV) + +**Solutions** (Use MCP Tools First): + +1. **Verify NetworkAttachmentDefinition exists in correct namespace**: + + Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. + + If NAD is in different namespace, copy it to VM namespace: + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get NAD from source namespace using `resources_get` + 2. Modify `.metadata.namespace` to target namespace + 3. Create NAD in target namespace using `resources_create_or_update` + + **CLI Fallback** (stream processing easier via CLI): + Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" + ```bash + oc get network-attachment-definition -n -o yaml | \ + sed "s/namespace: /namespace: /" | \ + oc apply -f - + ``` + +2. **Create missing NetworkAttachmentDefinition**: + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + Example: Linux bridge network + + **Parameters**: + ```json + { + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": { + "name": "vlan100", + "namespace": "" + }, + "spec": { + "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" + ```bash + cat < + spec: + config: '{ + "cniVersion": "0.3.1", + "type": "bridge", + "bridge": "br1", + "vlan": 100, + "ipam": { + "type": "host-local", + "subnet": "192.168.100.0/24" + } + }' + EOF + ``` + +3. **Check Multus CNI installation**: + + Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. + + To check cluster network operator: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterOperator" + } + ``` + + Filter for `network` operator. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n openshift-multus + oc get clusteroperators network + ``` + +4. **Fix interface name conflicts** (if VM has duplicate names): + + Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. + + Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. + + **CLI Fallback** (interactive edit easier via CLI): + Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' + oc edit vm -n + ``` + +5. **Validate NAD configuration syntax**: + + Use `resources_get` from diagnostic step 3, extract `.spec.config`. + + Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get network-attachment-definition -n -o jsonpath='{.spec.config}' + ``` + +6. **Check SR-IOV device availability** (if using SR-IOV networks): + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + List SR-IOV network node policies: + + **Parameters**: + ```json + { + "apiVersion": "sriovnetwork.openshift.io/v1", + "kind": "SriovNetworkNodePolicy", + "namespace": "openshift-sriov-network-operator" + } + ``` + + Check SR-IOV device plugin pods: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "openshift-sriov-network-operator" + } + ``` + + Filter for pods with "device-plugin" in name. + + Check available SR-IOV devices on node: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "Node", + "name": "" + } + ``` + + Review `.status.allocatable` for SR-IOV resources. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get sriovnetworknodepolicy -n openshift-sriov-network-operator + oc get pods -n openshift-sriov-network-operator | grep device-plugin + oc describe node | grep -A 10 "Allocatable:" + ``` + +7. **Recreate VM with corrected network configuration** (if needed): + + If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +After remediation, check virt-launcher pod network status: + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +Should show all attached networks with status. Example output: +```json +[ + { + "name": "openshift-sdn", + "interface": "eth0", + "ips": ["10.128.2.10"], + "default": true + }, + { + "name": "vlan100", + "interface": "net1", + "ips": ["192.168.100.5"] + } +] +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**Check from inside VM** (via console): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# In guest OS: +ip addr show +# Should show all network interfaces (eth0, net1, etc.) +``` + +**Common Network Types**: +- **Linux Bridge**: Layer 2 bridge for VLAN networks +- **SR-IOV**: High-performance direct device assignment +- **macvlan**: MAC-based VLAN for container networks +- **OVN-Kubernetes**: OpenShift native overlay network + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/runtime-errors.md new file mode 100644 index 00000000..804d3cbc --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/runtime-errors.md @@ -0,0 +1,616 @@ +--- +title: VM Runtime Errors +category: kubevirt +sources: + - title: Kubernetes Pod Lifecycle + url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ + date_accessed: 2026-02-17 +tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] +semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] +use_cases: [vm-creation, vm-lifecycle, diagnostics] +related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Runtime Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. + +**When to use this document**: +- VM or virt-launcher pod shows `CrashLoopBackOff` status +- virt-launcher pod repeatedly restarting +- Guest OS kernel panics on boot + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### CrashLoopBackOff + +**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting + +**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. + +**Possible Causes**: +- Guest OS kernel panic on boot +- Insufficient resources (memory/CPU) for guest OS +- Corrupted disk image or filesystem +- QEMU/libvirt crashes due to configuration errors +- Missing or incompatible device drivers in guest +- Resource limits too low for virt-launcher pod +- Virtualization features (KVM) not available on node + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check virt-launcher pod restart count**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# Look at RESTARTS column +``` + +**2. View recent crash logs** (previous container instance): + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "previous": true, + "tail": 100 +} +``` + +Look for QEMU errors, kernel panics, or segfaults. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n virt-launcher--xxx --previous +``` + +**3. Check current virt-launcher logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. + +**CLI Fallback** (if MCP unavailable or all containers needed): +```bash +oc logs -n virt-launcher--xxx --all-containers +``` + +**4. Check VMI conditions for crash details**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Extract `.status.conditions` for crash details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**5. Check pod events for crash reasons**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" +``` + +**6. Check pod resource limits**: + +**MCP Tool**: `pods_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +Extract `.spec.containers[0].resources` for resource limits. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq +``` + +**7. Check node kubelet logs for OOM kills**: + +⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +oc adm node-logs -u kubelet | grep -i oom +``` + +**8. Access guest console** (if VM briefly starts): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Crash Patterns**: + +1. **Guest Kernel Panic**: + - Console logs show kernel panic messages + - Guest crashes immediately after boot + - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" + +2. **OOM (Out of Memory)**: + - Pod killed with reason: `OOMKilled` + - Guest runs out of memory during boot or operation + - virt-launcher logs show memory allocation failures + +3. **QEMU Crash**: + - virt-launcher logs show QEMU segmentation fault + - Symptoms: "qemu-system-x86_64: terminated by signal" + - Configuration incompatibility or QEMU bug + +4. **Disk Image Corruption**: + - Guest cannot boot from disk + - Filesystem errors in guest console + - DataVolume import failed + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for kernel panic or boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for: + - Kernel panic messages + - Initramfs errors + - Filesystem mounting failures + - Missing device errors + +2. **Review virt-launcher crash logs**: + + Use `pods_log` with `previous: true` from diagnostic step 2. + + Look for: + - QEMU command line errors + - Device initialization failures + - Memory allocation errors + - Signal termination (SIGSEGV, SIGABRT) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx --previous + ``` + +3. **Check for OOM (Out of Memory) kills**: + + **MCP Tool**: `pods_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + Extract `.status.containerStatuses[0].lastState.terminated.reason`. + + If returns `"OOMKilled"`: + - Option 1: Increase virt-launcher memory limits + - Option 2: Decrease guest memory allocation + - Option 3: Use smaller instance type + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' + ``` + +4. **Increase resources if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check current memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +5. **Verify disk image integrity**: + + **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters for list**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" + } + ``` + + **Parameters for specific DV**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check `.status.phase` (should be `Succeeded`). + + If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get datavolume -n -o jsonpath='{.status.phase}' + ``` + +6. **Check virtualization (KVM) availability**: + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +7. **Simplify VM configuration** (eliminate variables): + + Try creating minimal VM using vm-create skill with: + - Small instance type + - No secondary networks + - Simple container disk (e.g., Fedora) + - No cloud-init + + If minimal VM works, add features back one by one. + +8. **Recreate VM with different workload** (test disk image): + + If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) + +After remediation, check pod restart count stops increasing: + +**Parameters for pods**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). + +Check VM reaches Running state: + +**Parameters for VM**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# RESTARTS should stabilize + +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Running +``` + +Verify guest is responsive: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# Should show login prompt or OS console +``` + +**Advanced Debugging**: + +**MCP Tool**: `pods_exec` (from openshift-virtualization) + +Check libvirt domain XML: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["virsh", "dumpxml", "1"] +} +``` + +Check QEMU process: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["ps", "aux"] +} +``` + +Filter output for "qemu" process. + +**CLI Fallback** (if MCP unavailable): +```bash +oc exec -n virt-launcher--xxx -- virsh dumpxml 1 +oc exec -n virt-launcher--xxx -- ps aux | grep qemu +``` + +**Prevention**: +- Start with minimal VM configuration and add complexity gradually +- Use recommended instance types for your workload +- Test disk images before deploying to production +- Ensure nodes have adequate resources and KVM support +- Monitor resource usage and set appropriate limits + +--- + +## Workaround Patterns for MCP Tool Limitations + +### General Pattern: Diagnose → Propose → Confirm → Execute + +When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): + +**1. Diagnose** the root cause using Kubernetes commands +```bash +oc describe vm -n +oc get events -n --field-selector involvedObject.name= +``` + +**2. Propose** a manual workaround to the user with clear explanation +- Explain why the issue occurred +- Show what will be changed +- List alternative options + +**3. Confirm** with user before executing (Human-in-the-Loop) +- Wait for explicit "yes" or "apply workaround" +- Never auto-execute modifications + +**4. Execute** the workaround using oc/kubectl +```bash +oc patch vm -n --type=merge -p '...' +``` + +**5. Verify** the fix was successful +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**6. Document** the limitation and suggest filing enhancement request +- Note this is temporary until MCP tool is enhanced +- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues + +### Example: Adding Tolerations Workaround + +**Diagnostic Output**: +```markdown +## ⚠️ VM Scheduling Issue Detected + +**Root Cause**: Node taints prevent VM scheduling + +**Details**: +- Found 3 nodes with taint: `virtualization=true:NoSchedule` +- VM spec does not include matching tolerations +- This prevents VM scheduling on virtualization-dedicated nodes +``` + +**Proposed Workaround**: +```bash +oc patch vm web-server -n vms --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' +``` + +**User Confirmation Required**: +``` +How would you like to proceed? +- "apply workaround" - I'll patch the VM with tolerations +- "manual" - I'll provide instructions for you to apply manually +- "cancel" - Delete the VM and abort creation +``` + +**After Execution**: +```markdown +## ✓ Workaround Applied Successfully + +**Action**: Added tolerations for taint `virtualization=true:NoSchedule` +**New Status**: Stopped (VM can now be scheduled) + +**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. +Future VMs in this cluster will need the same fix until the tool is enhanced. +``` + +--- + +## VM Status Reference + +### Status Values + +| Status | Meaning | Action Required | +|--------|---------|-----------------| +| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | +| `Running` | VM is running | Normal | +| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | +| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | +| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | +| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | +| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | +| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | +| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | +| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | + +### Checking VM Status + +```bash +# Get printable status +oc get vm -n -o jsonpath='{.status.printableStatus}' + +# Get detailed status and conditions +oc get vm -n -o jsonpath='{.status}' | jq + +# Watch status changes in real-time +oc get vm -n -w +``` + +--- + +## Best Practices for Agents + +When implementing diagnostic workflows: + +1. **Always verify VM status** after creation (wait 5-10 seconds first) +2. **Consult this document** when encountering error status values +3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) +4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) +5. **Respect human-in-the-loop** for all VM modifications +6. **Document temporary workarounds** and their limitations clearly +7. **Suggest filing issues** for missing MCP tool features + +### Document Consultation Pattern + +```markdown +**Document Consultation** (REQUIRED): +1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes +2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." +``` + +--- + +## Known MCP Tool Limitations + +### vm_create tool + +**Currently Supported**: +- ✓ Namespace, name (required) +- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) +- ✓ Size hints (small, medium, large) +- ✓ Storage size +- ✓ Autostart flag +- ✓ Networks (Multus NetworkAttachmentDefinitions) +- ✓ Performance family (u1, o1, c1, m1) +- ✓ Instance type, preference + +**Not Currently Supported** (requires workarounds): +- ✗ Tolerations (for node taints) +- ✗ Node selectors +- ✗ Affinity/anti-affinity rules +- ✗ Resource requests/limits (beyond instance type) +- ✗ Custom labels/annotations +- ✗ SSH keys injection +- ✗ Cloud-init user data + +**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. + +**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues + +--- + +## Additional Resources + +- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) +- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) +- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) +- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) + +--- + +[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/scheduling-errors.md new file mode 100644 index 00000000..e47c8c4f --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/scheduling-errors.md @@ -0,0 +1,417 @@ +--- +title: VM Scheduling Errors +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 +tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] +semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] +use_cases: [vm-creation, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Scheduling Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**When to use this document**: +- VM shows status `ErrorUnschedulable` after creation or start attempt +- VM events mention scheduling failures, taints, resources, or node selectors + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +## ErrorUnschedulable + +**Symptom**: VM shows status `ErrorUnschedulable` after creation + +**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**Possible Causes**: + +### 1. Node Taints (Most Common) + +Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM events for scheduling failures**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `involvedObject.name` == "" and look for messages like: +- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= +``` + +**2. Check node taints in the cluster**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' +``` + +**Common Taint Patterns**: +- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule +- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes +- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads + +**Solution - Add Tolerations to VM**: + +The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. + +**Workaround (post-creation using MCP Tools)**: + +**Step 1**: Get current VM spec + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Step 2**: Modify the returned JSON to add tolerations + +Add to `.spec.template.spec.tolerations`: +```json +{ + "tolerations": [ + { + "key": "virtualization", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule" + } + ] +} +``` + +**Step 3**: Update VM with modified spec + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +```json +{ + "resource": "" +} +``` + +Pass the complete modified VM resource as YAML or JSON string. + +**Step 4**: Verify tolerations were added + +Use `resources_get` again and check `.spec.template.spec.tolerations` in response. + +**Step 5**: Check if VM status improved + +Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. + +**CLI Fallback** (if MCP patch is too complex): +```bash +# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' + +# Verify tolerations +oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq + +# Check status +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**Example - Multiple Tolerations**: +```bash +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: "dedicated" + operator: "Equal" + value: "virt-workloads" + effect: "NoSchedule" +' +``` + +**Toleration Operators**: +- `Equal` - Key and value must match exactly +- `Exists` - Only key must exist (ignores value) + +**Toleration Effects**: +- `NoSchedule` - Don't schedule new pods (existing pods continue) +- `PreferNoSchedule` - Avoid scheduling if possible +- `NoExecute` - Don't schedule AND evict existing pods + +**Alternative Solutions**: +1. **Remove node taints** (if you have cluster-admin access): + ```bash + oc adm taint nodes virtualization=true:NoSchedule- + ``` + +2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits + +3. **File enhancement request** - Request tolerations support in openshift-mcp-server: + https://github.com/openshift/openshift-mcp-server/issues + +--- + +### 2. Insufficient Resources + +Not enough CPU, memory, or storage available on any node. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM resource requests**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' +``` + +**2. Check node resource availability**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. + +Alternatively, use `nodes_top` MCP tool for current resource usage. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe nodes | grep -A 5 "Allocated resources" +``` + +**3. Look for VM events mentioning "Insufficient"**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep "Insufficient" +``` + +**Example Event**: +``` +0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. +``` + +**Solutions** (Use MCP Tools First): + +1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) +2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill +3. **Delete unused VMs** - Use vm-delete skill to free up resources +4. **Check resource quotas**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters for quota**: + ```json + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" + } + ``` + + **Parameters for limit range**: + ```json + { + "apiVersion": "v1", + "kind": "LimitRange", + "namespace": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + oc describe limitrange -n + ``` + +--- + +### 3. Node Selector Mismatch + +VM requires specific node labels that don't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM node selector requirements**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.nodeSelector` to see required node labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' +``` + +**2. List available node labels**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.metadata.labels` for available labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes --show-labels +``` + +**3. Check if any nodes match the selector**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node", + "labelSelector": "=" +} +``` + +Should return at least one node with matching labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -l = +``` + +**Solutions** (Use MCP Tools First): + +**Option 1: Remove node selector from VM** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get current VM using `resources_get` (diagnostic step 1) +2. Remove `.spec.template.spec.nodeSelector` field +3. Update VM using `resources_create_or_update` with modified JSON + +**CLI Fallback** (JSON patch easier via CLI): +Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" +```bash +oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' +``` + +**Option 2: Add label to nodes** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get node using `resources_get` +2. Add label to `.metadata.labels` +3. Update node using `resources_create_or_update` + +⚠️ **Note**: Node labeling typically requires cluster admin privileges. + +**CLI Fallback** (simpler via CLI): +Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" +```bash +oc label node = +``` + +--- + +[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/storage-errors.md new file mode 100644 index 00000000..16ba584d --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/storage-errors.md @@ -0,0 +1,1011 @@ +--- +title: VM Storage Errors +category: kubevirt +sources: + - title: KubeVirt CDI - DataVolumes + url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ + date_accessed: 2026-02-17 + - title: Kubernetes Persistent Volumes + url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ + date_accessed: 2026-02-17 +tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] +semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] +use_cases: [vm-creation, vm-deletion, vm-cloning] +related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] +last_updated: 2026-02-17 +--- + +# VM Storage Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. + +**When to use this document**: +- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` +- Storage deletion fails after VM deletion +- DataVolume cloning operations fail +- PVC provisioning issues + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### ErrorDataVolumeNotReady + +**Symptom**: VM shows status `ErrorDataVolumeNotReady` + +**Description**: The DataVolume (persistent storage) backing the VM is not ready. + +**Possible Causes**: + +#### 1. DataVolume Still Provisioning + +Storage provisioning takes time, especially for large disks or when importing images. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Get detailed DataVolume information**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` and `.status.conditions` for provisioning details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o json +``` + +**3. Check PVC (PersistentVolumeClaim) bound status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Check `.status.phase` for each PVC (should be `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. + +#### 2. Storage Class Not Found + +The requested storage class doesn't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List available storage classes**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" +} +``` + +Review the list of available storage classes (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass +``` + +**2. Check DataVolume's requested storage class**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.spec.pvc.storageClassName` in the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' +``` + +**Solution**: +1. Use a valid storage class from the cluster +2. Recreate VM with correct storage class parameter + +#### 3. Insufficient Storage Quota + +Namespace has insufficient storage quota to provision the PVC. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check resource quotas**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**2. Check storage usage**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase +``` + +**Solution**: +1. Request quota increase from cluster admin +2. Delete unused PVCs to free quota +3. Reduce VM storage size + +--- + +### ErrorPvcNotFound + +**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List PVCs in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review the list of available PVCs (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check VM's PVC references**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' +``` + +**Solution**: +- Wait for DataVolume to create the PVC +- Manually create missing PVC +- Fix VM spec to reference correct PVC name + +--- + + +--- + +### Storage Deletion Failures + +**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace + +**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. + +**Possible Causes**: +- PVC still bound to active PersistentVolume with `Retain` policy +- DataVolume still being referenced by another resource +- CDI (Containerized Data Importer) controller issues +- Storage class retention policy preventing deletion +- Finalizers on PVC/DataVolume blocking cleanup +- PVC still mounted by a pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check PVC status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check specific PVC phase**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Released` or `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o jsonpath='{.status.phase}' +``` + +**3. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**4. Check what's using the PVC**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' +``` + +**5. Check PVC finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. + +**6. Check DataVolume finalizers**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.metadata.finalizers}' +``` + +**7. Check PV reclaim policy**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume" +} +``` + +Filter results for PV where `.spec.claimRef.name` matches ``. + +To get specific PV policy: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume", + "name": "" +} +``` + +Check `.spec.persistentVolumeReclaimPolicy`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pv | grep +oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' +``` + +**Common Finalizer Patterns**: +- `kubernetes.io/pvc-protection` - Protects PVC while in use +- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer + +**Solutions** (Use MCP Tools First): + +1. **Delete DataVolume first, then PVC**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete DataVolume first (often blocks PVC deletion): + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Wait a few seconds, then delete PVC: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete datavolume -n + oc delete pvc -n + ``` + +2. **Check for pods still using PVC**: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "" + } + ``` + + Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. + + Then delete the pods using `pods_delete`: + + **Parameters**: + ```json + { + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' + oc delete pod -n + ``` + +3. **Force delete PVC** (if safe to do so): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete pvc -n --grace-period=0 --force + ``` + +4. **Remove finalizers from PVC** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PVC using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update PVC using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Remove finalizers from DataVolume** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current DataVolume using `resources_get` (diagnostic step 6) + 2. Remove items from `.metadata.finalizers` array + 3. Update DataVolume using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Change PV reclaim policy** (if PV has Retain policy): + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PV policy using `resources_get` (diagnostic step 7) + 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` + 3. Update PV using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" + ```bash + oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' + oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' + ``` + +**Storage Quota Check** (Use MCP Tools First): + +After deletion, verify storage quota is freed: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters for quota check**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.used` to verify storage quota is freed. + +**Parameters for PVC verification**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +oc get pvc -n +``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Confirm PVC is deleted: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Confirm DataVolume is deleted: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +# Should return: Error from server (NotFound) + +oc get datavolume -n +# Should return: Error from server (NotFound) +``` + +--- + +### DataVolume Cloning Failures + +**Symptom**: VM clone created successfully but DataVolume clone operation fails + +**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. + +**Possible Causes**: +- CSI driver doesn't support volume cloning +- Source PVC storage class incompatible with cloning +- Cross-namespace cloning not permitted by storage backend +- Insufficient storage quota in target namespace +- Source PVC not in `Bound` state +- Storage class doesn't have volume cloning enabled +- CDI (Containerized Data Importer) controller issues + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume clone status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and `.metadata.name`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Check DataVolume events for errors**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe datavolume -n +``` + +**3. Check DataVolume phase**: + +Use `resources_get` from step 1, extract `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +``` + +**4. Check if storage class supports cloning**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "name": "" +} +``` + +Review the full YAML output for cloning-related configurations. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass -o yaml | grep -A 5 -i clone +``` + +**5. Check CSI driver capabilities**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "CSIDriver" +} +``` + +Review `.items[].metadata.name` for available CSI drivers. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get csidriver +``` + +**6. Check source PVC status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Bound` for cloning to work). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**7. Check target namespace storage quota**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**8. Check CDI controller logs**: + +**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) + +First, list pods in openshift-cnv namespace: + +**Parameters for pods_list_in_namespace**: +```json +{ + "namespace": "openshift-cnv", + "labelSelector": "app.kubernetes.io/component=cdi-deployment" +} +``` + +Then get logs using `pods_log`: + +**Parameters**: +```json +{ + "namespace": "openshift-cnv", + "name": "", + "tail": 100 +} +``` + +**CLI Fallback** (if MCP unavailable or easier via CLI): +```bash +oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') +``` + +**Common Error Messages**: +- `"volume cloning is not supported"` - CSI driver lacks clone capability +- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage +- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace +- `"insufficient quota"` - Target namespace lacks storage quota +- `"source PVC not bound"` - Source PVC must be in Bound state for cloning +- `"StorageClass does not support cloning"` - Storage class configuration issue + +**Solutions** (Use MCP Tools First): + +1. **Check storage class clone support**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" + } + ``` + + For each storage class in `.items[]`, check: + - `.metadata.name` (storage class name) + - `.provisioner` (CSI driver) + + Storage classes using CSI drivers typically support cloning. Look for provisioners like: + - `csi.ovirt.org` (oVirt CSI) + - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) + - `ebs.csi.aws.com` (AWS EBS CSI) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner + ``` + +2. **Verify source PVC is bound**: + + Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pvc -n -o jsonpath='{.status.phase}' + ``` + +3. **Check target namespace quota**: + + Use `resources_list` from diagnostic step 7 to check quota. + + If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + ``` + +4. **Use snapshot-based cloning** (alternative method): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Step 1**: Create VolumeSnapshot of source PVC + + **Parameters**: + ```json + { + "apiVersion": "snapshot.storage.k8s.io/v1", + "kind": "VolumeSnapshot", + "metadata": { + "name": "-snapshot", + "namespace": "" + }, + "spec": { + "source": { + "persistentVolumeClaimName": "" + } + } + } + ``` + + **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) + + **Step 3**: Create new DataVolume from snapshot + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": "-rootdisk", + "namespace": "" + }, + "spec": { + "source": { + "snapshot": { + "name": "-snapshot", + "namespace": "" + } + }, + "storage": { + "resources": { + "requests": { + "storage": "50Gi" + } + }, + "storageClassName": "" + } + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" + ```bash + cat <-snapshot + namespace: + spec: + source: + persistentVolumeClaimName: + EOF + + oc get volumesnapshot -snapshot -n + + cat <-rootdisk + namespace: + spec: + source: + snapshot: + name: -snapshot + namespace: + storage: + resources: + requests: + storage: 50Gi + storageClassName: + EOF + ``` + +5. **Use "new empty storage" option** (vm-clone skill): + - If cloning isn't supported, create VM with empty storage + - Manually copy data if needed + +6. **Cross-namespace cloning workaround**: + - Some storage backends require snapshot for cross-namespace cloning + - Create snapshot in source namespace, restore in target namespace (see solution 4 above) + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Check DataVolume reached Succeeded phase: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should return `Succeeded`). + +Check PVC was created and bound: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "-rootdisk" +} +``` + +Check `.status.phase` (should return `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +# Should return: Succeeded + +oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' +# Should return: Bound +``` + +**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-delete/environment/mcp-servers/mock-virt-mcp.py b/evaluation/with_skills/rh-virt__vm-delete/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..2aaace7d --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-delete/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1464 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), + ("decommission", {"env": "decommission"}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── decommission (instruction-specific) ────────────────────────────── + _vm("legacy-app", "decommission", "hv-prod-dc1-01", "rhel-8.6", None, + {"app": "legacy-app", "criticality": "low", "legacy": "true"}, + 2, 4, "Running", True, 30), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-virt__vm-delete/environment/skills/vm-delete/SKILL.md b/evaluation/with_skills/rh-virt__vm-delete/environment/skills/vm-delete/SKILL.md new file mode 100644 index 00000000..ee231a9a --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-delete/environment/skills/vm-delete/SKILL.md @@ -0,0 +1,403 @@ +--- +name: vm-delete +description: | + Permanently delete virtual machines and their associated resources from OpenShift Virtualization. + + Use when: + - "Delete VM [name]" + - "Remove virtual machine [name]" + - "Destroy VM [name]" + - "Clean up VM [name]" + + This skill handles permanent VM deletion with strict safety confirmations and typed verification. + + NOT for power management (use vm-lifecycle-manager to stop VMs). + +model: inherit +color: red +--- + +# /vm-delete Skill + +Permanently delete virtual machines and their associated resources (storage, DataVolumes) from OpenShift Virtualization clusters. This skill enforces strict safety protocols including typed confirmation and pre-deletion validation. + +## Prerequisites + +**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools**: +- `resources_get` (from openshift-virtualization) - Verify VM exists and get details +- `resources_delete` (from openshift-virtualization) - Delete Kubernetes resources +- `resources_list` (from openshift-virtualization) - List dependent resources (PVCs, DataVolumes) +- `resources_create_or_update` (from openshift-virtualization) - Update resources (e.g., remove finalizers) +- `vm_lifecycle` (from openshift-virtualization) - Stop running VMs before deletion +- `pods_list_in_namespace` (from openshift-virtualization) - List pods for diagnostics + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster (>= 4.19) +- OpenShift Virtualization operator installed +- ServiceAccount with RBAC permissions to delete VirtualMachine and PVC resources + +### Prerequisite Verification + +**Before executing:** + +1. Check `openshift-virtualization` exists in `.mcp.json` → If missing, report setup +2. Verify `KUBECONFIG` is set (presence only, never expose value) → If missing, report +3. Check RBAC permissions (optional) → Verify delete permissions for VirtualMachine and PVC + +**Human Notification Protocol:** `❌ Cannot execute vm-delete: MCP server not available. Setup: Add to .mcp.json, set KUBECONFIG, restart Claude Code. Docs: https://github.com/openshift/openshift-mcp-server` + +⚠️ **SECURITY**: Never display KUBECONFIG path or credential values. + +## When to Use This Skill + +**Trigger when:** +- User explicitly invokes `/vm-delete` command +- User requests permanent VM deletion +- User wants to clean up test/development VMs +- User needs to free cluster resources +- User wants to decommission VMs + +**User phrases:** +- "Delete VM test-vm in namespace dev" +- "Remove virtual machine web-server" +- "Destroy VM old-database" +- "/vm-delete" + +**Do NOT use when:** +- Stop VM temporarily → `/vm-lifecycle-manager` +- Create VM → `/vm-create` +- View VMs → `/vm-inventory` + +## Workflow + +### Step 1: Gather and Validate + +**CRITICAL**: Complete ALL validation BEFORE user confirmation. + +**Required from user:** VM Name, Namespace + +**1.1: Verify VM Exists** + +**MCP Tool**: `resources_get` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", namespace=``, name=``) + +**Errors:** +- Not found → Report error, suggest vm-inventory +- Permission denied → Report RBAC error + +**1.2: Check Protection Label** + +Check `metadata.labels` for `protected: "true"`. + +**If protected:** Report: `❌ Cannot Delete Protected VM. VM has protected label. Remove: oc label vm -n protected-. Operation cancelled.` **STOP workflow.** + +**1.3: Check Running State** + +Check `status.printableStatus` (Running/Starting/Migrating = running, Stopped/Halted = stopped). + +**If running:** Report: `⚠️ VM Running. Must stop before deletion. Options: stop-and-delete / cancel` **Wait for response.** + +**1.4: Stop VM (if applicable)** + +**ONLY if user chose "stop-and-delete".** + +**MCP Tool**: `vm_lifecycle` (namespace=``, name=``, action="stop") + +Report: `⏸️ Stopping VM... Wait 10-30s.` **Wait 10s**, verify stopped. + +**1.5: Discover Storage** + +**MCP Tool**: `resources_list` + +**DataVolumes**: apiVersion="cdi.kubevirt.io/v1beta1", kind="DataVolume", namespace=``, labelSelector="vm.kubevirt.io/name=``" + +**PVCs** (if no DVs): apiVersion="v1", kind="PersistentVolumeClaim", namespace=``, labelSelector="vm.kubevirt.io/name=``" + +Parse: Extract names, calculate total storage size. + +### Step 2: Present Scope and Get Options + +Display deletion scope in this format: + +```markdown +## ⚠️ VM Deletion - Review Scope + +**VM**: `` | **Namespace**: `` | **Status**: + +**Resources**: VM `` (Age: , vCPU: , Memory: ) +**Storage**: DataVolume `` (30Gi), PVC `` (30Gi) - Total: 30Gi +OR **Storage**: None (ephemeral) + +### Deletion Options +**1: VM Only** - Preserves storage for reuse +**2: VM + Storage** ← Recommended (test/dev) - Frees storage +**3: Cancel** + +Select (1, 2, or 3): +``` + +**Wait for selection.** Handle: 3→Cancel, 1→delete_storage=false, 2→delete_storage=true + +### Step 3: Typed Confirmation (MANDATORY) + +**CRITICAL**: User MUST type exact VM name. + +Display typed confirmation prompt (adjust based on delete_storage flag): + +```markdown +## 🔴 PERMANENT DELETION - Typed Confirmation Required + +**CANNOT BE UNDONE** + +**Will delete**: +✗ VirtualMachine: `` (namespace: ``) +[If delete_storage=true, show:] +✗ DataVolume: `` | ✗ PVC: `` | ✗ All data lost +[If delete_storage=false, show:] +✓ Storage PRESERVED + +Type `` to confirm: _____ +``` + +**Validation:** +- Match → Continue to Step 4 (Execute Deletion) +- Mismatch → Report: `❌ Confirmation Failed. You typed: . Expected: . Cancelled.` **STOP.** + +### Step 4: Execute Deletion + +**ONLY AFTER**: ✓ Validation ✓ Option selected ✓ Typed name confirmed + +**4.1: Delete VM** + +**MCP Tool**: `resources_delete` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", namespace=``, name=``) + +**Errors:** Fails → Report, don't delete storage; Not found → Continue + +Report: `🗑️ Deleting VM... ✓ Deleted` + +**4.2: Delete Storage (if delete_storage=true)** + +**For each DataVolume:** +**MCP Tool**: `resources_delete` (apiVersion="cdi.kubevirt.io/v1beta1", kind="DataVolume", namespace=``, name=``) + +**For each PVC:** +**MCP Tool**: `resources_delete` (apiVersion="v1", kind="PersistentVolumeClaim", namespace=``, name=``) + +**Errors:** Report which failed, continue with others + +Report: `🗑️ Deleting storage... ✓ DV deleted (storage freed) ✓ PVC deleted` + +### Step 5: Report Results + +**Success (with storage):** + +```markdown +## ✓ VM Deleted (Complete Cleanup) +**Deleted**: VM + DataVolume + PVC | **Freed**: +**Impact**: Permanent removal. Cannot recover. +**Verify**: "List VMs in namespace " - VM should not appear +``` + +**Success (storage preserved):** + +```markdown +## ✓ VM Deleted (Storage Preserved) +**Deleted**: VM | **Preserved**: DataVolume + PVC () +**Reuse**: Create new VM with existing DV/PVC +**Delete later**: `oc delete datavolume -n ` +``` + +**Partial failure (storage failed):** + +**OPTIONAL**: Read [storage-errors.md](../../docs/troubleshooting/storage-errors.md) for PVC cleanup. Output: "Consulted storage-errors.md for failure." + +```markdown +## ⚠️ Partial Deletion +**Deleted**: VM | **Failed**: DV/PVC (error: ) +**Action**: Manual cleanup: `oc delete datavolume -n ` +``` + +**Complete failure:** + +**OPTIONAL**: Read [lifecycle-errors.md](../../docs/troubleshooting/lifecycle-errors.md) for deletion failures. Output: "Consulted lifecycle-errors.md for failure causes." + +```markdown +## ❌ VM Deletion Failed +**Error**: +**Troubleshooting**: Check permissions, verify VM exists, check finalizers (see lifecycle-errors.md) +``` + +## Common Issues + +### Issue 1: VM Not Found +**Error**: "VirtualMachine not found" +**Solution**: Verify name/namespace with vm-inventory. Check spelling. + +### Issue 2: RBAC Permissions +**Error**: "Forbidden: Cannot delete VirtualMachines" +**Solution**: Verify delete permissions for VirtualMachine and PVC. Contact admin. Check: `oc auth can-i delete virtualmachines -n ` + +### Issue 3: VM Has Finalizers +**Error**: "VM deletion blocked by finalizers" +**Solution**: Consult [lifecycle-errors.md](../../docs/troubleshooting/lifecycle-errors.md) "VM Stuck in Terminating" for MCP-first approach using `resources_get` to check finalizers, `resources_create_or_update` to remove if needed. + +### Issue 4: Storage Deletion Failure +**Error**: "PVC deletion failed: resource in use" +**Solution**: Verify VM deleted first. Consult [storage-errors.md](../../docs/troubleshooting/storage-errors.md) for MCP-first diagnostics using `pods_list_in_namespace` to check mounts, `resources_get` for PVC status. + +### Issue 5: Confirmation Mismatch +**Error**: "Names do not match" +**Solution**: Type exact VM name (case-sensitive). Copy-paste from deletion scope. Retry. + +### Issue 6: Protected VM +**Error**: "VM has protected label" +**Solution**: Remove: `oc label vm -n protected-`. Retry deletion. + +## Dependencies + +### Required MCP Servers +- `openshift-virtualization` - OpenShift MCP with KubeVirt toolset (https://github.com/openshift/openshift-mcp-server) + +### Required MCP Tools +- `resources_get` - Get VM (apiVersion, kind, namespace, name) +- `resources_delete` - Delete resources (apiVersion, kind, namespace, name) +- `resources_list` - List resources (apiVersion, kind, namespace, labelSelector) +- `resources_create_or_update` - Update resources (resource JSON) - for finalizer removal +- `vm_lifecycle` - VM lifecycle (namespace, name, action: stop) +- `pods_list_in_namespace` - List pods (namespace) - for PVC mount diagnostics + +### Related Skills +- `vm-lifecycle-manager` - Stop VMs | `vm-inventory` - List VMs | `vm-create` - Create VMs | `vm-clone` - Clone VMs + +### Reference Documentation +- [lifecycle-errors.md](../../docs/troubleshooting/lifecycle-errors.md) - Deletion failures, finalizers, stuck Terminating (consulted on deletion failure) +- [storage-errors.md](../../docs/troubleshooting/storage-errors.md) - Storage deletion, PVC cleanup (consulted on storage failure) +- [Troubleshooting INDEX](../../docs/troubleshooting/INDEX.md) - Full error index +- [OpenShift Virt Docs](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index) +- [KubeVirt API](https://kubevirt.io/api-reference/) +- [K8s Finalizers](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/) + +## Critical: Human-in-the-Loop Requirements + +**CRITICAL: IRREVERSIBLE operations.** You MUST: + +1. **Pre-Deletion Validation** (Execute FIRST) + - Verify VM exists (`resources_get`) + - Check VM running state + - Discover dependent resources + - Check protection labels (`protected: "true"`) + +2. **Safety Checks** + - **REFUSE deletion** if protected label exists + - **REQUIRE VM stopped** if running + - **List all resources** to be deleted + +3. **Typed Confirmation (MANDATORY)** + - Display complete deletion scope + - **Require exact VM name** to confirm + - Accept only exact match (case-sensitive) + - Mismatch → Cancel, don't proceed + - Match → Proceed directly to deletion + +4. **Deletion Options** + - Ask: VM only / VM+Storage / Cancel + - **NEVER assume** which option + +5. **Never Auto-Execute** + - NEVER delete without typed confirmation + - NEVER proceed if typed name mismatches + - NEVER skip typed verification + +**Why**: Permanent data loss, service impact, accidental deletion prevention + +## Security Considerations + +- **RBAC**: Requires delete VirtualMachine, PVC, DataVolume +- **Typed Verification**: Prevents accidental deletion +- **Protection Labels**: `protected: "true"` blocks deletion +- **Audit**: Kubernetes audit logs +- **Graceful Shutdown**: Stops VMs before deletion +- **KUBECONFIG**: Never exposed +- **Namespace Isolation**: RBAC-enforced +- **No Force Delete**: Never uses `--force` or `--grace-period=0` + +## Example Usage + +### Example 1: Complete Deletion (VM + Storage) + +``` +User: "Delete test-vm in dev" +Agent: [Validates ✓, discovers 30Gi storage] + [Shows deletion scope - see Step 2 format] +User: "2" [VM+Storage] +Agent: [Shows typed confirmation - see Step 3 format] +User: "test-vm" +Agent: [Executes: VM ✓, DV ✓, PVC ✓] + [Shows success report - see Step 5 format] +## ✓ VM Deleted (Complete Cleanup) | Freed: 30Gi +``` + +### Example 2: VM Only (Preserve Storage) + +``` +User: "Remove web-server but keep storage" +Agent: [Validates, shows scope] +User: "1" [VM only] +Agent: [Typed confirmation] +User: "web-server" +Agent: [Executes deletion] +## ✓ VM Deleted (Storage Preserved) | Preserved: 50Gi +``` + +### Example 3: Protected VM (Blocked) + +``` +User: "Delete production-db" +Agent: [Validates: protected label found] +❌ Cannot Delete Protected VM. Remove label: oc label vm production-db protected- +``` + +### Example 4: Running VM (Stop First) + +``` +User: "Delete api-server" +Agent: [Validates: VM running] +⚠️ VM Running. Options: stop-and-delete / cancel +User: "stop-and-delete" +Agent: [Stops VM, waits, continues with deletion] +``` + +### Example 5: User Cancels + +``` +User: "Delete test-vm" +Agent: [Shows typed confirmation] +Type `test-vm` to confirm: +User: "wait, cancel" +❌ Confirmation Failed. You typed: wait, cancel. Expected: test-vm. Cancelled. +``` + +### Example 6: Dry-Run Preview + +``` +User: "What would be deleted if I delete test-vm?" +Agent: [Execute Step 1-2 only, stop before confirmation] +## 🔍 Deletion Preview +**Would delete (Option 2)**: VM + DV + PVC (30Gi freed) +This is preview only. No resources deleted. +``` + +## Advanced Features + +### Batch Deletion +Delete multiple VMs with confirmation for each: `"Delete VMs test-01, test-02, test-03 in dev"` → Process each individually with full workflow. Use typed confirmation: `DELETE-3-VMS` for batch. + +### Dry-Run Mode +Show deletion scope without executing: Execute Step 1-2, skip Steps 3-4. User request: "Show what would be deleted if I delete VM xyz" + +### Protected VM Label +Automatic enforcement: If VM has `protected: "true"` label, refuse deletion in Step 1.2. Example YAML: `metadata.labels.protected: "true"` diff --git a/evaluation/with_skills/rh-virt__vm-delete/instruction.md b/evaluation/with_skills/rh-virt__vm-delete/instruction.md new file mode 100644 index 00000000..5769196b --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-delete/instruction.md @@ -0,0 +1,12 @@ +# VM Deletion Task + +You are an OpenShift Virtualization administrator. Plan the safe deletion of VM `legacy-app` in namespace `decommission`. + +## Requirements +- Perform pre-deletion safety checks +- Define the deletion scope (VM only vs VM + storage) +- Include safeguards against accidental deletion + +Use MCP tools to examine the cluster. Document your methodology, findings, and deletion plan in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-virt__vm-delete/solution/solve.sh b/evaluation/with_skills/rh-virt__vm-delete/solution/solve.sh new file mode 100644 index 00000000..6d87b29d --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-delete/solution/solve.sh @@ -0,0 +1,31 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Deletion Plan + +## Target: legacy-app in decommission + +### Pre-Deletion Safety Checks +1. **Protection label**: Check `metadata.labels.protected` — if `"true"`, deletion is blocked. Remove with `oc label vm legacy-app -n decommission protected-` +2. **Running state**: If VM is running, stop it first via `vm_lifecycle` action=stop +3. **Storage discovery**: List DataVolumes with label `vm.kubevirt.io/name=legacy-app` + +### Deletion Scope Options +- **VM Only** — Keep associated storage (DataVolumes/PVCs) for data recovery +- **VM + Storage** (selected) — Full cleanup of VM and all associated DataVolumes/PVCs + +### Deletion Procedure +1. Verify VM exists and is stopped (use vm_lifecycle action=stop if running) +2. List all associated DataVolumes (apiVersion: cdi.kubevirt.io/v1beta1, labelSelector: vm.kubevirt.io/name=legacy-app) +3. Present deletion scope and storage list +4. **Typed confirmation required**: User must type exact VM name `legacy-app` to proceed +5. Delete VM via resources_delete +6. Delete associated DataVolumes and PVCs via resources_delete +7. Verify deletion completed (resource no longer exists) +8. If VM stuck Terminating: consult lifecycle-errors.md, check finalizers + +### Post-Deletion Verification +- Confirm VM resource is gone +- Confirm DataVolumes and PVCs are cleaned up +- Check for any orphaned resources (finalizers) + +REPORT_EOF diff --git a/evaluation/with_skills/rh-virt__vm-delete/task.toml b/evaluation/with_skills/rh-virt__vm-delete/task.toml new file mode 100644 index 00000000..063c79fd --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-delete/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-delete" +name = "rh-virt VM Deletion Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-delete", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-virt__vm-delete/tests/llm_judge.py b/evaluation/with_skills/rh-virt__vm-delete/tests/llm_judge.py new file mode 100644 index 00000000..e1bed079 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-delete/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "protected_label", "file": "/root/report.md", "question": "Does the report check for protected: true label that blocks deletion?", "reference": "A skilled report checks protection labels. An unskilled report attempts deletion without safety checks."}, + {"id": "storage_scope", "file": "/root/report.md", "question": "Does the report distinguish VM-only vs VM+storage deletion and warn about orphaned PVCs?", "reference": "A skilled report offers storage scope choice. An unskilled report deletes everything without distinction."}, + {"id": "typed_confirmation", "file": "/root/report.md", "question": "Does the report require typed VM name confirmation (exact, case-sensitive) before deletion?", "reference": "A skilled report requires exact name match confirmation. An unskilled report uses yes/no confirmation."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-virt__vm-delete/tests/test.sh b/evaluation/with_skills/rh-virt__vm-delete/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-delete/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-virt__vm-delete/tests/test_outputs.py b/evaluation/with_skills/rh-virt__vm-delete/tests/test_outputs.py new file mode 100644 index 00000000..a1c73806 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-delete/tests/test_outputs.py @@ -0,0 +1,82 @@ +""" +Tests for rh-virt__vm-delete per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_vm(self): + content = read_report().lower() + assert any(t in content for t in ["vm", "virtual machine", "virtualmachine"]), ( + "report should reference the target VM" + ) + + def test_mentions_deletion(self): + content = read_report().lower() + assert "delet" in content, "report should discuss deletion" + + +class TestSkillDependent: + def test_stop_before_delete(self): + """Skill: Must stop VM before deletion; vm_lifecycle stop.""" + c = read_report().lower() + assert any(t in c for t in ["stop before delet", "stop and delete", "vm_lifecycle", "halt", "must stop", "running"]) and ( + "stop" in c or "halt" in c + ), ( + "should require stopping VM before deletion" + ) + + def test_orphan_storage(self): + """Skill: VM-only vs VM+storage; orphan PVCs; delete DataVolume/PVC.""" + c = read_report().lower() + assert any(t in c for t in ["vm only", "vm+storage", "datavolume", "orphan", "preserve storage", "delete storage", "pvc"]) and ( + "storage" in c or "pvc" in c or "datavolume" in c + ), ( + "should address storage scope (VM-only vs VM+storage, orphan PVCs)" + ) + + def test_finalizer_handling(self): + """Skill: Finalizer blocking deletion; stuck Terminating.""" + c = read_report().lower() + assert any(t in c for t in ["finalizer", "terminating", "stuck", "resources_create_or_update", "remove finalizer"]), ( + "should address finalizer handling for stuck deletion" + ) + + def test_typed_confirmation(self): + """Skill: Typed VM name confirmation (exact match) before delete.""" + c = read_report().lower() + assert any(t in c for t in ["type", "typed", "exact name", "confirm", "to confirm"]) and ( + "name" in c or "vm" in c + ), ( + "should require typed VM name confirmation" + ) + + def test_protected_label(self): + """Skill: protected: true label blocks deletion.""" + c = read_report().lower() + assert any(t in c for t in ["protected", "protected label", "metadata.labels", "refuse delet"]), ( + "should address protected label blocking deletion" + ) + + def test_reclaim_policy_retain(self): + """Docs teach PV reclaim policy Retain blocks PVC deletion; must patch PV + to Delete first. Without docs, agents don't handle stuck PVC cleanup.""" + c = read_report().lower() + assert any(t in c for t in [ + "retain", "reclaim", "reclaimpolicy", "reclaim policy", + "patch pv", "delete policy", + ]), "should address PV reclaim policy Retain blocking cleanup" diff --git a/evaluation/with_skills/rh-virt__vm-inventory/environment/Dockerfile b/evaluation/with_skills/rh-virt__vm-inventory/environment/Dockerfile new file mode 100644 index 00000000..ae625e01 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-inventory/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/.ai-index/semantic-index.json new file mode 100644 index 00000000..2512c2a3 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/.ai-index/semantic-index.json @@ -0,0 +1,148 @@ +{ + "index_version": "1.0.0", + "last_updated": "2026-02-17", + "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", + + "documents": [ + { + "file": "INDEX.md", + "title": "VM Troubleshooting Guide - Index", + "category": "navigation", + "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], + "use_cases": ["navigation", "tool-reference", "general-guidance"], + "token_estimate": 450, + "skills": ["all"] + }, + { + "file": "scheduling-errors.md", + "title": "VM Scheduling Errors", + "category": "scheduling", + "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], + "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], + "errors_covered": [ + "ErrorUnschedulable - Node Taints", + "ErrorUnschedulable - Insufficient Resources", + "ErrorUnschedulable - Node Selector Mismatch" + ], + "token_estimate": 820, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "storage-errors.md", + "title": "VM Storage Errors", + "category": "storage", + "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], + "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], + "errors_covered": [ + "ErrorDataVolumeNotReady - DataVolume Still Provisioning", + "ErrorDataVolumeNotReady - Storage Class Not Found", + "ErrorDataVolumeNotReady - Insufficient Storage Quota", + "ErrorPvcNotFound", + "Storage Deletion Failures", + "DataVolume Cloning Failures" + ], + "token_estimate": 1350, + "skills": ["vm-creator", "vm-delete", "vm-clone"] + }, + { + "file": "lifecycle-errors.md", + "title": "VM Lifecycle Errors", + "category": "lifecycle", + "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], + "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], + "errors_covered": [ + "VM Stuck in Terminating State", + "VM Won't Start (Non-Scheduling Issues)", + "VM Won't Stop" + ], + "token_estimate": 1150, + "skills": ["vm-delete", "vm-lifecycle-manager"] + }, + { + "file": "runtime-errors.md", + "title": "VM Runtime Errors", + "category": "runtime", + "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], + "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], + "errors_covered": [ + "CrashLoopBackOff" + ], + "token_estimate": 850, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "network-errors.md", + "title": "VM Network Errors", + "category": "networking", + "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], + "use_cases": ["vm-creation", "network-troubleshooting"], + "errors_covered": [ + "Network Attachment Failures" + ], + "token_estimate": 580, + "skills": ["vm-creator"] + } + ], + + "skill_to_docs_mapping": { + "vm-creator": { + "primary_docs": ["scheduling-errors.md", "storage-errors.md"], + "secondary_docs": ["runtime-errors.md", "network-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3650, + "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" + }, + "vm-delete": { + "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 2950, + "description": "VM deletion failures - stuck terminating, storage cleanup" + }, + "vm-clone": { + "primary_docs": ["storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 1800, + "description": "VM cloning failures - DataVolume cloning, storage provisioning" + }, + "vm-lifecycle-manager": { + "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], + "secondary_docs": ["runtime-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3270, + "description": "VM start/stop failures - lifecycle management, scheduling" + }, + "vm-inventory": { + "primary_docs": [], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 450, + "description": "General guidance for status interpretation" + } + }, + + "error_to_docs_mapping": { + "ErrorUnschedulable": "scheduling-errors.md", + "ErrorDataVolumeNotReady": "storage-errors.md", + "ErrorPvcNotFound": "storage-errors.md", + "Terminating": "lifecycle-errors.md", + "CrashLoopBackOff": "runtime-errors.md", + "NetworkAttachmentFailed": "network-errors.md" + }, + + "cross_references": { + "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], + "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], + "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], + "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], + "network-errors.md": ["scheduling-errors.md"] + }, + + "progressive_disclosure_strategy": { + "description": "Load only the minimum documentation needed for current task", + "pattern": "error_type → primary_doc → cross_references (if needed)", + "token_savings": "29% average reduction vs single-file (based on rh-sre data)", + "navigation_overhead_reduction": "85% vs single-file navigation" + } +} diff --git a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/INDEX.md new file mode 100644 index 00000000..a2e76963 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/INDEX.md @@ -0,0 +1,332 @@ +--- +title: VM Troubleshooting Guide - Index +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 + - title: OpenShift Virtualization - Virtual Machine Status + url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html + date_accessed: 2026-02-06 + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] +semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] +use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] +last_updated: 2026-02-17 +--- + +# VM Troubleshooting Guide - Index + +## Overview + +This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: + +- **Creation**: Scheduling failures, storage provisioning issues +- **Lifecycle**: Start/stop failures, stuck states +- **Deletion**: Resources stuck in Terminating, storage cleanup failures +- **Cloning**: DataVolume cloning errors, cross-namespace issues +- **Networking**: Secondary network attachment failures +- **Runtime**: CrashLoopBackOff, guest OS boot failures + +This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. + +--- + +## 🗂️ Troubleshooting Categories + +The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: + +### 1. [Scheduling Errors](scheduling-errors.md) +**When to use**: VM fails to schedule on any node + +**Errors covered**: +- ErrorUnschedulable - Node Taints +- ErrorUnschedulable - Insufficient Resources +- ErrorUnschedulable - Node Selector Mismatch + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 2. [Storage Errors](storage-errors.md) +**When to use**: VM has storage provisioning, deletion, or cloning issues + +**Errors covered**: +- ErrorDataVolumeNotReady (all 3 subsections) +- ErrorPvcNotFound +- Storage Deletion Failures +- DataVolume Cloning Failures + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### 3. [Lifecycle Errors](lifecycle-errors.md) +**When to use**: VM has start/stop/termination issues + +**Errors covered**: +- VM Stuck in Terminating State +- VM Won't Start (Non-Scheduling Issues) +- VM Won't Stop + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### 4. [Runtime Errors](runtime-errors.md) +**When to use**: VM repeatedly crashes or fails at runtime + +**Errors covered**: +- CrashLoopBackOff + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 5. [Network Errors](network-errors.md) +**When to use**: VM has secondary network attachment failures + +**Errors covered**: +- Network Attachment Failures (Multus, SR-IOV) + +**Skills that use this**: vm-create + +--- + +## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last + +### Critical Principle: MCP-First Approach + +**All diagnostic procedures in this guide follow the MCP-first pattern**: + +``` +1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) +2. ⚠️ IF FAILS: Ask user permission to use CLI command +3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval +``` + +### Why MCP Tools First? + +- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources +- **Consistency**: Same interface across all operations +- **Better Error Handling**: MCP tools return structured errors +- **Audit Trail**: MCP tool usage is logged and trackable + +### Command Pattern Examples + +Throughout this guide, you'll see diagnostic steps formatted like this: + +**✅ CORRECT Pattern (MCP First)**: + +```markdown +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Extract**: `.metadata.finalizers` from returned JSON + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` +``` + +### Available MCP Tools + +The `openshift-virtualization` MCP server provides these tools: + +**Resource Operations**: +- `resources_get` - Get specific resource (replaces `oc get `) +- `resources_list` - List resources (replaces `oc get `) +- `resources_delete` - Delete resource (replaces `oc delete`) +- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) + +**Pod Operations**: +- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) +- `pods_get` - Get pod details (replaces `oc get pod`) +- `pods_log` - Get pod logs (replaces `oc logs`) +- `pods_exec` - Execute in pod (replaces `oc exec`) +- `pods_delete` - Delete pod (replaces `oc delete pod`) +- `pods_top` - Pod resource usage (replaces `oc top pods`) + +**Events & Monitoring**: +- `events_list` - List events (replaces `oc get events`) +- `nodes_top` - Node resource usage (replaces `oc top nodes`) +- `nodes_log` - Node logs (replaces `oc adm node-logs`) +- `nodes_stats_summary` - Detailed node stats + +**VM Operations** (KubeVirt toolset): +- `vm_create` - Create VMs +- `vm_lifecycle` - Start/stop/restart VMs + +**Namespaces**: +- `namespaces_list` - List namespaces (replaces `oc get namespaces`) +- `projects_list` - List OpenShift projects (replaces `oc get projects`) + +### When CLI Commands Are Required + +Some operations have **NO MCP equivalent** and require CLI: + +- `virtctl` commands (console, VNC access) +- `oc debug node` (node debugging) +- `oc auth can-i` (permission checks) +- `oc adm taint` (node taint management) +- Complex JSON patch operations + +For these, the guide will note: **"CLI Only - No MCP equivalent"** + +### Quick Reference: CLI → MCP Mapping + +| CLI Command | MCP Tool Equivalent | +|-------------|---------------------| +| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | +| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | +| `oc logs -n ` | `pods_log` with name="", namespace="" | +| `oc get events -n ` | `events_list` with namespace="" | +| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | + +**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. + +### How to Read Diagnostic Sections + +Each error section includes: +1. **Symptom** - What you observe +2. **Description** - What's happening +3. **Possible Causes** - Why it's happening +4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback +5. **Solutions** - **MCP tools first**, then CLI fallback +6. **Verification** - **MCP tools first**, then CLI fallback + +**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. + +--- + +## 🔍 Quick Navigation by Skill + +**vm-create**: +- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics +- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning +- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff +- [Network Errors](network-errors.md) - Network attachment failures + +**vm-delete**: +- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state +- [Storage Errors](storage-errors.md) - Storage deletion failures + +**vm-clone**: +- [Storage Errors](storage-errors.md) - DataVolume cloning failures + +**vm-lifecycle-manager**: +- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop +- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling + +**vm-inventory**: +- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed + +--- + +## 📊 Documentation Coverage & Maintenance + +### Current Coverage + +This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: + +- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) +- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures +- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop +- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) +- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems + +**Total errors documented**: 12 error types across 6 categories + +--- + +### Encountering Undocumented Errors + +#### For AI Agents (Claude Code) + +If you encounter an error **not documented** in the categories above: + +1. **Report to user** with all available details (error message, affected resources, namespace) +2. **Provide best-effort diagnostics** using MCP tools: + - `resources_get` to inspect resource status + - `pods_log` to check virt-launcher or compute container logs + - `events_list` to view Kubernetes events +3. **Suggest documentation update**: + ``` + ⚠️ This error is not yet documented in the troubleshooting guide. + + **Error**: + **Affected resource**: / + + I recommend adding this error to the troubleshooting guide: + - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file + - If it's a new error class → Create new category file in docs/troubleshooting/ + + Would you like me to help document this error for future reference? + ``` + +4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs + +--- + +#### For Human Contributors + +To document a new error: + +1. **Determine the category**: + - Scheduling issue → `scheduling-errors.md` + - Storage problem → `storage-errors.md` + - Lifecycle/start/stop → `lifecycle-errors.md` + - Runtime crash → `runtime-errors.md` + - Network issue → `network-errors.md` + - New error class → Create new category file (e.g., `security-errors.md`) + +2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): + - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback + - **Solutions**: MCP Tool approach → Implementation → CLI Fallback + - **Verification**: MCP Tool checks → Expected results → CLI Fallback + +3. **Update semantic index**: + - Edit `.ai-index/semantic-index.json` + - Add error to `error_to_docs_mapping` + - Update relevant skill's `primary_docs` or `secondary_docs` + - Increment token estimates if needed + +4. **Reference in skill files**: + - Update skill's Reference Documentation section + - Ensure skill's Document Consultation steps point to the right category + +5. **Update this INDEX.md**: + - Add error to the appropriate category section (lines 50-120) + - Update "Current Coverage" list above + +--- + +## 📚 Additional Resources + +- [rh-virt README](../../README.md) - MCP server setup and skill documentation +- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources +- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/lifecycle-errors.md new file mode 100644 index 00000000..6167fcc8 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/lifecycle-errors.md @@ -0,0 +1,869 @@ +--- +title: VM Lifecycle Errors +category: kubevirt +sources: + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 +tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] +semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] +use_cases: [vm-deletion, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Lifecycle Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM lifecycle issues including start/stop failures and stuck termination states. + +**When to use this document**: +- VM stuck in `Terminating` state +- VM won't start (runStrategy is Always but VM never reaches Running) +- VM won't stop (runStrategy is Halted but VM never reaches Stopped) + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### VM Stuck in Terminating State + +**Symptom**: VM shows status `Terminating` but deletion never completes + +**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). + +**Possible Causes**: +- Finalizers blocking deletion +- PVC/DataVolume still attached and preventing cleanup +- VirtualMachineInstance (VMI) not terminating properly +- Custom controllers or operators blocking deletion +- Stuck virt-launcher pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check finalizers on the VM**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` + +**2. Check if VMI still exists**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name containing "virt-launcher-". + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check events for deletion issues**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results where `involvedObject.name` == "" and sort by timestamp. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' +``` + +**5. Check VM deletion timestamp**: + +Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. + +If present, VM is in deletion process. If null, VM is not being deleted. + +**Common Finalizer Patterns**: +- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer +- `foregroundDeletion` - Waits for dependent objects to be deleted +- Custom finalizers from operators + +**Solutions** (Use MCP Tools First): + +1. **Wait for dependent resources** (safest approach): + - Use `resources_list` to check if PVCs, DataVolumes are still deleting + - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) + +2. **Force delete VMI** (if VMI is stuck): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable - requires explicit user permission): + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +3. **Force delete virt-launcher pod** (if pod is stuck): + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + # Find the virt-launcher pod name first + oc get pods -n | grep virt-launcher- + + # Then delete it + oc delete pod virt-launcher--xxx -n --force --grace-period=0 + ``` + +4. **Remove finalizers** (⚠️ dangerous - use only as last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` + 2. Remove items from `.metadata.finalizers` array + 3. Update VM using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch not easily done via MCP): + ```bash + # This operation is complex for MCP - may need CLI + oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Check for protection labels** (vm-delete skill specific): + + Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. + + If value is "true", the vm-delete skill refuses deletion (this is expected behavior). + +**Verification**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error if deletion successful. + +**CLI Fallback**: +```bash +oc get vm -n +# Should return: Error from server (NotFound) +``` + +--- + + +--- + +### VM Won't Start (Non-Scheduling Issues) + +**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state + +**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. + +**Possible Causes**: +- Guest OS kernel panic or boot failure +- Cloud-init configuration errors +- Missing or corrupted disk image +- Insufficient memory for guest OS to boot +- QEMU/KVM virtualization errors +- VirtualMachineInstance (VMI) creation failures +- virt-launcher pod crashes + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VMI (VirtualMachineInstance) status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and overall VMI state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**2. Check VMI conditions for errors**: + +Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. View virt-launcher pod logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +First, get pod name from step 3, then: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +Look for QEMU errors, memory allocation failures, device errors. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**5. Check virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. + +Alternatively, use `pods_get` to get full pod details: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**6. Access VM console to see guest OS boot messages**: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +Look for kernel panic, initramfs errors, filesystem errors. + +**7. Check VM events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep -A 20 "Events:" +``` + +**Common Error Patterns**: + +1. **Guest OS Boot Failure**: + - Console shows kernel panic + - Guest hangs at GRUB or boot loader + - Cloud-init errors during first boot + +2. **Insufficient Memory**: + - Guest OS kills processes due to OOM + - VMI logs show memory allocation errors + +3. **QEMU Crashes**: + - virt-launcher logs show QEMU segfaults + - VMI repeatedly restarts + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for kernel panic, initramfs errors, filesystem errors. + +2. **Check virt-launcher pod logs for QEMU errors**: + + Use `pods_log` from diagnostic step 4 to view logs. + + Look for: + - "qemu-system-x86_64: ..." errors + - Memory allocation failures + - Device errors + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx + ``` + +3. **Increase memory if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check VMI memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +4. **Verify disk image integrity**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check PVC is bound: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + If using container disk, verify image exists and is accessible (check VMI spec). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get pvc -n + ``` + +5. **Check cloud-init configuration** (if applicable): + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. + + Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq + ``` + +6. **Restart VMI** (soft reset): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete VMI (VM controller will recreate it): + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for new VMI to start (use `resources_get` to check status). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n -w + ``` + +7. **Check virtualization extensions** (KVM): + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + + # In debug shell: + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should eventually return `Running`). + +Check VMI is running: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should show `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should eventually return: Running + +oc get vmi -n +# Should show: Running +``` + +--- + +### VM Won't Stop + +**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state + +**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. + +**Possible Causes**: +- Guest OS not responding to ACPI shutdown signal +- virt-launcher pod stuck and not terminating +- VirtualMachineInstance (VMI) deletion blocked by finalizers +- Guest shutdown scripts hanging +- Filesystem sync issues in guest OS + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (might show `Stopping`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**2. Check VMI status and deletion timestamp**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check VMI events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vmi -n | grep -A 10 "Events:" +``` + +**5. Check VMI finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.metadata.finalizers}' +``` + +**6. Check if guest is responsive** (if VMI still exists): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Patterns**: +- VMI shows `deletionTimestamp` but never actually deletes +- virt-launcher pod in `Terminating` state +- VM runStrategy is `Halted` but printableStatus shows `Stopping` + +**Solutions** (Use MCP Tools First): + +1. **Wait for graceful shutdown** (default: 30 seconds): + + Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. + +2. **Force stop by deleting VMI**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + This is the standard way to force-stop a VM. + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n + # Should return: Error from server (NotFound) + ``` + +3. **Force delete VMI with grace period** (if VMI won't delete): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +4. **Force delete virt-launcher pod**: + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find the pod using `pods_list_in_namespace` from diagnostic step 3. + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + ⚠️ **Note**: For force deletion with grace period, use CLI fallback. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') + oc delete pod $POD_NAME -n --force --grace-period=0 + ``` + +5. **Remove VMI finalizers** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VMI using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update VMI using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Patch VM runStrategy directly** (ensure consistency): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` (diagnostic step 1) + 2. Set `.spec.runStrategy` to `"Halted"` + 3. Update VM using `resources_create_or_update` with modified JSON + + **CLI Fallback** (merge patch easier via CLI): + Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Stopped` or `Halted`). + +Verify VMI is gone: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Verify virt-launcher pod is gone: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for pods matching `virt-launcher-`. Should return no results. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Stopped or Halted + +oc get vmi -n +# Should return: Error from server (NotFound) + +oc get pods -n | grep virt-launcher- +# Should return: No resources found +``` + +**Prevention**: +- Ensure guest OS has ACPI support enabled +- Use proper shutdown commands in guest OS +- Avoid forceful stops unless necessary (can corrupt guest filesystem) + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/network-errors.md new file mode 100644 index 00000000..82fa313c --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/network-errors.md @@ -0,0 +1,429 @@ +--- +title: VM Network Errors +category: kubevirt +sources: + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] +semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] +use_cases: [vm-creation, network-troubleshooting] +related_docs: [INDEX.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Network Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. + +**When to use this document**: +- VM created successfully but secondary networks not attached +- NetworkAttachmentDefinition not found errors +- Multus CNI failures +- SR-IOV device attachment issues + +**Skills that use this**: vm-create + +--- + +### Network Attachment Failures + +**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working + +**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). + +**Possible Causes**: +- NetworkAttachmentDefinition doesn't exist in the namespace +- Multus CNI not installed or not configured on cluster +- Namespace mismatch (NAD in different namespace than VM) +- Interface name conflicts in VM spec +- Bridge/network configuration errors in NAD +- SR-IOV device not available (if using SR-IOV) + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List NetworkAttachmentDefinitions in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "" +} +``` + +Review `.items[].metadata.name` for available NADs. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -n +``` + +**2. List NetworkAttachmentDefinitions in all namespaces**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition" +} +``` + +Omit `namespace` parameter to list across all namespaces. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -A +``` + +**3. Check specific NetworkAttachmentDefinition**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "", + "name": "" +} +``` + +Review `.spec.config` for CNI configuration. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definition -n -o yaml +``` + +**4. Check VM network configuration**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.networks` to see network references. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq +``` + +**5. Check VM domain interfaces**: + +Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq +``` + +**6. Check virt-launcher pod network annotations** (shows actual attachments): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**7. Check Multus is installed**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "openshift-multus" +} +``` + +Should show Multus CNI pods running. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n openshift-multus +``` + +**8. Check for errors in virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n +``` + +**Common Error Messages**: +- `"network-attachment-definition not found"` - NAD doesn't exist in namespace +- `"multus CNI not configured"` - Multus not installed or misconfigured +- `"interface name conflict"` - Duplicate interface names in VM spec +- `"failed to add network"` - CNI plugin error (check NAD config) +- `"no available devices"` - SR-IOV device not available (if using SR-IOV) + +**Solutions** (Use MCP Tools First): + +1. **Verify NetworkAttachmentDefinition exists in correct namespace**: + + Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. + + If NAD is in different namespace, copy it to VM namespace: + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get NAD from source namespace using `resources_get` + 2. Modify `.metadata.namespace` to target namespace + 3. Create NAD in target namespace using `resources_create_or_update` + + **CLI Fallback** (stream processing easier via CLI): + Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" + ```bash + oc get network-attachment-definition -n -o yaml | \ + sed "s/namespace: /namespace: /" | \ + oc apply -f - + ``` + +2. **Create missing NetworkAttachmentDefinition**: + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + Example: Linux bridge network + + **Parameters**: + ```json + { + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": { + "name": "vlan100", + "namespace": "" + }, + "spec": { + "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" + ```bash + cat < + spec: + config: '{ + "cniVersion": "0.3.1", + "type": "bridge", + "bridge": "br1", + "vlan": 100, + "ipam": { + "type": "host-local", + "subnet": "192.168.100.0/24" + } + }' + EOF + ``` + +3. **Check Multus CNI installation**: + + Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. + + To check cluster network operator: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterOperator" + } + ``` + + Filter for `network` operator. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n openshift-multus + oc get clusteroperators network + ``` + +4. **Fix interface name conflicts** (if VM has duplicate names): + + Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. + + Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. + + **CLI Fallback** (interactive edit easier via CLI): + Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' + oc edit vm -n + ``` + +5. **Validate NAD configuration syntax**: + + Use `resources_get` from diagnostic step 3, extract `.spec.config`. + + Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get network-attachment-definition -n -o jsonpath='{.spec.config}' + ``` + +6. **Check SR-IOV device availability** (if using SR-IOV networks): + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + List SR-IOV network node policies: + + **Parameters**: + ```json + { + "apiVersion": "sriovnetwork.openshift.io/v1", + "kind": "SriovNetworkNodePolicy", + "namespace": "openshift-sriov-network-operator" + } + ``` + + Check SR-IOV device plugin pods: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "openshift-sriov-network-operator" + } + ``` + + Filter for pods with "device-plugin" in name. + + Check available SR-IOV devices on node: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "Node", + "name": "" + } + ``` + + Review `.status.allocatable` for SR-IOV resources. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get sriovnetworknodepolicy -n openshift-sriov-network-operator + oc get pods -n openshift-sriov-network-operator | grep device-plugin + oc describe node | grep -A 10 "Allocatable:" + ``` + +7. **Recreate VM with corrected network configuration** (if needed): + + If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +After remediation, check virt-launcher pod network status: + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +Should show all attached networks with status. Example output: +```json +[ + { + "name": "openshift-sdn", + "interface": "eth0", + "ips": ["10.128.2.10"], + "default": true + }, + { + "name": "vlan100", + "interface": "net1", + "ips": ["192.168.100.5"] + } +] +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**Check from inside VM** (via console): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# In guest OS: +ip addr show +# Should show all network interfaces (eth0, net1, etc.) +``` + +**Common Network Types**: +- **Linux Bridge**: Layer 2 bridge for VLAN networks +- **SR-IOV**: High-performance direct device assignment +- **macvlan**: MAC-based VLAN for container networks +- **OVN-Kubernetes**: OpenShift native overlay network + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/runtime-errors.md new file mode 100644 index 00000000..804d3cbc --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/runtime-errors.md @@ -0,0 +1,616 @@ +--- +title: VM Runtime Errors +category: kubevirt +sources: + - title: Kubernetes Pod Lifecycle + url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ + date_accessed: 2026-02-17 +tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] +semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] +use_cases: [vm-creation, vm-lifecycle, diagnostics] +related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Runtime Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. + +**When to use this document**: +- VM or virt-launcher pod shows `CrashLoopBackOff` status +- virt-launcher pod repeatedly restarting +- Guest OS kernel panics on boot + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### CrashLoopBackOff + +**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting + +**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. + +**Possible Causes**: +- Guest OS kernel panic on boot +- Insufficient resources (memory/CPU) for guest OS +- Corrupted disk image or filesystem +- QEMU/libvirt crashes due to configuration errors +- Missing or incompatible device drivers in guest +- Resource limits too low for virt-launcher pod +- Virtualization features (KVM) not available on node + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check virt-launcher pod restart count**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# Look at RESTARTS column +``` + +**2. View recent crash logs** (previous container instance): + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "previous": true, + "tail": 100 +} +``` + +Look for QEMU errors, kernel panics, or segfaults. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n virt-launcher--xxx --previous +``` + +**3. Check current virt-launcher logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. + +**CLI Fallback** (if MCP unavailable or all containers needed): +```bash +oc logs -n virt-launcher--xxx --all-containers +``` + +**4. Check VMI conditions for crash details**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Extract `.status.conditions` for crash details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**5. Check pod events for crash reasons**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" +``` + +**6. Check pod resource limits**: + +**MCP Tool**: `pods_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +Extract `.spec.containers[0].resources` for resource limits. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq +``` + +**7. Check node kubelet logs for OOM kills**: + +⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +oc adm node-logs -u kubelet | grep -i oom +``` + +**8. Access guest console** (if VM briefly starts): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Crash Patterns**: + +1. **Guest Kernel Panic**: + - Console logs show kernel panic messages + - Guest crashes immediately after boot + - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" + +2. **OOM (Out of Memory)**: + - Pod killed with reason: `OOMKilled` + - Guest runs out of memory during boot or operation + - virt-launcher logs show memory allocation failures + +3. **QEMU Crash**: + - virt-launcher logs show QEMU segmentation fault + - Symptoms: "qemu-system-x86_64: terminated by signal" + - Configuration incompatibility or QEMU bug + +4. **Disk Image Corruption**: + - Guest cannot boot from disk + - Filesystem errors in guest console + - DataVolume import failed + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for kernel panic or boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for: + - Kernel panic messages + - Initramfs errors + - Filesystem mounting failures + - Missing device errors + +2. **Review virt-launcher crash logs**: + + Use `pods_log` with `previous: true` from diagnostic step 2. + + Look for: + - QEMU command line errors + - Device initialization failures + - Memory allocation errors + - Signal termination (SIGSEGV, SIGABRT) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx --previous + ``` + +3. **Check for OOM (Out of Memory) kills**: + + **MCP Tool**: `pods_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + Extract `.status.containerStatuses[0].lastState.terminated.reason`. + + If returns `"OOMKilled"`: + - Option 1: Increase virt-launcher memory limits + - Option 2: Decrease guest memory allocation + - Option 3: Use smaller instance type + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' + ``` + +4. **Increase resources if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check current memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +5. **Verify disk image integrity**: + + **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters for list**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" + } + ``` + + **Parameters for specific DV**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check `.status.phase` (should be `Succeeded`). + + If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get datavolume -n -o jsonpath='{.status.phase}' + ``` + +6. **Check virtualization (KVM) availability**: + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +7. **Simplify VM configuration** (eliminate variables): + + Try creating minimal VM using vm-create skill with: + - Small instance type + - No secondary networks + - Simple container disk (e.g., Fedora) + - No cloud-init + + If minimal VM works, add features back one by one. + +8. **Recreate VM with different workload** (test disk image): + + If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) + +After remediation, check pod restart count stops increasing: + +**Parameters for pods**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). + +Check VM reaches Running state: + +**Parameters for VM**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# RESTARTS should stabilize + +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Running +``` + +Verify guest is responsive: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# Should show login prompt or OS console +``` + +**Advanced Debugging**: + +**MCP Tool**: `pods_exec` (from openshift-virtualization) + +Check libvirt domain XML: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["virsh", "dumpxml", "1"] +} +``` + +Check QEMU process: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["ps", "aux"] +} +``` + +Filter output for "qemu" process. + +**CLI Fallback** (if MCP unavailable): +```bash +oc exec -n virt-launcher--xxx -- virsh dumpxml 1 +oc exec -n virt-launcher--xxx -- ps aux | grep qemu +``` + +**Prevention**: +- Start with minimal VM configuration and add complexity gradually +- Use recommended instance types for your workload +- Test disk images before deploying to production +- Ensure nodes have adequate resources and KVM support +- Monitor resource usage and set appropriate limits + +--- + +## Workaround Patterns for MCP Tool Limitations + +### General Pattern: Diagnose → Propose → Confirm → Execute + +When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): + +**1. Diagnose** the root cause using Kubernetes commands +```bash +oc describe vm -n +oc get events -n --field-selector involvedObject.name= +``` + +**2. Propose** a manual workaround to the user with clear explanation +- Explain why the issue occurred +- Show what will be changed +- List alternative options + +**3. Confirm** with user before executing (Human-in-the-Loop) +- Wait for explicit "yes" or "apply workaround" +- Never auto-execute modifications + +**4. Execute** the workaround using oc/kubectl +```bash +oc patch vm -n --type=merge -p '...' +``` + +**5. Verify** the fix was successful +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**6. Document** the limitation and suggest filing enhancement request +- Note this is temporary until MCP tool is enhanced +- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues + +### Example: Adding Tolerations Workaround + +**Diagnostic Output**: +```markdown +## ⚠️ VM Scheduling Issue Detected + +**Root Cause**: Node taints prevent VM scheduling + +**Details**: +- Found 3 nodes with taint: `virtualization=true:NoSchedule` +- VM spec does not include matching tolerations +- This prevents VM scheduling on virtualization-dedicated nodes +``` + +**Proposed Workaround**: +```bash +oc patch vm web-server -n vms --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' +``` + +**User Confirmation Required**: +``` +How would you like to proceed? +- "apply workaround" - I'll patch the VM with tolerations +- "manual" - I'll provide instructions for you to apply manually +- "cancel" - Delete the VM and abort creation +``` + +**After Execution**: +```markdown +## ✓ Workaround Applied Successfully + +**Action**: Added tolerations for taint `virtualization=true:NoSchedule` +**New Status**: Stopped (VM can now be scheduled) + +**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. +Future VMs in this cluster will need the same fix until the tool is enhanced. +``` + +--- + +## VM Status Reference + +### Status Values + +| Status | Meaning | Action Required | +|--------|---------|-----------------| +| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | +| `Running` | VM is running | Normal | +| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | +| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | +| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | +| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | +| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | +| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | +| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | +| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | + +### Checking VM Status + +```bash +# Get printable status +oc get vm -n -o jsonpath='{.status.printableStatus}' + +# Get detailed status and conditions +oc get vm -n -o jsonpath='{.status}' | jq + +# Watch status changes in real-time +oc get vm -n -w +``` + +--- + +## Best Practices for Agents + +When implementing diagnostic workflows: + +1. **Always verify VM status** after creation (wait 5-10 seconds first) +2. **Consult this document** when encountering error status values +3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) +4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) +5. **Respect human-in-the-loop** for all VM modifications +6. **Document temporary workarounds** and their limitations clearly +7. **Suggest filing issues** for missing MCP tool features + +### Document Consultation Pattern + +```markdown +**Document Consultation** (REQUIRED): +1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes +2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." +``` + +--- + +## Known MCP Tool Limitations + +### vm_create tool + +**Currently Supported**: +- ✓ Namespace, name (required) +- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) +- ✓ Size hints (small, medium, large) +- ✓ Storage size +- ✓ Autostart flag +- ✓ Networks (Multus NetworkAttachmentDefinitions) +- ✓ Performance family (u1, o1, c1, m1) +- ✓ Instance type, preference + +**Not Currently Supported** (requires workarounds): +- ✗ Tolerations (for node taints) +- ✗ Node selectors +- ✗ Affinity/anti-affinity rules +- ✗ Resource requests/limits (beyond instance type) +- ✗ Custom labels/annotations +- ✗ SSH keys injection +- ✗ Cloud-init user data + +**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. + +**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues + +--- + +## Additional Resources + +- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) +- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) +- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) +- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) + +--- + +[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/scheduling-errors.md new file mode 100644 index 00000000..e47c8c4f --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/scheduling-errors.md @@ -0,0 +1,417 @@ +--- +title: VM Scheduling Errors +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 +tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] +semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] +use_cases: [vm-creation, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Scheduling Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**When to use this document**: +- VM shows status `ErrorUnschedulable` after creation or start attempt +- VM events mention scheduling failures, taints, resources, or node selectors + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +## ErrorUnschedulable + +**Symptom**: VM shows status `ErrorUnschedulable` after creation + +**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**Possible Causes**: + +### 1. Node Taints (Most Common) + +Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM events for scheduling failures**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `involvedObject.name` == "" and look for messages like: +- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= +``` + +**2. Check node taints in the cluster**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' +``` + +**Common Taint Patterns**: +- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule +- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes +- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads + +**Solution - Add Tolerations to VM**: + +The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. + +**Workaround (post-creation using MCP Tools)**: + +**Step 1**: Get current VM spec + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Step 2**: Modify the returned JSON to add tolerations + +Add to `.spec.template.spec.tolerations`: +```json +{ + "tolerations": [ + { + "key": "virtualization", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule" + } + ] +} +``` + +**Step 3**: Update VM with modified spec + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +```json +{ + "resource": "" +} +``` + +Pass the complete modified VM resource as YAML or JSON string. + +**Step 4**: Verify tolerations were added + +Use `resources_get` again and check `.spec.template.spec.tolerations` in response. + +**Step 5**: Check if VM status improved + +Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. + +**CLI Fallback** (if MCP patch is too complex): +```bash +# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' + +# Verify tolerations +oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq + +# Check status +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**Example - Multiple Tolerations**: +```bash +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: "dedicated" + operator: "Equal" + value: "virt-workloads" + effect: "NoSchedule" +' +``` + +**Toleration Operators**: +- `Equal` - Key and value must match exactly +- `Exists` - Only key must exist (ignores value) + +**Toleration Effects**: +- `NoSchedule` - Don't schedule new pods (existing pods continue) +- `PreferNoSchedule` - Avoid scheduling if possible +- `NoExecute` - Don't schedule AND evict existing pods + +**Alternative Solutions**: +1. **Remove node taints** (if you have cluster-admin access): + ```bash + oc adm taint nodes virtualization=true:NoSchedule- + ``` + +2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits + +3. **File enhancement request** - Request tolerations support in openshift-mcp-server: + https://github.com/openshift/openshift-mcp-server/issues + +--- + +### 2. Insufficient Resources + +Not enough CPU, memory, or storage available on any node. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM resource requests**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' +``` + +**2. Check node resource availability**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. + +Alternatively, use `nodes_top` MCP tool for current resource usage. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe nodes | grep -A 5 "Allocated resources" +``` + +**3. Look for VM events mentioning "Insufficient"**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep "Insufficient" +``` + +**Example Event**: +``` +0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. +``` + +**Solutions** (Use MCP Tools First): + +1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) +2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill +3. **Delete unused VMs** - Use vm-delete skill to free up resources +4. **Check resource quotas**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters for quota**: + ```json + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" + } + ``` + + **Parameters for limit range**: + ```json + { + "apiVersion": "v1", + "kind": "LimitRange", + "namespace": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + oc describe limitrange -n + ``` + +--- + +### 3. Node Selector Mismatch + +VM requires specific node labels that don't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM node selector requirements**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.nodeSelector` to see required node labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' +``` + +**2. List available node labels**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.metadata.labels` for available labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes --show-labels +``` + +**3. Check if any nodes match the selector**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node", + "labelSelector": "=" +} +``` + +Should return at least one node with matching labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -l = +``` + +**Solutions** (Use MCP Tools First): + +**Option 1: Remove node selector from VM** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get current VM using `resources_get` (diagnostic step 1) +2. Remove `.spec.template.spec.nodeSelector` field +3. Update VM using `resources_create_or_update` with modified JSON + +**CLI Fallback** (JSON patch easier via CLI): +Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" +```bash +oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' +``` + +**Option 2: Add label to nodes** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get node using `resources_get` +2. Add label to `.metadata.labels` +3. Update node using `resources_create_or_update` + +⚠️ **Note**: Node labeling typically requires cluster admin privileges. + +**CLI Fallback** (simpler via CLI): +Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" +```bash +oc label node = +``` + +--- + +[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/storage-errors.md new file mode 100644 index 00000000..16ba584d --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/storage-errors.md @@ -0,0 +1,1011 @@ +--- +title: VM Storage Errors +category: kubevirt +sources: + - title: KubeVirt CDI - DataVolumes + url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ + date_accessed: 2026-02-17 + - title: Kubernetes Persistent Volumes + url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ + date_accessed: 2026-02-17 +tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] +semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] +use_cases: [vm-creation, vm-deletion, vm-cloning] +related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] +last_updated: 2026-02-17 +--- + +# VM Storage Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. + +**When to use this document**: +- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` +- Storage deletion fails after VM deletion +- DataVolume cloning operations fail +- PVC provisioning issues + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### ErrorDataVolumeNotReady + +**Symptom**: VM shows status `ErrorDataVolumeNotReady` + +**Description**: The DataVolume (persistent storage) backing the VM is not ready. + +**Possible Causes**: + +#### 1. DataVolume Still Provisioning + +Storage provisioning takes time, especially for large disks or when importing images. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Get detailed DataVolume information**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` and `.status.conditions` for provisioning details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o json +``` + +**3. Check PVC (PersistentVolumeClaim) bound status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Check `.status.phase` for each PVC (should be `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. + +#### 2. Storage Class Not Found + +The requested storage class doesn't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List available storage classes**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" +} +``` + +Review the list of available storage classes (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass +``` + +**2. Check DataVolume's requested storage class**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.spec.pvc.storageClassName` in the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' +``` + +**Solution**: +1. Use a valid storage class from the cluster +2. Recreate VM with correct storage class parameter + +#### 3. Insufficient Storage Quota + +Namespace has insufficient storage quota to provision the PVC. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check resource quotas**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**2. Check storage usage**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase +``` + +**Solution**: +1. Request quota increase from cluster admin +2. Delete unused PVCs to free quota +3. Reduce VM storage size + +--- + +### ErrorPvcNotFound + +**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List PVCs in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review the list of available PVCs (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check VM's PVC references**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' +``` + +**Solution**: +- Wait for DataVolume to create the PVC +- Manually create missing PVC +- Fix VM spec to reference correct PVC name + +--- + + +--- + +### Storage Deletion Failures + +**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace + +**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. + +**Possible Causes**: +- PVC still bound to active PersistentVolume with `Retain` policy +- DataVolume still being referenced by another resource +- CDI (Containerized Data Importer) controller issues +- Storage class retention policy preventing deletion +- Finalizers on PVC/DataVolume blocking cleanup +- PVC still mounted by a pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check PVC status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check specific PVC phase**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Released` or `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o jsonpath='{.status.phase}' +``` + +**3. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**4. Check what's using the PVC**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' +``` + +**5. Check PVC finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. + +**6. Check DataVolume finalizers**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.metadata.finalizers}' +``` + +**7. Check PV reclaim policy**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume" +} +``` + +Filter results for PV where `.spec.claimRef.name` matches ``. + +To get specific PV policy: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume", + "name": "" +} +``` + +Check `.spec.persistentVolumeReclaimPolicy`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pv | grep +oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' +``` + +**Common Finalizer Patterns**: +- `kubernetes.io/pvc-protection` - Protects PVC while in use +- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer + +**Solutions** (Use MCP Tools First): + +1. **Delete DataVolume first, then PVC**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete DataVolume first (often blocks PVC deletion): + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Wait a few seconds, then delete PVC: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete datavolume -n + oc delete pvc -n + ``` + +2. **Check for pods still using PVC**: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "" + } + ``` + + Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. + + Then delete the pods using `pods_delete`: + + **Parameters**: + ```json + { + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' + oc delete pod -n + ``` + +3. **Force delete PVC** (if safe to do so): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete pvc -n --grace-period=0 --force + ``` + +4. **Remove finalizers from PVC** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PVC using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update PVC using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Remove finalizers from DataVolume** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current DataVolume using `resources_get` (diagnostic step 6) + 2. Remove items from `.metadata.finalizers` array + 3. Update DataVolume using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Change PV reclaim policy** (if PV has Retain policy): + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PV policy using `resources_get` (diagnostic step 7) + 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` + 3. Update PV using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" + ```bash + oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' + oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' + ``` + +**Storage Quota Check** (Use MCP Tools First): + +After deletion, verify storage quota is freed: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters for quota check**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.used` to verify storage quota is freed. + +**Parameters for PVC verification**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +oc get pvc -n +``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Confirm PVC is deleted: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Confirm DataVolume is deleted: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +# Should return: Error from server (NotFound) + +oc get datavolume -n +# Should return: Error from server (NotFound) +``` + +--- + +### DataVolume Cloning Failures + +**Symptom**: VM clone created successfully but DataVolume clone operation fails + +**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. + +**Possible Causes**: +- CSI driver doesn't support volume cloning +- Source PVC storage class incompatible with cloning +- Cross-namespace cloning not permitted by storage backend +- Insufficient storage quota in target namespace +- Source PVC not in `Bound` state +- Storage class doesn't have volume cloning enabled +- CDI (Containerized Data Importer) controller issues + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume clone status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and `.metadata.name`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Check DataVolume events for errors**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe datavolume -n +``` + +**3. Check DataVolume phase**: + +Use `resources_get` from step 1, extract `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +``` + +**4. Check if storage class supports cloning**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "name": "" +} +``` + +Review the full YAML output for cloning-related configurations. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass -o yaml | grep -A 5 -i clone +``` + +**5. Check CSI driver capabilities**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "CSIDriver" +} +``` + +Review `.items[].metadata.name` for available CSI drivers. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get csidriver +``` + +**6. Check source PVC status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Bound` for cloning to work). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**7. Check target namespace storage quota**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**8. Check CDI controller logs**: + +**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) + +First, list pods in openshift-cnv namespace: + +**Parameters for pods_list_in_namespace**: +```json +{ + "namespace": "openshift-cnv", + "labelSelector": "app.kubernetes.io/component=cdi-deployment" +} +``` + +Then get logs using `pods_log`: + +**Parameters**: +```json +{ + "namespace": "openshift-cnv", + "name": "", + "tail": 100 +} +``` + +**CLI Fallback** (if MCP unavailable or easier via CLI): +```bash +oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') +``` + +**Common Error Messages**: +- `"volume cloning is not supported"` - CSI driver lacks clone capability +- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage +- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace +- `"insufficient quota"` - Target namespace lacks storage quota +- `"source PVC not bound"` - Source PVC must be in Bound state for cloning +- `"StorageClass does not support cloning"` - Storage class configuration issue + +**Solutions** (Use MCP Tools First): + +1. **Check storage class clone support**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" + } + ``` + + For each storage class in `.items[]`, check: + - `.metadata.name` (storage class name) + - `.provisioner` (CSI driver) + + Storage classes using CSI drivers typically support cloning. Look for provisioners like: + - `csi.ovirt.org` (oVirt CSI) + - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) + - `ebs.csi.aws.com` (AWS EBS CSI) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner + ``` + +2. **Verify source PVC is bound**: + + Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pvc -n -o jsonpath='{.status.phase}' + ``` + +3. **Check target namespace quota**: + + Use `resources_list` from diagnostic step 7 to check quota. + + If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + ``` + +4. **Use snapshot-based cloning** (alternative method): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Step 1**: Create VolumeSnapshot of source PVC + + **Parameters**: + ```json + { + "apiVersion": "snapshot.storage.k8s.io/v1", + "kind": "VolumeSnapshot", + "metadata": { + "name": "-snapshot", + "namespace": "" + }, + "spec": { + "source": { + "persistentVolumeClaimName": "" + } + } + } + ``` + + **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) + + **Step 3**: Create new DataVolume from snapshot + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": "-rootdisk", + "namespace": "" + }, + "spec": { + "source": { + "snapshot": { + "name": "-snapshot", + "namespace": "" + } + }, + "storage": { + "resources": { + "requests": { + "storage": "50Gi" + } + }, + "storageClassName": "" + } + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" + ```bash + cat <-snapshot + namespace: + spec: + source: + persistentVolumeClaimName: + EOF + + oc get volumesnapshot -snapshot -n + + cat <-rootdisk + namespace: + spec: + source: + snapshot: + name: -snapshot + namespace: + storage: + resources: + requests: + storage: 50Gi + storageClassName: + EOF + ``` + +5. **Use "new empty storage" option** (vm-clone skill): + - If cloning isn't supported, create VM with empty storage + - Manually copy data if needed + +6. **Cross-namespace cloning workaround**: + - Some storage backends require snapshot for cross-namespace cloning + - Create snapshot in source namespace, restore in target namespace (see solution 4 above) + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Check DataVolume reached Succeeded phase: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should return `Succeeded`). + +Check PVC was created and bound: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "-rootdisk" +} +``` + +Check `.status.phase` (should return `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +# Should return: Succeeded + +oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' +# Should return: Bound +``` + +**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-inventory/environment/mcp-servers/mock-virt-mcp.py b/evaluation/with_skills/rh-virt__vm-inventory/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..2e083d72 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-inventory/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1458 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-virt__vm-inventory/environment/skills/vm-inventory/SKILL.md b/evaluation/with_skills/rh-virt__vm-inventory/environment/skills/vm-inventory/SKILL.md new file mode 100644 index 00000000..32873b81 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-inventory/environment/skills/vm-inventory/SKILL.md @@ -0,0 +1,390 @@ +--- +name: vm-inventory +description: | + List and view virtual machines across namespaces with status, resource usage, and health information. + + Use when: + - "List all VMs" + - "Show VMs in namespace [name]" + - "What VMs are running?" + - "Get details of VM [name]" + + This skill provides comprehensive VM inventory and status reporting. + + NOT for creating or modifying VMs (use vm-create or vm-lifecycle-manager instead). + +model: inherit +color: cyan +--- + +# /vm-inventory Skill + +List and inspect virtual machines in OpenShift Virtualization clusters. This skill provides read-only access to VM information without making any modifications. + +## Prerequisites + +**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools**: +- `resources_list` (from openshift-virtualization) - List Kubernetes resources including VirtualMachines +- `resources_get` (from openshift-virtualization) - Get specific Kubernetes resource details + +**Fallback CLI Commands** (if MCP tools unavailable): +- `oc get virtualmachines` / `oc get vm` - List VirtualMachines +- `oc get vm -n -o yaml` - Get VM details + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster (>= 4.19) +- OpenShift Virtualization operator installed +- ServiceAccount with RBAC permissions to list and get VirtualMachine resources + +### Prerequisite Verification + +**Before executing:** + +1. Check `openshift-virtualization` exists in `.mcp.json` → If missing, report setup +2. Verify `KUBECONFIG` is set (presence only, never expose value) → If missing, report +3. (Optional) Test basic connectivity to cluster → If fails, report connection error + +**Human Notification Protocol:** `❌ Cannot execute vm-inventory: MCP server not available. Setup: Add to .mcp.json, set KUBECONFIG, restart Claude Code. Docs: https://github.com/openshift/openshift-mcp-server` + +⚠️ **SECURITY**: Never display KUBECONFIG path or credential values. + +**Note on Fallback**: If MCP server unavailable but KUBECONFIG set, offer CLI fallback with user confirmation. + +## When to Use This Skill + +**Trigger when:** +- User explicitly invokes `/vm-inventory` command +- User wants to see all VMs or VMs in a specific namespace +- User asks about VM status or health +- User needs to find a VM by name +- User wants details about a specific VM configuration + +**User phrases:** +- "List all VMs" +- "Show VMs in production namespace" +- "What VMs are running?" +- "Get details of VM web-server" +- "Show me the status of database-vm" +- "/vm-inventory" (explicit command) + +**Do NOT use when:** +- User wants to create a VM → Use `/vm-create` skill instead +- User wants to start/stop VMs → Use `/vm-lifecycle-manager` skill instead +- User wants to modify VM configuration → Different operation (not inventory) + +## Workflow + +**CRITICAL EXECUTION PATTERN**: +1. **ALWAYS attempt MCP server tools FIRST** - Try `resources_list` or `resources_get` +2. **If MCP tools fail** - Propose CLI commands (`oc get vm`) with user confirmation +3. **Never skip MCP attempt** - Always try them first + +**Tool Execution Priority**: MCP tools (primary) → CLI commands (fallback with confirmation) + +### Workflow A: List All VMs (Across All Namespaces) + +**Step 1: Query VirtualMachine Resources Using MCP Tool** + +**MCP Tool**: `resources_list` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", allNamespaces=true) + +**Errors:** Tool not found/connection error → Report, offer CLI fallback: `oc get virtualmachines -A -o json` + +**Step 2: Get Resource Details for Running VMs** + +**CRITICAL**: To display complete VM information, query VirtualMachineInstance (VMI) resources: + +**MCP Tool**: `resources_list` (apiVersion="kubevirt.io/v1", kind="VirtualMachineInstance") + +**For each VMI, extract**: +- `.spec.domain.cpu.sockets` and `.spec.domain.memory.guest` - Resources column ("X vCPU, YGi") +- `.status.volumeStatus[].persistentVolumeClaimInfo.capacity.storage` - Storage column (sum all PVC volumes, exclude container disks/cloudinit) +- `.status.guestOSInfo.prettyName` or `.status.guestOSInfo.name` + version - Guest OS column +- `.status.interfaces[0].ipAddress` - IP column (primary interface) +- `.status.nodeName` - Node column +- `.status.conditions[]` - Conditions column (Ready, AgentConnected, LiveMigratable) + +**Stopped VMs**: Use VirtualMachine spec for Resources only; Storage/Guest OS/IP/Conditions show "-" + +**Step 3: Format and Display Results** + +**ALWAYS display in table format** ordered by namespace and status: + +```markdown +## 📋 Virtual Machines (All Namespaces) + +| Namespace | VM Name | Status | Age | Resources | Storage | Guest OS | Node | IP | Conditions | +|-----------|---------|--------|-----|-----------|---------|----------|------|----|------------| +| development | debug-vm | ⚠ Pending | 2d | 2 vCPU, 4Gi | 30Gi | - | - | - | ⚠ Not Ready | +| development | test-vm | ✓ Running | 5d | 2 vCPU, 4Gi | 30Gi | Ubuntu 24.04 | worker-03 | 10.131.0.20 | ✓ Ready, ✓ Live Migration | +| production | database-vm | ✗ Stopped | 30d | 8 vCPU, 16Gi | - | - | - | - | - | +| production | web-server-01 | ✓ Running | 15d | 4 vCPU, 8Gi | 100Gi | RHEL 9.7 | worker-01 | 10.131.0.15 | ✓ Ready, ✓ Agent, ✗ Live Migration | +| production | web-server-02 | ✓ Running | 15d | 4 vCPU, 8Gi | 100Gi | RHEL 9.7 | worker-02 | 10.131.0.16 | ✓ Ready, ✓ Agent, ✗ Live Migration | + +**Summary:** +- **Total VMs**: 5 +- **Running**: 3 +- **Stopped**: 1 +- **Pending**: 1 +``` + +**Table Ordering Rules:** +1. **Primary sort**: Namespace (alphabetical) +2. **Secondary sort**: Status (Running → Pending → Stopped → Failed/Error) +3. **Tertiary sort**: VM Name (alphabetical within same namespace and status) + +**Status Indicators:** +- ✓ Running/Ready +- ✗ Stopped/Halted +- ⚠ Pending/Starting/Terminating +- ❌ Failed/Error + +**Resources Column Format**: MUST show "X vCPU, YGi" (query VMI `.spec.domain.cpu.sockets` and `.spec.domain.memory.guest`), NOT instance type names (e.g., NOT "u1.medium") + +### Workflow B: List VMs in Specific Namespace + +**Step 1: Gather Namespace** + +Ask user for namespace if not provided. + +**Step 2: Query VMs in Namespace Using MCP Tool** + +**MCP Tool**: `resources_list` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", namespace=``) + +**Errors:** Tool fails → Report, offer CLI fallback: `oc get virtualmachines -n -o json` + +**Step 3: Get Resource Details and Display** + +Follow same format rules as Workflow A Step 2-3. Use namespace-specific header: + +```markdown +## 📋 Virtual Machines in '' + +| Name | Status | vCPU | Memory | Age | Node | +|------|--------|------|--------|-----|------| +| web-server-01 | Running | 4 | 8Gi | 15d | worker-01 | +| web-server-02 | Running | 4 | 8Gi | 15d | worker-02 | +| database-vm | Stopped | 8 | 16Gi | 30d | - | + +**Summary**: 3 VMs (2 running, 1 stopped) +``` + +### Workflow C: Get Details of Specific VM + +**Step 1: Gather VM Information** + +Required: VM name, Namespace (ask if not provided) + +**Step 2: Retrieve VM Resource Details Using MCP Tool** + +**MCP Tool**: `resources_get` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", namespace=``, name=``) + +**Errors:** Tool fails → Report, offer CLI fallback: `oc get vm -n -o yaml` + +**Step 3: Interpret Status and Conditions (Optional)** + +**OPTIONAL**: If VM has error status (ErrorUnschedulable, ErrorDataVolumeNotReady, CrashLoopBackOff), consult [troubleshooting/INDEX.md](../../docs/troubleshooting/INDEX.md) using Read tool. Output: "Consulted INDEX.md to interpret status." + +**When to consult**: VM status is Error/Warning or stuck state (CrashLoopBackOff, Terminating) +**When NOT to consult**: VM status is normal (Running, Stopped, Provisioning) + +**Step 4: Display Detailed Information** + +```markdown +## 🖥️ Virtual Machine Details + +### Basic Information +- **Name**: `web-server-01` +- **Namespace**: `production` +- **Status**: Running +- **Created**: 15 days ago + +### Configuration +- **Instance Type**: u1.medium +- **Workload**: Fedora +- **Run Strategy**: Always (auto-restart on crash) + +### Resources +- **vCPU**: 4 cores +- **Memory**: 8Gi +- **Storage**: 50Gi +- **Storage Class**: ocs-storagecluster-ceph-rbd + +### Network +- **Primary**: default (pod network) +- **Secondary**: vlan100 (multus - 192.168.100.5) + +### Volumes +- **rootdisk**: 50Gi (DataVolume/PVC) + +### Current State +- **Phase**: Running +- **Ready**: True +- **Node**: worker-01 +- **Pod IP**: 10.129.2.45 +- **Guest OS Uptime**: 12 days + +### Conditions +- ✓ Ready +- ✓ LiveMigratable +- ✓ AgentConnected + +### Labels +- app: web +- env: production +- tier: frontend +``` + +### Workflow D: Filter VMs by Criteria + +**Step 1: Query VMs with Filters Using MCP Tool** + +**MCP Tool**: `resources_list` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", allNamespaces=true, labelSelector=``) + +**Filtering options**: +- By Labels (via labelSelector): `"app=web"`, `"app=web,env=production"`, `"tier in (frontend,backend)"` +- By Status (post-processing): Filter results by `status.printableStatus` field +- By Resource Size (post-processing): Parse instance type or VMI resource specs + +**Errors:** Tool fails → Report, offer CLI fallback: `oc get virtualmachines -A -l -o json` + +**Step 2: Display Filtered Results** + +Display with explanation: `## 📋 VMs with label 'app=web'` + list/table using Workflow A format + +## Common Issues + +### Issue 1: No VMs Found +**Error**: Empty list | **Causes**: No VMs exist, wrong namespace, insufficient RBAC | **Response**: Report no VMs found, suggest create VM (vm-create), list namespaces, check permissions + +### Issue 2: Permission Denied +**Error**: "Forbidden: User cannot list VirtualMachines" | **Solution**: Verify KUBECONFIG has list/get permissions, contact admin + +### Issue 3: Cluster Connection Error +**Error**: "Unable to connect to cluster" | **Solution**: Verify KUBECONFIG valid, check `oc cluster-info`, verify network, check credentials expiry + +## Output Formatting Guidelines + +**Use consistent status indicators:** +- ✓ Running/Healthy/Ready +- ✗ Stopped/Halted +- ⚠ Warning/Pending/Migrating +- ❌ Critical/Failed/Error + +**Include key information always:** +- VM name and namespace +- Current status +- Resource allocation (vCPU, memory) +- Age/creation time +- Node placement (for running VMs) + +**Organize by namespace** when showing multiple VMs for logical grouping and clear separation. + +**Provide actionable next steps:** How to start stopped VMs, get more details, when to use other skills + +## Integration with Other Skills + +**Before creating a VM** (vm-create): Use vm-inventory to check if VM name exists, verify namespace has capacity +**Before lifecycle operations** (vm-lifecycle-manager): Check current VM status, verify VM exists +**For troubleshooting**: Get VM overview with vm-inventory first, then use vm-troubleshooter for deep diagnostics + +## Dependencies + +### Required MCP Servers +- `openshift-virtualization` - OpenShift MCP server (https://github.com/openshift/openshift-mcp-server) + +### Required MCP Tools (PRIMARY - Always try first) +- `resources_list` - List resources (apiVersion, kind, namespace optional, allNamespaces optional, labelSelector optional) +- `resources_get` - Get resource details (apiVersion, kind, namespace, name) + +### CLI Fallback Commands (Use only if MCP tools fail) +- `oc get virtualmachines` / `oc get vm` - List VirtualMachines +- `oc get vm -n ` - Get specific VM +- `oc get vm -A` - List VMs across all namespaces +- `oc get vm -n ` - List VMs in specific namespace +- `oc get vm -l ` - Filter VMs by label selector + +**Important**: Always attempt MCP tools first. Only use CLI commands after MCP tool failure and with user confirmation. + +### Related Skills +- `vm-create` - Create VMs after checking inventory +- `vm-lifecycle-manager` - Manage VMs discovered in inventory +- `vm-troubleshooter` (planned) - Diagnose problematic VMs from inventory + +### Reference Documentation +- [Troubleshooting INDEX](../../docs/troubleshooting/INDEX.md) - VM status interpretation (optionally consulted when displaying VM details with error states) +- [OpenShift Virtualization Documentation](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt/about_virt/about-virt.html) +- [KubeVirt VirtualMachine API](https://kubevirt.io/api-reference/) +- [Accessing VMs](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt/virtual_machines/virt-accessing-vm-consoles.html) +- [VM Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) + +## Critical: Human-in-the-Loop Requirements + +**Not applicable** - This skill performs read-only operations and does not modify any cluster resources. No user confirmation required. + +**Read-only operations:** +- Listing VirtualMachines across namespaces or in specific namespaces +- Retrieving VM details, status, and resource configurations +- Displaying VM health conditions and resource usage +- Filtering VMs by labels or field selectors +- Viewing VM network, storage, and node placement information + +**No modifications performed:** +- ✓ Does not change VM state (start/stop/restart) +- ✓ Does not modify VM configuration +- ✓ Does not delete VMs or resources +- ✓ Does not consume cluster resources + +## Security Considerations + +- Read-only operations - no modifications to VMs +- Respects Kubernetes RBAC permissions +- Only shows VMs in namespaces user has access to +- KUBECONFIG credentials never exposed in output +- No sensitive VM configuration details displayed by default +- All queries audited in Kubernetes API logs + +## Example Usage + +### Example 1: List all VMs (table format) + +``` +User: "List all VMs" +Agent: [MCP: resources_list(apiVersion="kubevirt.io/v1", kind="VirtualMachine", allNamespaces=true)] + [Queries VMI resources for CPU/memory] + [Displays table format from Workflow A Step 3] +``` + +### Example 2: CLI fallback when MCP unavailable + +``` +User: "List all VMs" +Agent: [MCP tool fails] + ⚠️ MCP tool 'resources_list' not available. Use CLI: `oc get virtualmachines -A`? +User: "yes" +Agent: [Executes: oc get virtualmachines -A -o json] + [Displays table format] +``` + +### Example 3: Get specific VM details + +``` +User: "Show me details of web-server-01 in production" +Agent: [MCP: resources_get(kind="VirtualMachine", namespace="production", name="web-server-01")] + [Displays VM Details format from Workflow C Step 4] +``` + +### Example 4: Filter running VMs + +``` +User: "Show me all running VMs" +Agent: [Lists all VMs, filters by status.printableStatus == "Running"] + ## ✓ Running Virtual Machines + ### production: web-server-01 (4 vCPU, 8Gi, worker-01) | web-server-02 (4 vCPU, 8Gi, worker-02) + ### development: test-vm (2 vCPU, 4Gi, worker-03) + **Total**: 3 running VMs +``` diff --git a/evaluation/with_skills/rh-virt__vm-inventory/instruction.md b/evaluation/with_skills/rh-virt__vm-inventory/instruction.md new file mode 100644 index 00000000..28107e57 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-inventory/instruction.md @@ -0,0 +1,14 @@ +# VM Inventory Task + +You are an OpenShift Virtualization administrator. Your team needs a complete picture of every VM in the cluster for capacity planning and compliance reporting. + +## Requirements +- List every VM across all namespaces, grouped by namespace +- For each VM report: name, status (Running/Stopped/Paused), CPU and memory allocation, operating system, and IP address if running +- Identify any VMs with issues: stopped unexpectedly, guest agent not responding, degraded conditions, or running end-of-life operating systems +- Summarize totals: how many VMs per namespace, how many running vs stopped, total resource allocation +- Sort results by namespace, then by VM name + +Write the inventory report in `/root/report.md`. + +Use MCP tools to gather VM data. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-virt__vm-inventory/solution/solve.sh b/evaluation/with_skills/rh-virt__vm-inventory/solution/solve.sh new file mode 100644 index 00000000..3473c6d5 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-inventory/solution/solve.sh @@ -0,0 +1,32 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Inventory Report + +## Cluster VM Summary + +| Namespace | VM Name | Status | CPU | Memory | Storage | OS | IP | Node | +|-----------|---------|--------|-----|--------|---------|----|----|------| +| prod-vms | production-db | Running | 4 vCPU, 16Gi | 100Gi | RHEL 9.3 | 10.128.2.15 | worker-01 | +| prod-vms | web-frontend | Running | 2 vCPU, 4Gi | 50Gi | Fedora 39 | 10.128.2.16 | worker-02 | +| dev-vms | dev-test | Stopped | 2 vCPU, 8Gi | 50Gi | Ubuntu 22.04 | — | — | + +### Status Summary +- Running: 2 +- Stopped: 1 +- Total: 3 + +### Data Sources +- VM status: `status.printableStatus` from VirtualMachine resource +- Resource details: Extracted from VirtualMachineInstance (VMI) when running via resources_list (apiVersion kubevirt.io/v1, allNamespaces=true for cluster-wide) +- CPU: `.spec.domain.cpu.sockets` (displayed as vCPU) +- Memory: `.spec.domain.memory.guest` +- Storage: `.status.volumeStatus[].persistentVolumeClaimInfo.capacity.storage` +- OS: `.status.guestOSInfo.prettyName` +- IP: `.status.interfaces[0].ipAddress` +- Node: `.status.nodeName` +- Conditions: Ready, AgentConnected, LiveMigratable + +### Sort Order +Sorted by: Namespace → Status (Running → Pending → Stopped → Failed) → VM Name + +REPORT_EOF diff --git a/evaluation/with_skills/rh-virt__vm-inventory/task.toml b/evaluation/with_skills/rh-virt__vm-inventory/task.toml new file mode 100644 index 00000000..6a756f27 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-inventory/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-inventory" +name = "rh-virt VM Inventory Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-inventory", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-virt__vm-inventory/tests/llm_judge.py b/evaluation/with_skills/rh-virt__vm-inventory/tests/llm_judge.py new file mode 100644 index 00000000..aabb1dab --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-inventory/tests/llm_judge.py @@ -0,0 +1,92 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "vmi_vs_vm_distinction", "file": "/root/report.md", "question": "Does the report distinguish between VirtualMachine (spec/desired state) and VirtualMachineInstance (runtime state) as separate resources to query?", "reference": "A skilled report queries both VM and VMI, understanding VM defines the spec while VMI reflects the running state. An unskilled report only queries VirtualMachine without VMI runtime data."}, + {"id": "status_ordering", "file": "/root/report.md", "question": "Does the report organize or sort VMs by operational status (e.g., Running first, then Pending, Stopped, Failed) rather than just listing alphabetically?", "reference": "A skilled report groups or sorts VMs by status priority. An unskilled report lists VMs in arbitrary order without status-based organization."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-virt__vm-inventory/tests/test.sh b/evaluation/with_skills/rh-virt__vm-inventory/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-inventory/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-virt__vm-inventory/tests/test_outputs.py b/evaluation/with_skills/rh-virt__vm-inventory/tests/test_outputs.py new file mode 100644 index 00000000..16ded70a --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-inventory/tests/test_outputs.py @@ -0,0 +1,67 @@ +""" +Tests for rh-virt__vm-inventory per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_has_structured_data(self): + content = read_report() + has_table = "|" in content and content.count("|") >= 4 + has_list = content.count("- ") >= 5 + assert has_table or has_list, "report should present VM inventory in a structured format (table or list)" + + def test_mentions_namespace(self): + content = read_report().lower() + assert "namespace" in content, "report should organize by namespace" + + +class TestSkillDependent: + def test_vmi_runtime_data(self): + """Skill: Query VirtualMachineInstance (VMI) for running VM runtime data.""" + c = read_report().lower() + assert any(t in c for t in ["virtualmachineinstance", "vmi", "virtual machine instance"]), ( + "should reference VMI for runtime data, not just VirtualMachine" + ) + + def test_resource_format(self): + """Skill: Resources as 'X vCPU, YGi' format, not instance type names like u1.medium.""" + c = read_report().lower() + assert any(t in c for t in ["vcpu", "vcpus"]) and any(t in c for t in ["gi", "gib"]), ( + "should use vCPU/Gi resource format, not instance type names" + ) + + def test_status_based_grouping(self): + """Skill: Sort by namespace, then status (Running > Pending > Stopped > Failed), then name.""" + c = read_report().lower() + status_terms = sum(1 for t in ["running", "stopped", "pending", "failed"] if t in c) + has_organization = any(t in c for t in [ + "group", "sort", "order", "organiz", "by namespace", + "by status", "running first", "namespace", + ]) + assert status_terms >= 2 and has_organization, ( + "should organize VMs with status awareness (Running/Stopped/etc) by namespace" + ) + + def test_conditions_awareness(self): + """Skill: KubeVirt-specific conditions — AgentConnected, LiveMigratable.""" + c = read_report().lower() + assert any(t in c for t in [ + "agentconnected", "agent connected", "agent_connected", + "livemigratable", "live migratable", "live_migratable", + "guest agent", + ]), "should mention KubeVirt-specific conditions (AgentConnected, LiveMigratable)" diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/Dockerfile b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/Dockerfile new file mode 100644 index 00000000..ae625e01 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/.ai-index/semantic-index.json new file mode 100644 index 00000000..2512c2a3 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/.ai-index/semantic-index.json @@ -0,0 +1,148 @@ +{ + "index_version": "1.0.0", + "last_updated": "2026-02-17", + "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", + + "documents": [ + { + "file": "INDEX.md", + "title": "VM Troubleshooting Guide - Index", + "category": "navigation", + "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], + "use_cases": ["navigation", "tool-reference", "general-guidance"], + "token_estimate": 450, + "skills": ["all"] + }, + { + "file": "scheduling-errors.md", + "title": "VM Scheduling Errors", + "category": "scheduling", + "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], + "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], + "errors_covered": [ + "ErrorUnschedulable - Node Taints", + "ErrorUnschedulable - Insufficient Resources", + "ErrorUnschedulable - Node Selector Mismatch" + ], + "token_estimate": 820, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "storage-errors.md", + "title": "VM Storage Errors", + "category": "storage", + "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], + "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], + "errors_covered": [ + "ErrorDataVolumeNotReady - DataVolume Still Provisioning", + "ErrorDataVolumeNotReady - Storage Class Not Found", + "ErrorDataVolumeNotReady - Insufficient Storage Quota", + "ErrorPvcNotFound", + "Storage Deletion Failures", + "DataVolume Cloning Failures" + ], + "token_estimate": 1350, + "skills": ["vm-creator", "vm-delete", "vm-clone"] + }, + { + "file": "lifecycle-errors.md", + "title": "VM Lifecycle Errors", + "category": "lifecycle", + "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], + "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], + "errors_covered": [ + "VM Stuck in Terminating State", + "VM Won't Start (Non-Scheduling Issues)", + "VM Won't Stop" + ], + "token_estimate": 1150, + "skills": ["vm-delete", "vm-lifecycle-manager"] + }, + { + "file": "runtime-errors.md", + "title": "VM Runtime Errors", + "category": "runtime", + "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], + "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], + "errors_covered": [ + "CrashLoopBackOff" + ], + "token_estimate": 850, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "network-errors.md", + "title": "VM Network Errors", + "category": "networking", + "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], + "use_cases": ["vm-creation", "network-troubleshooting"], + "errors_covered": [ + "Network Attachment Failures" + ], + "token_estimate": 580, + "skills": ["vm-creator"] + } + ], + + "skill_to_docs_mapping": { + "vm-creator": { + "primary_docs": ["scheduling-errors.md", "storage-errors.md"], + "secondary_docs": ["runtime-errors.md", "network-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3650, + "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" + }, + "vm-delete": { + "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 2950, + "description": "VM deletion failures - stuck terminating, storage cleanup" + }, + "vm-clone": { + "primary_docs": ["storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 1800, + "description": "VM cloning failures - DataVolume cloning, storage provisioning" + }, + "vm-lifecycle-manager": { + "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], + "secondary_docs": ["runtime-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3270, + "description": "VM start/stop failures - lifecycle management, scheduling" + }, + "vm-inventory": { + "primary_docs": [], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 450, + "description": "General guidance for status interpretation" + } + }, + + "error_to_docs_mapping": { + "ErrorUnschedulable": "scheduling-errors.md", + "ErrorDataVolumeNotReady": "storage-errors.md", + "ErrorPvcNotFound": "storage-errors.md", + "Terminating": "lifecycle-errors.md", + "CrashLoopBackOff": "runtime-errors.md", + "NetworkAttachmentFailed": "network-errors.md" + }, + + "cross_references": { + "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], + "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], + "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], + "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], + "network-errors.md": ["scheduling-errors.md"] + }, + + "progressive_disclosure_strategy": { + "description": "Load only the minimum documentation needed for current task", + "pattern": "error_type → primary_doc → cross_references (if needed)", + "token_savings": "29% average reduction vs single-file (based on rh-sre data)", + "navigation_overhead_reduction": "85% vs single-file navigation" + } +} diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/INDEX.md new file mode 100644 index 00000000..a2e76963 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/INDEX.md @@ -0,0 +1,332 @@ +--- +title: VM Troubleshooting Guide - Index +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 + - title: OpenShift Virtualization - Virtual Machine Status + url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html + date_accessed: 2026-02-06 + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] +semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] +use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] +last_updated: 2026-02-17 +--- + +# VM Troubleshooting Guide - Index + +## Overview + +This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: + +- **Creation**: Scheduling failures, storage provisioning issues +- **Lifecycle**: Start/stop failures, stuck states +- **Deletion**: Resources stuck in Terminating, storage cleanup failures +- **Cloning**: DataVolume cloning errors, cross-namespace issues +- **Networking**: Secondary network attachment failures +- **Runtime**: CrashLoopBackOff, guest OS boot failures + +This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. + +--- + +## 🗂️ Troubleshooting Categories + +The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: + +### 1. [Scheduling Errors](scheduling-errors.md) +**When to use**: VM fails to schedule on any node + +**Errors covered**: +- ErrorUnschedulable - Node Taints +- ErrorUnschedulable - Insufficient Resources +- ErrorUnschedulable - Node Selector Mismatch + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 2. [Storage Errors](storage-errors.md) +**When to use**: VM has storage provisioning, deletion, or cloning issues + +**Errors covered**: +- ErrorDataVolumeNotReady (all 3 subsections) +- ErrorPvcNotFound +- Storage Deletion Failures +- DataVolume Cloning Failures + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### 3. [Lifecycle Errors](lifecycle-errors.md) +**When to use**: VM has start/stop/termination issues + +**Errors covered**: +- VM Stuck in Terminating State +- VM Won't Start (Non-Scheduling Issues) +- VM Won't Stop + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### 4. [Runtime Errors](runtime-errors.md) +**When to use**: VM repeatedly crashes or fails at runtime + +**Errors covered**: +- CrashLoopBackOff + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 5. [Network Errors](network-errors.md) +**When to use**: VM has secondary network attachment failures + +**Errors covered**: +- Network Attachment Failures (Multus, SR-IOV) + +**Skills that use this**: vm-create + +--- + +## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last + +### Critical Principle: MCP-First Approach + +**All diagnostic procedures in this guide follow the MCP-first pattern**: + +``` +1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) +2. ⚠️ IF FAILS: Ask user permission to use CLI command +3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval +``` + +### Why MCP Tools First? + +- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources +- **Consistency**: Same interface across all operations +- **Better Error Handling**: MCP tools return structured errors +- **Audit Trail**: MCP tool usage is logged and trackable + +### Command Pattern Examples + +Throughout this guide, you'll see diagnostic steps formatted like this: + +**✅ CORRECT Pattern (MCP First)**: + +```markdown +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Extract**: `.metadata.finalizers` from returned JSON + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` +``` + +### Available MCP Tools + +The `openshift-virtualization` MCP server provides these tools: + +**Resource Operations**: +- `resources_get` - Get specific resource (replaces `oc get `) +- `resources_list` - List resources (replaces `oc get `) +- `resources_delete` - Delete resource (replaces `oc delete`) +- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) + +**Pod Operations**: +- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) +- `pods_get` - Get pod details (replaces `oc get pod`) +- `pods_log` - Get pod logs (replaces `oc logs`) +- `pods_exec` - Execute in pod (replaces `oc exec`) +- `pods_delete` - Delete pod (replaces `oc delete pod`) +- `pods_top` - Pod resource usage (replaces `oc top pods`) + +**Events & Monitoring**: +- `events_list` - List events (replaces `oc get events`) +- `nodes_top` - Node resource usage (replaces `oc top nodes`) +- `nodes_log` - Node logs (replaces `oc adm node-logs`) +- `nodes_stats_summary` - Detailed node stats + +**VM Operations** (KubeVirt toolset): +- `vm_create` - Create VMs +- `vm_lifecycle` - Start/stop/restart VMs + +**Namespaces**: +- `namespaces_list` - List namespaces (replaces `oc get namespaces`) +- `projects_list` - List OpenShift projects (replaces `oc get projects`) + +### When CLI Commands Are Required + +Some operations have **NO MCP equivalent** and require CLI: + +- `virtctl` commands (console, VNC access) +- `oc debug node` (node debugging) +- `oc auth can-i` (permission checks) +- `oc adm taint` (node taint management) +- Complex JSON patch operations + +For these, the guide will note: **"CLI Only - No MCP equivalent"** + +### Quick Reference: CLI → MCP Mapping + +| CLI Command | MCP Tool Equivalent | +|-------------|---------------------| +| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | +| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | +| `oc logs -n ` | `pods_log` with name="", namespace="" | +| `oc get events -n ` | `events_list` with namespace="" | +| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | + +**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. + +### How to Read Diagnostic Sections + +Each error section includes: +1. **Symptom** - What you observe +2. **Description** - What's happening +3. **Possible Causes** - Why it's happening +4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback +5. **Solutions** - **MCP tools first**, then CLI fallback +6. **Verification** - **MCP tools first**, then CLI fallback + +**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. + +--- + +## 🔍 Quick Navigation by Skill + +**vm-create**: +- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics +- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning +- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff +- [Network Errors](network-errors.md) - Network attachment failures + +**vm-delete**: +- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state +- [Storage Errors](storage-errors.md) - Storage deletion failures + +**vm-clone**: +- [Storage Errors](storage-errors.md) - DataVolume cloning failures + +**vm-lifecycle-manager**: +- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop +- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling + +**vm-inventory**: +- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed + +--- + +## 📊 Documentation Coverage & Maintenance + +### Current Coverage + +This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: + +- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) +- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures +- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop +- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) +- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems + +**Total errors documented**: 12 error types across 6 categories + +--- + +### Encountering Undocumented Errors + +#### For AI Agents (Claude Code) + +If you encounter an error **not documented** in the categories above: + +1. **Report to user** with all available details (error message, affected resources, namespace) +2. **Provide best-effort diagnostics** using MCP tools: + - `resources_get` to inspect resource status + - `pods_log` to check virt-launcher or compute container logs + - `events_list` to view Kubernetes events +3. **Suggest documentation update**: + ``` + ⚠️ This error is not yet documented in the troubleshooting guide. + + **Error**: + **Affected resource**: / + + I recommend adding this error to the troubleshooting guide: + - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file + - If it's a new error class → Create new category file in docs/troubleshooting/ + + Would you like me to help document this error for future reference? + ``` + +4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs + +--- + +#### For Human Contributors + +To document a new error: + +1. **Determine the category**: + - Scheduling issue → `scheduling-errors.md` + - Storage problem → `storage-errors.md` + - Lifecycle/start/stop → `lifecycle-errors.md` + - Runtime crash → `runtime-errors.md` + - Network issue → `network-errors.md` + - New error class → Create new category file (e.g., `security-errors.md`) + +2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): + - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback + - **Solutions**: MCP Tool approach → Implementation → CLI Fallback + - **Verification**: MCP Tool checks → Expected results → CLI Fallback + +3. **Update semantic index**: + - Edit `.ai-index/semantic-index.json` + - Add error to `error_to_docs_mapping` + - Update relevant skill's `primary_docs` or `secondary_docs` + - Increment token estimates if needed + +4. **Reference in skill files**: + - Update skill's Reference Documentation section + - Ensure skill's Document Consultation steps point to the right category + +5. **Update this INDEX.md**: + - Add error to the appropriate category section (lines 50-120) + - Update "Current Coverage" list above + +--- + +## 📚 Additional Resources + +- [rh-virt README](../../README.md) - MCP server setup and skill documentation +- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources +- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/lifecycle-errors.md new file mode 100644 index 00000000..6167fcc8 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/lifecycle-errors.md @@ -0,0 +1,869 @@ +--- +title: VM Lifecycle Errors +category: kubevirt +sources: + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 +tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] +semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] +use_cases: [vm-deletion, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Lifecycle Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM lifecycle issues including start/stop failures and stuck termination states. + +**When to use this document**: +- VM stuck in `Terminating` state +- VM won't start (runStrategy is Always but VM never reaches Running) +- VM won't stop (runStrategy is Halted but VM never reaches Stopped) + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### VM Stuck in Terminating State + +**Symptom**: VM shows status `Terminating` but deletion never completes + +**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). + +**Possible Causes**: +- Finalizers blocking deletion +- PVC/DataVolume still attached and preventing cleanup +- VirtualMachineInstance (VMI) not terminating properly +- Custom controllers or operators blocking deletion +- Stuck virt-launcher pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check finalizers on the VM**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` + +**2. Check if VMI still exists**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name containing "virt-launcher-". + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check events for deletion issues**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results where `involvedObject.name` == "" and sort by timestamp. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' +``` + +**5. Check VM deletion timestamp**: + +Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. + +If present, VM is in deletion process. If null, VM is not being deleted. + +**Common Finalizer Patterns**: +- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer +- `foregroundDeletion` - Waits for dependent objects to be deleted +- Custom finalizers from operators + +**Solutions** (Use MCP Tools First): + +1. **Wait for dependent resources** (safest approach): + - Use `resources_list` to check if PVCs, DataVolumes are still deleting + - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) + +2. **Force delete VMI** (if VMI is stuck): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable - requires explicit user permission): + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +3. **Force delete virt-launcher pod** (if pod is stuck): + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + # Find the virt-launcher pod name first + oc get pods -n | grep virt-launcher- + + # Then delete it + oc delete pod virt-launcher--xxx -n --force --grace-period=0 + ``` + +4. **Remove finalizers** (⚠️ dangerous - use only as last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` + 2. Remove items from `.metadata.finalizers` array + 3. Update VM using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch not easily done via MCP): + ```bash + # This operation is complex for MCP - may need CLI + oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Check for protection labels** (vm-delete skill specific): + + Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. + + If value is "true", the vm-delete skill refuses deletion (this is expected behavior). + +**Verification**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error if deletion successful. + +**CLI Fallback**: +```bash +oc get vm -n +# Should return: Error from server (NotFound) +``` + +--- + + +--- + +### VM Won't Start (Non-Scheduling Issues) + +**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state + +**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. + +**Possible Causes**: +- Guest OS kernel panic or boot failure +- Cloud-init configuration errors +- Missing or corrupted disk image +- Insufficient memory for guest OS to boot +- QEMU/KVM virtualization errors +- VirtualMachineInstance (VMI) creation failures +- virt-launcher pod crashes + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VMI (VirtualMachineInstance) status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and overall VMI state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**2. Check VMI conditions for errors**: + +Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. View virt-launcher pod logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +First, get pod name from step 3, then: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +Look for QEMU errors, memory allocation failures, device errors. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**5. Check virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. + +Alternatively, use `pods_get` to get full pod details: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**6. Access VM console to see guest OS boot messages**: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +Look for kernel panic, initramfs errors, filesystem errors. + +**7. Check VM events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep -A 20 "Events:" +``` + +**Common Error Patterns**: + +1. **Guest OS Boot Failure**: + - Console shows kernel panic + - Guest hangs at GRUB or boot loader + - Cloud-init errors during first boot + +2. **Insufficient Memory**: + - Guest OS kills processes due to OOM + - VMI logs show memory allocation errors + +3. **QEMU Crashes**: + - virt-launcher logs show QEMU segfaults + - VMI repeatedly restarts + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for kernel panic, initramfs errors, filesystem errors. + +2. **Check virt-launcher pod logs for QEMU errors**: + + Use `pods_log` from diagnostic step 4 to view logs. + + Look for: + - "qemu-system-x86_64: ..." errors + - Memory allocation failures + - Device errors + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx + ``` + +3. **Increase memory if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check VMI memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +4. **Verify disk image integrity**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check PVC is bound: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + If using container disk, verify image exists and is accessible (check VMI spec). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get pvc -n + ``` + +5. **Check cloud-init configuration** (if applicable): + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. + + Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq + ``` + +6. **Restart VMI** (soft reset): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete VMI (VM controller will recreate it): + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for new VMI to start (use `resources_get` to check status). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n -w + ``` + +7. **Check virtualization extensions** (KVM): + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + + # In debug shell: + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should eventually return `Running`). + +Check VMI is running: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should show `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should eventually return: Running + +oc get vmi -n +# Should show: Running +``` + +--- + +### VM Won't Stop + +**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state + +**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. + +**Possible Causes**: +- Guest OS not responding to ACPI shutdown signal +- virt-launcher pod stuck and not terminating +- VirtualMachineInstance (VMI) deletion blocked by finalizers +- Guest shutdown scripts hanging +- Filesystem sync issues in guest OS + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (might show `Stopping`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**2. Check VMI status and deletion timestamp**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check VMI events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vmi -n | grep -A 10 "Events:" +``` + +**5. Check VMI finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.metadata.finalizers}' +``` + +**6. Check if guest is responsive** (if VMI still exists): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Patterns**: +- VMI shows `deletionTimestamp` but never actually deletes +- virt-launcher pod in `Terminating` state +- VM runStrategy is `Halted` but printableStatus shows `Stopping` + +**Solutions** (Use MCP Tools First): + +1. **Wait for graceful shutdown** (default: 30 seconds): + + Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. + +2. **Force stop by deleting VMI**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + This is the standard way to force-stop a VM. + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n + # Should return: Error from server (NotFound) + ``` + +3. **Force delete VMI with grace period** (if VMI won't delete): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +4. **Force delete virt-launcher pod**: + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find the pod using `pods_list_in_namespace` from diagnostic step 3. + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + ⚠️ **Note**: For force deletion with grace period, use CLI fallback. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') + oc delete pod $POD_NAME -n --force --grace-period=0 + ``` + +5. **Remove VMI finalizers** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VMI using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update VMI using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Patch VM runStrategy directly** (ensure consistency): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` (diagnostic step 1) + 2. Set `.spec.runStrategy` to `"Halted"` + 3. Update VM using `resources_create_or_update` with modified JSON + + **CLI Fallback** (merge patch easier via CLI): + Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Stopped` or `Halted`). + +Verify VMI is gone: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Verify virt-launcher pod is gone: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for pods matching `virt-launcher-`. Should return no results. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Stopped or Halted + +oc get vmi -n +# Should return: Error from server (NotFound) + +oc get pods -n | grep virt-launcher- +# Should return: No resources found +``` + +**Prevention**: +- Ensure guest OS has ACPI support enabled +- Use proper shutdown commands in guest OS +- Avoid forceful stops unless necessary (can corrupt guest filesystem) + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/network-errors.md new file mode 100644 index 00000000..82fa313c --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/network-errors.md @@ -0,0 +1,429 @@ +--- +title: VM Network Errors +category: kubevirt +sources: + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] +semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] +use_cases: [vm-creation, network-troubleshooting] +related_docs: [INDEX.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Network Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. + +**When to use this document**: +- VM created successfully but secondary networks not attached +- NetworkAttachmentDefinition not found errors +- Multus CNI failures +- SR-IOV device attachment issues + +**Skills that use this**: vm-create + +--- + +### Network Attachment Failures + +**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working + +**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). + +**Possible Causes**: +- NetworkAttachmentDefinition doesn't exist in the namespace +- Multus CNI not installed or not configured on cluster +- Namespace mismatch (NAD in different namespace than VM) +- Interface name conflicts in VM spec +- Bridge/network configuration errors in NAD +- SR-IOV device not available (if using SR-IOV) + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List NetworkAttachmentDefinitions in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "" +} +``` + +Review `.items[].metadata.name` for available NADs. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -n +``` + +**2. List NetworkAttachmentDefinitions in all namespaces**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition" +} +``` + +Omit `namespace` parameter to list across all namespaces. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -A +``` + +**3. Check specific NetworkAttachmentDefinition**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "", + "name": "" +} +``` + +Review `.spec.config` for CNI configuration. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definition -n -o yaml +``` + +**4. Check VM network configuration**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.networks` to see network references. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq +``` + +**5. Check VM domain interfaces**: + +Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq +``` + +**6. Check virt-launcher pod network annotations** (shows actual attachments): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**7. Check Multus is installed**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "openshift-multus" +} +``` + +Should show Multus CNI pods running. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n openshift-multus +``` + +**8. Check for errors in virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n +``` + +**Common Error Messages**: +- `"network-attachment-definition not found"` - NAD doesn't exist in namespace +- `"multus CNI not configured"` - Multus not installed or misconfigured +- `"interface name conflict"` - Duplicate interface names in VM spec +- `"failed to add network"` - CNI plugin error (check NAD config) +- `"no available devices"` - SR-IOV device not available (if using SR-IOV) + +**Solutions** (Use MCP Tools First): + +1. **Verify NetworkAttachmentDefinition exists in correct namespace**: + + Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. + + If NAD is in different namespace, copy it to VM namespace: + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get NAD from source namespace using `resources_get` + 2. Modify `.metadata.namespace` to target namespace + 3. Create NAD in target namespace using `resources_create_or_update` + + **CLI Fallback** (stream processing easier via CLI): + Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" + ```bash + oc get network-attachment-definition -n -o yaml | \ + sed "s/namespace: /namespace: /" | \ + oc apply -f - + ``` + +2. **Create missing NetworkAttachmentDefinition**: + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + Example: Linux bridge network + + **Parameters**: + ```json + { + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": { + "name": "vlan100", + "namespace": "" + }, + "spec": { + "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" + ```bash + cat < + spec: + config: '{ + "cniVersion": "0.3.1", + "type": "bridge", + "bridge": "br1", + "vlan": 100, + "ipam": { + "type": "host-local", + "subnet": "192.168.100.0/24" + } + }' + EOF + ``` + +3. **Check Multus CNI installation**: + + Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. + + To check cluster network operator: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterOperator" + } + ``` + + Filter for `network` operator. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n openshift-multus + oc get clusteroperators network + ``` + +4. **Fix interface name conflicts** (if VM has duplicate names): + + Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. + + Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. + + **CLI Fallback** (interactive edit easier via CLI): + Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' + oc edit vm -n + ``` + +5. **Validate NAD configuration syntax**: + + Use `resources_get` from diagnostic step 3, extract `.spec.config`. + + Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get network-attachment-definition -n -o jsonpath='{.spec.config}' + ``` + +6. **Check SR-IOV device availability** (if using SR-IOV networks): + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + List SR-IOV network node policies: + + **Parameters**: + ```json + { + "apiVersion": "sriovnetwork.openshift.io/v1", + "kind": "SriovNetworkNodePolicy", + "namespace": "openshift-sriov-network-operator" + } + ``` + + Check SR-IOV device plugin pods: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "openshift-sriov-network-operator" + } + ``` + + Filter for pods with "device-plugin" in name. + + Check available SR-IOV devices on node: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "Node", + "name": "" + } + ``` + + Review `.status.allocatable` for SR-IOV resources. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get sriovnetworknodepolicy -n openshift-sriov-network-operator + oc get pods -n openshift-sriov-network-operator | grep device-plugin + oc describe node | grep -A 10 "Allocatable:" + ``` + +7. **Recreate VM with corrected network configuration** (if needed): + + If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +After remediation, check virt-launcher pod network status: + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +Should show all attached networks with status. Example output: +```json +[ + { + "name": "openshift-sdn", + "interface": "eth0", + "ips": ["10.128.2.10"], + "default": true + }, + { + "name": "vlan100", + "interface": "net1", + "ips": ["192.168.100.5"] + } +] +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**Check from inside VM** (via console): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# In guest OS: +ip addr show +# Should show all network interfaces (eth0, net1, etc.) +``` + +**Common Network Types**: +- **Linux Bridge**: Layer 2 bridge for VLAN networks +- **SR-IOV**: High-performance direct device assignment +- **macvlan**: MAC-based VLAN for container networks +- **OVN-Kubernetes**: OpenShift native overlay network + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/runtime-errors.md new file mode 100644 index 00000000..804d3cbc --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/runtime-errors.md @@ -0,0 +1,616 @@ +--- +title: VM Runtime Errors +category: kubevirt +sources: + - title: Kubernetes Pod Lifecycle + url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ + date_accessed: 2026-02-17 +tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] +semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] +use_cases: [vm-creation, vm-lifecycle, diagnostics] +related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Runtime Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. + +**When to use this document**: +- VM or virt-launcher pod shows `CrashLoopBackOff` status +- virt-launcher pod repeatedly restarting +- Guest OS kernel panics on boot + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### CrashLoopBackOff + +**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting + +**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. + +**Possible Causes**: +- Guest OS kernel panic on boot +- Insufficient resources (memory/CPU) for guest OS +- Corrupted disk image or filesystem +- QEMU/libvirt crashes due to configuration errors +- Missing or incompatible device drivers in guest +- Resource limits too low for virt-launcher pod +- Virtualization features (KVM) not available on node + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check virt-launcher pod restart count**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# Look at RESTARTS column +``` + +**2. View recent crash logs** (previous container instance): + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "previous": true, + "tail": 100 +} +``` + +Look for QEMU errors, kernel panics, or segfaults. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n virt-launcher--xxx --previous +``` + +**3. Check current virt-launcher logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. + +**CLI Fallback** (if MCP unavailable or all containers needed): +```bash +oc logs -n virt-launcher--xxx --all-containers +``` + +**4. Check VMI conditions for crash details**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Extract `.status.conditions` for crash details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**5. Check pod events for crash reasons**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" +``` + +**6. Check pod resource limits**: + +**MCP Tool**: `pods_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +Extract `.spec.containers[0].resources` for resource limits. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq +``` + +**7. Check node kubelet logs for OOM kills**: + +⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +oc adm node-logs -u kubelet | grep -i oom +``` + +**8. Access guest console** (if VM briefly starts): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Crash Patterns**: + +1. **Guest Kernel Panic**: + - Console logs show kernel panic messages + - Guest crashes immediately after boot + - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" + +2. **OOM (Out of Memory)**: + - Pod killed with reason: `OOMKilled` + - Guest runs out of memory during boot or operation + - virt-launcher logs show memory allocation failures + +3. **QEMU Crash**: + - virt-launcher logs show QEMU segmentation fault + - Symptoms: "qemu-system-x86_64: terminated by signal" + - Configuration incompatibility or QEMU bug + +4. **Disk Image Corruption**: + - Guest cannot boot from disk + - Filesystem errors in guest console + - DataVolume import failed + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for kernel panic or boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for: + - Kernel panic messages + - Initramfs errors + - Filesystem mounting failures + - Missing device errors + +2. **Review virt-launcher crash logs**: + + Use `pods_log` with `previous: true` from diagnostic step 2. + + Look for: + - QEMU command line errors + - Device initialization failures + - Memory allocation errors + - Signal termination (SIGSEGV, SIGABRT) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx --previous + ``` + +3. **Check for OOM (Out of Memory) kills**: + + **MCP Tool**: `pods_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + Extract `.status.containerStatuses[0].lastState.terminated.reason`. + + If returns `"OOMKilled"`: + - Option 1: Increase virt-launcher memory limits + - Option 2: Decrease guest memory allocation + - Option 3: Use smaller instance type + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' + ``` + +4. **Increase resources if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check current memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +5. **Verify disk image integrity**: + + **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters for list**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" + } + ``` + + **Parameters for specific DV**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check `.status.phase` (should be `Succeeded`). + + If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get datavolume -n -o jsonpath='{.status.phase}' + ``` + +6. **Check virtualization (KVM) availability**: + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +7. **Simplify VM configuration** (eliminate variables): + + Try creating minimal VM using vm-create skill with: + - Small instance type + - No secondary networks + - Simple container disk (e.g., Fedora) + - No cloud-init + + If minimal VM works, add features back one by one. + +8. **Recreate VM with different workload** (test disk image): + + If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) + +After remediation, check pod restart count stops increasing: + +**Parameters for pods**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). + +Check VM reaches Running state: + +**Parameters for VM**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# RESTARTS should stabilize + +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Running +``` + +Verify guest is responsive: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# Should show login prompt or OS console +``` + +**Advanced Debugging**: + +**MCP Tool**: `pods_exec` (from openshift-virtualization) + +Check libvirt domain XML: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["virsh", "dumpxml", "1"] +} +``` + +Check QEMU process: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["ps", "aux"] +} +``` + +Filter output for "qemu" process. + +**CLI Fallback** (if MCP unavailable): +```bash +oc exec -n virt-launcher--xxx -- virsh dumpxml 1 +oc exec -n virt-launcher--xxx -- ps aux | grep qemu +``` + +**Prevention**: +- Start with minimal VM configuration and add complexity gradually +- Use recommended instance types for your workload +- Test disk images before deploying to production +- Ensure nodes have adequate resources and KVM support +- Monitor resource usage and set appropriate limits + +--- + +## Workaround Patterns for MCP Tool Limitations + +### General Pattern: Diagnose → Propose → Confirm → Execute + +When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): + +**1. Diagnose** the root cause using Kubernetes commands +```bash +oc describe vm -n +oc get events -n --field-selector involvedObject.name= +``` + +**2. Propose** a manual workaround to the user with clear explanation +- Explain why the issue occurred +- Show what will be changed +- List alternative options + +**3. Confirm** with user before executing (Human-in-the-Loop) +- Wait for explicit "yes" or "apply workaround" +- Never auto-execute modifications + +**4. Execute** the workaround using oc/kubectl +```bash +oc patch vm -n --type=merge -p '...' +``` + +**5. Verify** the fix was successful +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**6. Document** the limitation and suggest filing enhancement request +- Note this is temporary until MCP tool is enhanced +- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues + +### Example: Adding Tolerations Workaround + +**Diagnostic Output**: +```markdown +## ⚠️ VM Scheduling Issue Detected + +**Root Cause**: Node taints prevent VM scheduling + +**Details**: +- Found 3 nodes with taint: `virtualization=true:NoSchedule` +- VM spec does not include matching tolerations +- This prevents VM scheduling on virtualization-dedicated nodes +``` + +**Proposed Workaround**: +```bash +oc patch vm web-server -n vms --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' +``` + +**User Confirmation Required**: +``` +How would you like to proceed? +- "apply workaround" - I'll patch the VM with tolerations +- "manual" - I'll provide instructions for you to apply manually +- "cancel" - Delete the VM and abort creation +``` + +**After Execution**: +```markdown +## ✓ Workaround Applied Successfully + +**Action**: Added tolerations for taint `virtualization=true:NoSchedule` +**New Status**: Stopped (VM can now be scheduled) + +**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. +Future VMs in this cluster will need the same fix until the tool is enhanced. +``` + +--- + +## VM Status Reference + +### Status Values + +| Status | Meaning | Action Required | +|--------|---------|-----------------| +| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | +| `Running` | VM is running | Normal | +| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | +| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | +| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | +| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | +| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | +| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | +| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | +| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | + +### Checking VM Status + +```bash +# Get printable status +oc get vm -n -o jsonpath='{.status.printableStatus}' + +# Get detailed status and conditions +oc get vm -n -o jsonpath='{.status}' | jq + +# Watch status changes in real-time +oc get vm -n -w +``` + +--- + +## Best Practices for Agents + +When implementing diagnostic workflows: + +1. **Always verify VM status** after creation (wait 5-10 seconds first) +2. **Consult this document** when encountering error status values +3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) +4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) +5. **Respect human-in-the-loop** for all VM modifications +6. **Document temporary workarounds** and their limitations clearly +7. **Suggest filing issues** for missing MCP tool features + +### Document Consultation Pattern + +```markdown +**Document Consultation** (REQUIRED): +1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes +2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." +``` + +--- + +## Known MCP Tool Limitations + +### vm_create tool + +**Currently Supported**: +- ✓ Namespace, name (required) +- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) +- ✓ Size hints (small, medium, large) +- ✓ Storage size +- ✓ Autostart flag +- ✓ Networks (Multus NetworkAttachmentDefinitions) +- ✓ Performance family (u1, o1, c1, m1) +- ✓ Instance type, preference + +**Not Currently Supported** (requires workarounds): +- ✗ Tolerations (for node taints) +- ✗ Node selectors +- ✗ Affinity/anti-affinity rules +- ✗ Resource requests/limits (beyond instance type) +- ✗ Custom labels/annotations +- ✗ SSH keys injection +- ✗ Cloud-init user data + +**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. + +**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues + +--- + +## Additional Resources + +- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) +- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) +- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) +- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) + +--- + +[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/scheduling-errors.md new file mode 100644 index 00000000..e47c8c4f --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/scheduling-errors.md @@ -0,0 +1,417 @@ +--- +title: VM Scheduling Errors +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 +tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] +semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] +use_cases: [vm-creation, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Scheduling Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**When to use this document**: +- VM shows status `ErrorUnschedulable` after creation or start attempt +- VM events mention scheduling failures, taints, resources, or node selectors + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +## ErrorUnschedulable + +**Symptom**: VM shows status `ErrorUnschedulable` after creation + +**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**Possible Causes**: + +### 1. Node Taints (Most Common) + +Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM events for scheduling failures**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `involvedObject.name` == "" and look for messages like: +- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= +``` + +**2. Check node taints in the cluster**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' +``` + +**Common Taint Patterns**: +- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule +- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes +- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads + +**Solution - Add Tolerations to VM**: + +The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. + +**Workaround (post-creation using MCP Tools)**: + +**Step 1**: Get current VM spec + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Step 2**: Modify the returned JSON to add tolerations + +Add to `.spec.template.spec.tolerations`: +```json +{ + "tolerations": [ + { + "key": "virtualization", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule" + } + ] +} +``` + +**Step 3**: Update VM with modified spec + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +```json +{ + "resource": "" +} +``` + +Pass the complete modified VM resource as YAML or JSON string. + +**Step 4**: Verify tolerations were added + +Use `resources_get` again and check `.spec.template.spec.tolerations` in response. + +**Step 5**: Check if VM status improved + +Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. + +**CLI Fallback** (if MCP patch is too complex): +```bash +# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' + +# Verify tolerations +oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq + +# Check status +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**Example - Multiple Tolerations**: +```bash +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: "dedicated" + operator: "Equal" + value: "virt-workloads" + effect: "NoSchedule" +' +``` + +**Toleration Operators**: +- `Equal` - Key and value must match exactly +- `Exists` - Only key must exist (ignores value) + +**Toleration Effects**: +- `NoSchedule` - Don't schedule new pods (existing pods continue) +- `PreferNoSchedule` - Avoid scheduling if possible +- `NoExecute` - Don't schedule AND evict existing pods + +**Alternative Solutions**: +1. **Remove node taints** (if you have cluster-admin access): + ```bash + oc adm taint nodes virtualization=true:NoSchedule- + ``` + +2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits + +3. **File enhancement request** - Request tolerations support in openshift-mcp-server: + https://github.com/openshift/openshift-mcp-server/issues + +--- + +### 2. Insufficient Resources + +Not enough CPU, memory, or storage available on any node. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM resource requests**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' +``` + +**2. Check node resource availability**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. + +Alternatively, use `nodes_top` MCP tool for current resource usage. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe nodes | grep -A 5 "Allocated resources" +``` + +**3. Look for VM events mentioning "Insufficient"**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep "Insufficient" +``` + +**Example Event**: +``` +0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. +``` + +**Solutions** (Use MCP Tools First): + +1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) +2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill +3. **Delete unused VMs** - Use vm-delete skill to free up resources +4. **Check resource quotas**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters for quota**: + ```json + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" + } + ``` + + **Parameters for limit range**: + ```json + { + "apiVersion": "v1", + "kind": "LimitRange", + "namespace": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + oc describe limitrange -n + ``` + +--- + +### 3. Node Selector Mismatch + +VM requires specific node labels that don't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM node selector requirements**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.nodeSelector` to see required node labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' +``` + +**2. List available node labels**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.metadata.labels` for available labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes --show-labels +``` + +**3. Check if any nodes match the selector**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node", + "labelSelector": "=" +} +``` + +Should return at least one node with matching labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -l = +``` + +**Solutions** (Use MCP Tools First): + +**Option 1: Remove node selector from VM** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get current VM using `resources_get` (diagnostic step 1) +2. Remove `.spec.template.spec.nodeSelector` field +3. Update VM using `resources_create_or_update` with modified JSON + +**CLI Fallback** (JSON patch easier via CLI): +Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" +```bash +oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' +``` + +**Option 2: Add label to nodes** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get node using `resources_get` +2. Add label to `.metadata.labels` +3. Update node using `resources_create_or_update` + +⚠️ **Note**: Node labeling typically requires cluster admin privileges. + +**CLI Fallback** (simpler via CLI): +Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" +```bash +oc label node = +``` + +--- + +[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/storage-errors.md new file mode 100644 index 00000000..16ba584d --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/storage-errors.md @@ -0,0 +1,1011 @@ +--- +title: VM Storage Errors +category: kubevirt +sources: + - title: KubeVirt CDI - DataVolumes + url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ + date_accessed: 2026-02-17 + - title: Kubernetes Persistent Volumes + url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ + date_accessed: 2026-02-17 +tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] +semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] +use_cases: [vm-creation, vm-deletion, vm-cloning] +related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] +last_updated: 2026-02-17 +--- + +# VM Storage Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. + +**When to use this document**: +- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` +- Storage deletion fails after VM deletion +- DataVolume cloning operations fail +- PVC provisioning issues + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### ErrorDataVolumeNotReady + +**Symptom**: VM shows status `ErrorDataVolumeNotReady` + +**Description**: The DataVolume (persistent storage) backing the VM is not ready. + +**Possible Causes**: + +#### 1. DataVolume Still Provisioning + +Storage provisioning takes time, especially for large disks or when importing images. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Get detailed DataVolume information**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` and `.status.conditions` for provisioning details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o json +``` + +**3. Check PVC (PersistentVolumeClaim) bound status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Check `.status.phase` for each PVC (should be `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. + +#### 2. Storage Class Not Found + +The requested storage class doesn't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List available storage classes**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" +} +``` + +Review the list of available storage classes (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass +``` + +**2. Check DataVolume's requested storage class**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.spec.pvc.storageClassName` in the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' +``` + +**Solution**: +1. Use a valid storage class from the cluster +2. Recreate VM with correct storage class parameter + +#### 3. Insufficient Storage Quota + +Namespace has insufficient storage quota to provision the PVC. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check resource quotas**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**2. Check storage usage**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase +``` + +**Solution**: +1. Request quota increase from cluster admin +2. Delete unused PVCs to free quota +3. Reduce VM storage size + +--- + +### ErrorPvcNotFound + +**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List PVCs in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review the list of available PVCs (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check VM's PVC references**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' +``` + +**Solution**: +- Wait for DataVolume to create the PVC +- Manually create missing PVC +- Fix VM spec to reference correct PVC name + +--- + + +--- + +### Storage Deletion Failures + +**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace + +**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. + +**Possible Causes**: +- PVC still bound to active PersistentVolume with `Retain` policy +- DataVolume still being referenced by another resource +- CDI (Containerized Data Importer) controller issues +- Storage class retention policy preventing deletion +- Finalizers on PVC/DataVolume blocking cleanup +- PVC still mounted by a pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check PVC status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check specific PVC phase**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Released` or `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o jsonpath='{.status.phase}' +``` + +**3. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**4. Check what's using the PVC**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' +``` + +**5. Check PVC finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. + +**6. Check DataVolume finalizers**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.metadata.finalizers}' +``` + +**7. Check PV reclaim policy**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume" +} +``` + +Filter results for PV where `.spec.claimRef.name` matches ``. + +To get specific PV policy: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume", + "name": "" +} +``` + +Check `.spec.persistentVolumeReclaimPolicy`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pv | grep +oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' +``` + +**Common Finalizer Patterns**: +- `kubernetes.io/pvc-protection` - Protects PVC while in use +- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer + +**Solutions** (Use MCP Tools First): + +1. **Delete DataVolume first, then PVC**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete DataVolume first (often blocks PVC deletion): + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Wait a few seconds, then delete PVC: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete datavolume -n + oc delete pvc -n + ``` + +2. **Check for pods still using PVC**: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "" + } + ``` + + Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. + + Then delete the pods using `pods_delete`: + + **Parameters**: + ```json + { + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' + oc delete pod -n + ``` + +3. **Force delete PVC** (if safe to do so): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete pvc -n --grace-period=0 --force + ``` + +4. **Remove finalizers from PVC** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PVC using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update PVC using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Remove finalizers from DataVolume** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current DataVolume using `resources_get` (diagnostic step 6) + 2. Remove items from `.metadata.finalizers` array + 3. Update DataVolume using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Change PV reclaim policy** (if PV has Retain policy): + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PV policy using `resources_get` (diagnostic step 7) + 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` + 3. Update PV using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" + ```bash + oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' + oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' + ``` + +**Storage Quota Check** (Use MCP Tools First): + +After deletion, verify storage quota is freed: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters for quota check**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.used` to verify storage quota is freed. + +**Parameters for PVC verification**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +oc get pvc -n +``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Confirm PVC is deleted: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Confirm DataVolume is deleted: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +# Should return: Error from server (NotFound) + +oc get datavolume -n +# Should return: Error from server (NotFound) +``` + +--- + +### DataVolume Cloning Failures + +**Symptom**: VM clone created successfully but DataVolume clone operation fails + +**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. + +**Possible Causes**: +- CSI driver doesn't support volume cloning +- Source PVC storage class incompatible with cloning +- Cross-namespace cloning not permitted by storage backend +- Insufficient storage quota in target namespace +- Source PVC not in `Bound` state +- Storage class doesn't have volume cloning enabled +- CDI (Containerized Data Importer) controller issues + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume clone status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and `.metadata.name`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Check DataVolume events for errors**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe datavolume -n +``` + +**3. Check DataVolume phase**: + +Use `resources_get` from step 1, extract `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +``` + +**4. Check if storage class supports cloning**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "name": "" +} +``` + +Review the full YAML output for cloning-related configurations. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass -o yaml | grep -A 5 -i clone +``` + +**5. Check CSI driver capabilities**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "CSIDriver" +} +``` + +Review `.items[].metadata.name` for available CSI drivers. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get csidriver +``` + +**6. Check source PVC status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Bound` for cloning to work). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**7. Check target namespace storage quota**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**8. Check CDI controller logs**: + +**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) + +First, list pods in openshift-cnv namespace: + +**Parameters for pods_list_in_namespace**: +```json +{ + "namespace": "openshift-cnv", + "labelSelector": "app.kubernetes.io/component=cdi-deployment" +} +``` + +Then get logs using `pods_log`: + +**Parameters**: +```json +{ + "namespace": "openshift-cnv", + "name": "", + "tail": 100 +} +``` + +**CLI Fallback** (if MCP unavailable or easier via CLI): +```bash +oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') +``` + +**Common Error Messages**: +- `"volume cloning is not supported"` - CSI driver lacks clone capability +- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage +- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace +- `"insufficient quota"` - Target namespace lacks storage quota +- `"source PVC not bound"` - Source PVC must be in Bound state for cloning +- `"StorageClass does not support cloning"` - Storage class configuration issue + +**Solutions** (Use MCP Tools First): + +1. **Check storage class clone support**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" + } + ``` + + For each storage class in `.items[]`, check: + - `.metadata.name` (storage class name) + - `.provisioner` (CSI driver) + + Storage classes using CSI drivers typically support cloning. Look for provisioners like: + - `csi.ovirt.org` (oVirt CSI) + - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) + - `ebs.csi.aws.com` (AWS EBS CSI) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner + ``` + +2. **Verify source PVC is bound**: + + Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pvc -n -o jsonpath='{.status.phase}' + ``` + +3. **Check target namespace quota**: + + Use `resources_list` from diagnostic step 7 to check quota. + + If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + ``` + +4. **Use snapshot-based cloning** (alternative method): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Step 1**: Create VolumeSnapshot of source PVC + + **Parameters**: + ```json + { + "apiVersion": "snapshot.storage.k8s.io/v1", + "kind": "VolumeSnapshot", + "metadata": { + "name": "-snapshot", + "namespace": "" + }, + "spec": { + "source": { + "persistentVolumeClaimName": "" + } + } + } + ``` + + **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) + + **Step 3**: Create new DataVolume from snapshot + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": "-rootdisk", + "namespace": "" + }, + "spec": { + "source": { + "snapshot": { + "name": "-snapshot", + "namespace": "" + } + }, + "storage": { + "resources": { + "requests": { + "storage": "50Gi" + } + }, + "storageClassName": "" + } + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" + ```bash + cat <-snapshot + namespace: + spec: + source: + persistentVolumeClaimName: + EOF + + oc get volumesnapshot -snapshot -n + + cat <-rootdisk + namespace: + spec: + source: + snapshot: + name: -snapshot + namespace: + storage: + resources: + requests: + storage: 50Gi + storageClassName: + EOF + ``` + +5. **Use "new empty storage" option** (vm-clone skill): + - If cloning isn't supported, create VM with empty storage + - Manually copy data if needed + +6. **Cross-namespace cloning workaround**: + - Some storage backends require snapshot for cross-namespace cloning + - Create snapshot in source namespace, restore in target namespace (see solution 4 above) + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Check DataVolume reached Succeeded phase: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should return `Succeeded`). + +Check PVC was created and bound: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "-rootdisk" +} +``` + +Check `.status.phase` (should return `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +# Should return: Succeeded + +oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' +# Should return: Bound +``` + +**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/mcp-servers/mock-virt-mcp.py b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..31b95dd3 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1467 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), + ("prod-vms", {"env": "production"}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── prod-vms (instruction-specific) ────────────────────────────────── + _vm("web-frontend", "prod-vms", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "customer-facing": "true", "criticality": "high"}, + 4, 8, "Running", True, 1), + _vm("production-db", "prod-vms", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true"}, + 8, 16, "Running", True, 1), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/skills/vm-lifecycle-manager/SKILL.md b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/skills/vm-lifecycle-manager/SKILL.md new file mode 100644 index 00000000..e41c3f78 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/skills/vm-lifecycle-manager/SKILL.md @@ -0,0 +1,308 @@ +--- +name: vm-lifecycle-manager +description: | + Manage virtual machine lifecycle operations including start, stop, and restart. + + Use when: + - "Start VM [name]" + - "Stop the virtual machine [name]" + - "Restart VM [name]" + - "Power on/off VM [name]" + + This skill handles VM state transitions safely with user confirmation for each action. + + NOT for creating VMs (use vm-create) or deleting VMs (use vm-delete). + +model: inherit +color: blue +--- + +# /vm-lifecycle-manager Skill + +Control virtual machine power state in OpenShift Virtualization using the `vm_lifecycle` tool. + +## Prerequisites + +**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools**: +- `vm_lifecycle` (from openshift-virtualization) - Manage VM power state + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster (>= 4.19) +- OpenShift Virtualization operator installed +- ServiceAccount with RBAC permissions to update VirtualMachine resources + +### Prerequisite Verification + +**Before executing:** + +1. Check `openshift-virtualization` exists in `.mcp.json` → If missing, report setup +2. Verify `KUBECONFIG` is set (presence only, never expose value) → If missing, report + +**Human Notification Protocol:** `❌ Cannot execute vm-lifecycle-manager: MCP server not available. Setup: Add to .mcp.json, set KUBECONFIG, restart Claude Code. Docs: https://github.com/openshift/openshift-mcp-server` + +⚠️ **SECURITY**: Never display KUBECONFIG path or credential values. + +## When to Use This Skill + +**Trigger when:** +- User explicitly invokes `/vm-lifecycle-manager` command +- User requests starting/stopping/restarting a VM +- User wants to change VM power state + +**User phrases:** +- "Start VM web-server in namespace vms" +- "Stop the database VM" +- "Restart test-vm" +- "Power on the VM called api-server" +- "/vm-lifecycle-manager" (explicit command) + +**Do NOT use when:** +- Create VM → `/vm-create` +- List VMs → `/vm-inventory` +- Delete VM → `/vm-delete` + +## Workflow + +### Step 1: Gather Parameters and Confirm + +**Required from user:** VM Name, Namespace, Action (start|stop|restart) + +**Present for confirmation:** + +```markdown +## VM Lifecycle Operation + +| Parameter | Value | Impact | +|-----------|-------|--------| +| VM Name | `` | from user | +| Namespace | `` | from user | +| Action | `` | start: consumes resources / stop: graceful shutdown / restart: brief interruption (~1-2min) | + +Confirm: yes/no +``` + +Proceed automatically unless the user explicitly objects. + +### Step 2: Execute Lifecycle Operation + +**ONLY AFTER user confirmation in Step 1.** + +**For start or stop actions:** + +**MCP Tool**: `vm_lifecycle` (namespace=``, name=``, action=``) + +**For restart action (composite operation):** + +**CRITICAL**: Implement restart as two separate operations to avoid resourceVersion conflicts: + +1. **Stop VM**: `vm_lifecycle` (namespace=``, name=``, action="stop") +2. **Verify stopped**: `resources_get` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", namespace=``, name=``) → Check `status.printableStatus` == "Stopped" +3. **Wait**: 5 seconds for VM to fully stop +4. **Start VM**: `vm_lifecycle` (namespace=``, name=``, action="start") +5. **Verify started**: `resources_get` → Check `status.printableStatus` == "Running" + +**Errors:** +- VM not found → Report, suggest vm-inventory +- Permission denied → Report RBAC error +- Already in desired state → Inform user +- Stop fails during restart → Report, do not proceed to start +- Start fails during restart → Report, VM is stopped +- Transition fails → Report details + +### Step 3: Report Operation Status + +**On Success:** + +```markdown +## ✓ VM Successful + +**VM**: `` | **Namespace**: `` | **Action**: | **RunStrategy**: + +**Impact**: +- **start**: Running, consuming resources (CPU/memory). Access: virtctl console or SSH. RunStrategy: Always (auto-restart on crash) +- **stop**: Stopped, resources freed. State preserved. Start: "Start VM ". RunStrategy: Halted (stays off) +- **restart**: Running after stop+start. Brief interruption (~1-2min). Monitor app logs. RunStrategy: Always + +**Next**: "Show status of VM " or "List VMs in namespace " +``` + +**On Failure:** + +**OPTIONAL**: Read [lifecycle-errors.md](../../docs/troubleshooting/lifecycle-errors.md) for start/stop failures or [scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) for ErrorUnschedulable. Output: "Consulted lifecycle-errors.md for failure." + +**When to consult**: Start/stop failures, stuck transitions, unexpected errors. **NOT**: Already in state, not found, RBAC errors. + +```markdown +## ❌ Lifecycle Operation Failed + +**Error**: + +**Causes**: VM not found | RBAC denied | Already in desired state | VM in transition (wait 30-60s) | Resource constraints (start) + +**Troubleshoot**: +1. vm-inventory to verify VM exists +2. Check RBAC: `oc auth can-i update virtualmachines -n ` +3. View VM status and events +4. Check node capacity (for start operations) +``` + +## Common Issues + +### Issue 1: VM Not Found +**Error**: "VirtualMachine 'xyz' not found in namespace 'abc'" +**Solution**: Verify spelling, check namespace, use vm-inventory, VM may be deleted + +### Issue 2: VM Already in Desired State +**Warning**: "VM is already running" (when attempting start) +**Solution**: Not an error - VM already in desired state. Use `restart` if intended to restart + +### Issue 3: Permission Denied +**Error**: "Forbidden: User cannot update VirtualMachines" +**Solution**: Verify RBAC permissions (update VirtualMachine resources), contact admin + +### Issue 4: VM Stuck in Transitioning State +**Error**: "VM stuck in 'Terminating' or 'Starting'" +**Solution**: Wait 30-60s, check events (`oc describe vm`), use vm-troubleshooter, check virt-launcher pod + +### Issue 5: Insufficient Resources (Start) +**Error**: "Insufficient CPU/memory to start VM" +**Solution**: Check cluster availability, stop other VMs, scale nodes, resize VM to smaller instance type + +### Issue 6: Restart Implementation +**Note**: Restart is implemented as two separate operations (stop → verify → start → verify) +**Reason**: Avoids Kubernetes resourceVersion conflicts when using single restart action +**Behavior**: If stop succeeds but start fails, VM remains stopped. Check VM status with vm-inventory + +## Understanding RunStrategy + +| Action | RunStrategy | Behavior | +|--------|------------|----------| +| start | Always | Runs, auto-restarts on crash | +| stop | Halted | Stops, stays off | +| restart | Always | Stops, starts, auto-restarts | + +## Dependencies + +### Required MCP Servers +- `openshift-virtualization` - OpenShift MCP server with KubeVirt toolset + +### Required MCP Tools +- `vm_lifecycle` - Manage VM power state (start/stop/restart) + +### Related Skills +- `vm-create` - Create VMs +- `vm-inventory` - Check VM status +- `vm-troubleshooter` (planned) - Diagnose startup/shutdown issues + +### Reference Documentation +- [lifecycle-errors.md](../../docs/troubleshooting/lifecycle-errors.md) - Start/stop failures, stuck transitions (consulted on failures) +- [scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) - ErrorUnschedulable, resource constraints (consulted when VM won't start) +- [Troubleshooting INDEX](../../docs/troubleshooting/INDEX.md) - Navigation hub for error categories +- [OpenShift Virt Docs](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt/about_virt/about-virt.html) +- [KubeVirt Lifecycle](https://kubevirt.io/user-guide/virtual_machines/lifecycle/) +- [RunStrategy Docs](https://kubevirt.io/user-guide/virtual_machines/run_strategies/) +- [OpenShift MCP](https://github.com/openshift/openshift-mcp-server) + +## Critical: Human-in-the-Loop Requirements + +**IMPORTANT:** This skill requires explicit user confirmation before executing. You MUST: + +1. **Wait for user confirmation** on specific action (start/stop/restart) before executing `vm_lifecycle` +2. **Do NOT proceed** until user explicitly approves +3. **Present action clearly** with VM name, namespace, intended state change +4. **Never auto-execute** - these operations impact running services +5. **Warn about consequences** - stopping interrupts services, starting consumes resources, restarting causes downtime + +If user says "no" or wants to reconsider, do not proceed. + +**Why**: start (consumes resources), stop (interrupts services), restart (brief downtime). User should verify correct VM and understand impact. + +## Security Considerations + +- Requires RBAC permissions (update VirtualMachine resources) +- Stop operations attempt graceful shutdown; guest OS/app data consistency depends on workload +- Start operations consume resources (quota limits apply) +- All operations audited in Kubernetes API logs +- KUBECONFIG credentials never exposed +- ServiceAccount permissions limit controllable VMs + +## Example Usage + +### Example 1: Start Stopped VM + +``` +User: "Start VM web-server in namespace vms" +Agent: [vm-lifecycle-manager skill] +## VM Lifecycle Operation +| VM Name | `web-server` | Namespace | `vms` | Action | `start` | consumes resources | +Confirm: yes/no +User: "yes" +Agent: [vm_lifecycle(namespace="vms", name="web-server", action="start")] +## ✓ VM Start Successful +VM: `web-server` | Namespace: `vms` | RunStrategy: Always +Impact: Running, consuming resources. Access: virtctl console or SSH +``` + +### Example 2: Stop with User Reconsidering + +``` +User: "Stop the database VM in production namespace" +Agent: [vm-lifecycle-manager skill] +| VM | `database` | Namespace | `production` | Action | `stop` | graceful shutdown | +Confirm: yes/no +User: "wait, no, don't stop it" +Agent: Operation cancelled. VM 'database' in namespace 'production' remains running. +``` + +### Example 3: Restart VM + +``` +User: "Restart test-vm in namespace dev" +Agent: [vm-lifecycle-manager skill] +| VM | `test-vm` | Namespace | `dev` | Action | `restart` | brief interruption ~1-2min | +User: "yes" +Agent: [Composite restart operation] + 1. vm_lifecycle(namespace="dev", name="test-vm", action="stop") + 2. Verify stopped: resources_get → status.printableStatus="Stopped" + 3. Wait 5s + 4. vm_lifecycle(namespace="dev", name="test-vm", action="start") + 5. Verify started: resources_get → status.printableStatus="Running" +## ✓ VM Restart Successful +VM: `test-vm` | Namespace: `dev` | RunStrategy: Always +Impact: Running after stop+start. Brief interruption. Monitor app logs. +``` + +### Example 4: VM Already in Desired State + +``` +User: "Start web-server in namespace vms" +Agent: [vm-lifecycle-manager skill] + [vm_lifecycle(action="start")] +## ℹ️ VM Already Running +VM: `web-server` | Namespace: `vms` | Status: Running +Result: No action taken - VM already in desired state. +To restart: "Restart VM web-server in namespace vms" +``` + +### Example 5: Batch Operations + +``` +User: "Stop VMs web-01, web-02, web-03 in namespace production" +Agent: [vm-lifecycle-manager skill - batch mode] +## Batch Lifecycle Operation +Stopping 3 VMs in 'production': web-01, web-02, web-03 +Impact: All 3 VMs will shut down, services interrupted. +Confirm: yes/no +User: "yes" +Agent: [Executes vm_lifecycle for each VM sequentially] +## ✓ Batch Stop Successful +- web-01: Stopped +- web-02: Stopped +- web-03: Stopped +All VMs stopped. Resources freed. +``` diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/instruction.md b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/instruction.md new file mode 100644 index 00000000..622a3d38 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/instruction.md @@ -0,0 +1,12 @@ +# VM Lifecycle Operations Task + +You are an OpenShift Virtualization administrator. Plan lifecycle operations for VMs in the cluster: stop `web-frontend` and restart `production-db`, both in namespace `prod-vms`. + +## Requirements +- Define the procedure for each operation +- Address the correct sequencing for restart (not a single atomic operation) +- Include verification steps + +Use MCP tools to examine the cluster. Document your methodology, procedures, and verification steps in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/solution/solve.sh b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/solution/solve.sh new file mode 100644 index 00000000..37e96d65 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/solution/solve.sh @@ -0,0 +1,31 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Lifecycle Operations Plan + +## Operation 1: Stop web-frontend +- Tool: `vm_lifecycle(namespace="prod-vms", name="web-frontend", action="stop")` +- Effect: Sets runStrategy to Halted +- Verify: `status.printableStatus` changes to "Stopped" + +## Operation 2: Restart production-db +Restart requires TWO separate calls to avoid resourceVersion conflicts: +1. `vm_lifecycle(namespace="prod-vms", name="production-db", action="stop")` +2. Wait for `status.printableStatus == "Stopped"` (poll every 5 seconds) +3. `vm_lifecycle(namespace="prod-vms", name="production-db", action="start")` + +### RunStrategy Mapping +| Action | RunStrategy Set | +|--------|----------------| +| start | Always | +| stop | Halted | +| restart | Always (after stop completes) | + +### Caveats +- Restart is NOT a single atomic operation — it's stop + wait + start +- Avoid resourceVersion conflicts: use resources_get to verify printableStatus before start +- Graceful shutdown: VM guest agent handles ACPI shutdown signal +- If VM doesn't stop within timeout, force stop may be needed +- Always verify stopped status before issuing start to avoid conflicts +- Consult lifecycle-errors.md for start/stop failures + +REPORT_EOF diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/task.toml b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/task.toml new file mode 100644 index 00000000..29808afd --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-lifecycle-manager" +name = "rh-virt VM Lifecycle Management Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-lifecycle-manager", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/tests/llm_judge.py b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/tests/llm_judge.py new file mode 100644 index 00000000..1e8ef2e1 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "two_step_restart", "file": "/root/report.md", "question": "Does the report implement restart as stop→verify stopped→start rather than a single atomic operation?", "reference": "A skilled report separates stop and start to avoid resourceVersion conflicts. An unskilled report uses a single restart command."}, + {"id": "run_strategy_mapping", "file": "/root/report.md", "question": "Does the report map start to RunStrategy: Always and stop to RunStrategy: Halted?", "reference": "A skilled report uses RunStrategy for lifecycle control. An unskilled report uses power state concepts."}, + {"id": "state_verification", "file": "/root/report.md", "question": "Does the report verify VM reached expected state (Stopped/Running) before proceeding to the next operation?", "reference": "A skilled report verifies printableStatus between operations. An unskilled report assumes instant state changes."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/tests/test.sh b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/tests/test_outputs.py b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/tests/test_outputs.py new file mode 100644 index 00000000..98907dad --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/tests/test_outputs.py @@ -0,0 +1,75 @@ +""" +Tests for rh-virt__vm-lifecycle-manager per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_operations(self): + c = read_report().lower() + assert ("stop" in c or "halt" in c) and ("restart" in c or "start" in c), ( + "report should discuss stop and restart operations" + ) + + def test_mentions_vms(self): + c = read_report().lower() + assert any(t in c for t in ["vm", "virtual machine", "virtualmachine"]), ( + "report should reference the target VMs" + ) + + +class TestSkillDependent: + def test_two_step_restart(self): + """Skill: Restart = stop then start (not single atomic); resourceVersion conflict.""" + c = read_report().lower() + assert ("stop" in c and "start" in c) and any(t in c for t in ["two", "separate", "sequence", "then", "first", "resourceversion", "conflict"]), ( + "should explain restart as stop-then-start, not single operation" + ) + + def test_run_strategy_control(self): + """Skill: RunStrategy Always/Halted for start/stop; not generic power state.""" + c = read_report().lower() + assert any(t in c for t in ["runstrategy", "run strategy", "always", "halted"]) and ( + "start" in c or "stop" in c + ), ( + "should map start/stop to RunStrategy (Always/Halted)" + ) + + def test_ready_verification(self): + """Skill: Verify status.printableStatus Stopped/Running after each step.""" + c = read_report().lower() + assert any(t in c for t in ["printablestatus", "printable status", "status", "stopped", "running"]) and ( + any(t in c for t in ["verify", "check", "poll", "wait", "before start"]) + ), ( + "should verify VM reached expected state before proceeding" + ) + + def test_vm_lifecycle_tool(self): + """Skill: vm_lifecycle MCP tool for start/stop/restart.""" + c = read_report().lower() + assert any(t in c for t in ["vm_lifecycle", "vm lifecycle", "lifecycle tool", "mcp"]), ( + "should reference vm_lifecycle or MCP lifecycle tool" + ) + + def test_restart_composite(self): + """Skill: Restart implemented as stop → verify stopped → wait → start.""" + c = read_report().lower() + has_stop_start = "stop" in c and "start" in c + has_wait = any(t in c for t in ["wait", "5 second", "poll", "verify stopped"]) + assert has_stop_start and has_wait, ( + "should include wait/verify between stop and start for restart" + ) diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/Dockerfile b/evaluation/with_skills/rh-virt__vm-rebalance/environment/Dockerfile new file mode 100644 index 00000000..ae625e01 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/.ai-index/semantic-index.json new file mode 100644 index 00000000..2512c2a3 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/.ai-index/semantic-index.json @@ -0,0 +1,148 @@ +{ + "index_version": "1.0.0", + "last_updated": "2026-02-17", + "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", + + "documents": [ + { + "file": "INDEX.md", + "title": "VM Troubleshooting Guide - Index", + "category": "navigation", + "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], + "use_cases": ["navigation", "tool-reference", "general-guidance"], + "token_estimate": 450, + "skills": ["all"] + }, + { + "file": "scheduling-errors.md", + "title": "VM Scheduling Errors", + "category": "scheduling", + "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], + "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], + "errors_covered": [ + "ErrorUnschedulable - Node Taints", + "ErrorUnschedulable - Insufficient Resources", + "ErrorUnschedulable - Node Selector Mismatch" + ], + "token_estimate": 820, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "storage-errors.md", + "title": "VM Storage Errors", + "category": "storage", + "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], + "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], + "errors_covered": [ + "ErrorDataVolumeNotReady - DataVolume Still Provisioning", + "ErrorDataVolumeNotReady - Storage Class Not Found", + "ErrorDataVolumeNotReady - Insufficient Storage Quota", + "ErrorPvcNotFound", + "Storage Deletion Failures", + "DataVolume Cloning Failures" + ], + "token_estimate": 1350, + "skills": ["vm-creator", "vm-delete", "vm-clone"] + }, + { + "file": "lifecycle-errors.md", + "title": "VM Lifecycle Errors", + "category": "lifecycle", + "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], + "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], + "errors_covered": [ + "VM Stuck in Terminating State", + "VM Won't Start (Non-Scheduling Issues)", + "VM Won't Stop" + ], + "token_estimate": 1150, + "skills": ["vm-delete", "vm-lifecycle-manager"] + }, + { + "file": "runtime-errors.md", + "title": "VM Runtime Errors", + "category": "runtime", + "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], + "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], + "errors_covered": [ + "CrashLoopBackOff" + ], + "token_estimate": 850, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "network-errors.md", + "title": "VM Network Errors", + "category": "networking", + "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], + "use_cases": ["vm-creation", "network-troubleshooting"], + "errors_covered": [ + "Network Attachment Failures" + ], + "token_estimate": 580, + "skills": ["vm-creator"] + } + ], + + "skill_to_docs_mapping": { + "vm-creator": { + "primary_docs": ["scheduling-errors.md", "storage-errors.md"], + "secondary_docs": ["runtime-errors.md", "network-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3650, + "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" + }, + "vm-delete": { + "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 2950, + "description": "VM deletion failures - stuck terminating, storage cleanup" + }, + "vm-clone": { + "primary_docs": ["storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 1800, + "description": "VM cloning failures - DataVolume cloning, storage provisioning" + }, + "vm-lifecycle-manager": { + "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], + "secondary_docs": ["runtime-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3270, + "description": "VM start/stop failures - lifecycle management, scheduling" + }, + "vm-inventory": { + "primary_docs": [], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 450, + "description": "General guidance for status interpretation" + } + }, + + "error_to_docs_mapping": { + "ErrorUnschedulable": "scheduling-errors.md", + "ErrorDataVolumeNotReady": "storage-errors.md", + "ErrorPvcNotFound": "storage-errors.md", + "Terminating": "lifecycle-errors.md", + "CrashLoopBackOff": "runtime-errors.md", + "NetworkAttachmentFailed": "network-errors.md" + }, + + "cross_references": { + "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], + "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], + "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], + "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], + "network-errors.md": ["scheduling-errors.md"] + }, + + "progressive_disclosure_strategy": { + "description": "Load only the minimum documentation needed for current task", + "pattern": "error_type → primary_doc → cross_references (if needed)", + "token_savings": "29% average reduction vs single-file (based on rh-sre data)", + "navigation_overhead_reduction": "85% vs single-file navigation" + } +} diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/INDEX.md new file mode 100644 index 00000000..a2e76963 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/INDEX.md @@ -0,0 +1,332 @@ +--- +title: VM Troubleshooting Guide - Index +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 + - title: OpenShift Virtualization - Virtual Machine Status + url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html + date_accessed: 2026-02-06 + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] +semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] +use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] +last_updated: 2026-02-17 +--- + +# VM Troubleshooting Guide - Index + +## Overview + +This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: + +- **Creation**: Scheduling failures, storage provisioning issues +- **Lifecycle**: Start/stop failures, stuck states +- **Deletion**: Resources stuck in Terminating, storage cleanup failures +- **Cloning**: DataVolume cloning errors, cross-namespace issues +- **Networking**: Secondary network attachment failures +- **Runtime**: CrashLoopBackOff, guest OS boot failures + +This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. + +--- + +## 🗂️ Troubleshooting Categories + +The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: + +### 1. [Scheduling Errors](scheduling-errors.md) +**When to use**: VM fails to schedule on any node + +**Errors covered**: +- ErrorUnschedulable - Node Taints +- ErrorUnschedulable - Insufficient Resources +- ErrorUnschedulable - Node Selector Mismatch + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 2. [Storage Errors](storage-errors.md) +**When to use**: VM has storage provisioning, deletion, or cloning issues + +**Errors covered**: +- ErrorDataVolumeNotReady (all 3 subsections) +- ErrorPvcNotFound +- Storage Deletion Failures +- DataVolume Cloning Failures + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### 3. [Lifecycle Errors](lifecycle-errors.md) +**When to use**: VM has start/stop/termination issues + +**Errors covered**: +- VM Stuck in Terminating State +- VM Won't Start (Non-Scheduling Issues) +- VM Won't Stop + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### 4. [Runtime Errors](runtime-errors.md) +**When to use**: VM repeatedly crashes or fails at runtime + +**Errors covered**: +- CrashLoopBackOff + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 5. [Network Errors](network-errors.md) +**When to use**: VM has secondary network attachment failures + +**Errors covered**: +- Network Attachment Failures (Multus, SR-IOV) + +**Skills that use this**: vm-create + +--- + +## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last + +### Critical Principle: MCP-First Approach + +**All diagnostic procedures in this guide follow the MCP-first pattern**: + +``` +1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) +2. ⚠️ IF FAILS: Ask user permission to use CLI command +3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval +``` + +### Why MCP Tools First? + +- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources +- **Consistency**: Same interface across all operations +- **Better Error Handling**: MCP tools return structured errors +- **Audit Trail**: MCP tool usage is logged and trackable + +### Command Pattern Examples + +Throughout this guide, you'll see diagnostic steps formatted like this: + +**✅ CORRECT Pattern (MCP First)**: + +```markdown +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Extract**: `.metadata.finalizers` from returned JSON + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` +``` + +### Available MCP Tools + +The `openshift-virtualization` MCP server provides these tools: + +**Resource Operations**: +- `resources_get` - Get specific resource (replaces `oc get `) +- `resources_list` - List resources (replaces `oc get `) +- `resources_delete` - Delete resource (replaces `oc delete`) +- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) + +**Pod Operations**: +- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) +- `pods_get` - Get pod details (replaces `oc get pod`) +- `pods_log` - Get pod logs (replaces `oc logs`) +- `pods_exec` - Execute in pod (replaces `oc exec`) +- `pods_delete` - Delete pod (replaces `oc delete pod`) +- `pods_top` - Pod resource usage (replaces `oc top pods`) + +**Events & Monitoring**: +- `events_list` - List events (replaces `oc get events`) +- `nodes_top` - Node resource usage (replaces `oc top nodes`) +- `nodes_log` - Node logs (replaces `oc adm node-logs`) +- `nodes_stats_summary` - Detailed node stats + +**VM Operations** (KubeVirt toolset): +- `vm_create` - Create VMs +- `vm_lifecycle` - Start/stop/restart VMs + +**Namespaces**: +- `namespaces_list` - List namespaces (replaces `oc get namespaces`) +- `projects_list` - List OpenShift projects (replaces `oc get projects`) + +### When CLI Commands Are Required + +Some operations have **NO MCP equivalent** and require CLI: + +- `virtctl` commands (console, VNC access) +- `oc debug node` (node debugging) +- `oc auth can-i` (permission checks) +- `oc adm taint` (node taint management) +- Complex JSON patch operations + +For these, the guide will note: **"CLI Only - No MCP equivalent"** + +### Quick Reference: CLI → MCP Mapping + +| CLI Command | MCP Tool Equivalent | +|-------------|---------------------| +| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | +| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | +| `oc logs -n ` | `pods_log` with name="", namespace="" | +| `oc get events -n ` | `events_list` with namespace="" | +| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | + +**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. + +### How to Read Diagnostic Sections + +Each error section includes: +1. **Symptom** - What you observe +2. **Description** - What's happening +3. **Possible Causes** - Why it's happening +4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback +5. **Solutions** - **MCP tools first**, then CLI fallback +6. **Verification** - **MCP tools first**, then CLI fallback + +**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. + +--- + +## 🔍 Quick Navigation by Skill + +**vm-create**: +- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics +- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning +- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff +- [Network Errors](network-errors.md) - Network attachment failures + +**vm-delete**: +- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state +- [Storage Errors](storage-errors.md) - Storage deletion failures + +**vm-clone**: +- [Storage Errors](storage-errors.md) - DataVolume cloning failures + +**vm-lifecycle-manager**: +- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop +- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling + +**vm-inventory**: +- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed + +--- + +## 📊 Documentation Coverage & Maintenance + +### Current Coverage + +This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: + +- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) +- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures +- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop +- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) +- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems + +**Total errors documented**: 12 error types across 6 categories + +--- + +### Encountering Undocumented Errors + +#### For AI Agents (Claude Code) + +If you encounter an error **not documented** in the categories above: + +1. **Report to user** with all available details (error message, affected resources, namespace) +2. **Provide best-effort diagnostics** using MCP tools: + - `resources_get` to inspect resource status + - `pods_log` to check virt-launcher or compute container logs + - `events_list` to view Kubernetes events +3. **Suggest documentation update**: + ``` + ⚠️ This error is not yet documented in the troubleshooting guide. + + **Error**: + **Affected resource**: / + + I recommend adding this error to the troubleshooting guide: + - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file + - If it's a new error class → Create new category file in docs/troubleshooting/ + + Would you like me to help document this error for future reference? + ``` + +4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs + +--- + +#### For Human Contributors + +To document a new error: + +1. **Determine the category**: + - Scheduling issue → `scheduling-errors.md` + - Storage problem → `storage-errors.md` + - Lifecycle/start/stop → `lifecycle-errors.md` + - Runtime crash → `runtime-errors.md` + - Network issue → `network-errors.md` + - New error class → Create new category file (e.g., `security-errors.md`) + +2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): + - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback + - **Solutions**: MCP Tool approach → Implementation → CLI Fallback + - **Verification**: MCP Tool checks → Expected results → CLI Fallback + +3. **Update semantic index**: + - Edit `.ai-index/semantic-index.json` + - Add error to `error_to_docs_mapping` + - Update relevant skill's `primary_docs` or `secondary_docs` + - Increment token estimates if needed + +4. **Reference in skill files**: + - Update skill's Reference Documentation section + - Ensure skill's Document Consultation steps point to the right category + +5. **Update this INDEX.md**: + - Add error to the appropriate category section (lines 50-120) + - Update "Current Coverage" list above + +--- + +## 📚 Additional Resources + +- [rh-virt README](../../README.md) - MCP server setup and skill documentation +- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources +- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/lifecycle-errors.md new file mode 100644 index 00000000..6167fcc8 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/lifecycle-errors.md @@ -0,0 +1,869 @@ +--- +title: VM Lifecycle Errors +category: kubevirt +sources: + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 +tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] +semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] +use_cases: [vm-deletion, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Lifecycle Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM lifecycle issues including start/stop failures and stuck termination states. + +**When to use this document**: +- VM stuck in `Terminating` state +- VM won't start (runStrategy is Always but VM never reaches Running) +- VM won't stop (runStrategy is Halted but VM never reaches Stopped) + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### VM Stuck in Terminating State + +**Symptom**: VM shows status `Terminating` but deletion never completes + +**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). + +**Possible Causes**: +- Finalizers blocking deletion +- PVC/DataVolume still attached and preventing cleanup +- VirtualMachineInstance (VMI) not terminating properly +- Custom controllers or operators blocking deletion +- Stuck virt-launcher pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check finalizers on the VM**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` + +**2. Check if VMI still exists**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name containing "virt-launcher-". + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check events for deletion issues**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results where `involvedObject.name` == "" and sort by timestamp. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' +``` + +**5. Check VM deletion timestamp**: + +Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. + +If present, VM is in deletion process. If null, VM is not being deleted. + +**Common Finalizer Patterns**: +- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer +- `foregroundDeletion` - Waits for dependent objects to be deleted +- Custom finalizers from operators + +**Solutions** (Use MCP Tools First): + +1. **Wait for dependent resources** (safest approach): + - Use `resources_list` to check if PVCs, DataVolumes are still deleting + - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) + +2. **Force delete VMI** (if VMI is stuck): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable - requires explicit user permission): + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +3. **Force delete virt-launcher pod** (if pod is stuck): + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + # Find the virt-launcher pod name first + oc get pods -n | grep virt-launcher- + + # Then delete it + oc delete pod virt-launcher--xxx -n --force --grace-period=0 + ``` + +4. **Remove finalizers** (⚠️ dangerous - use only as last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` + 2. Remove items from `.metadata.finalizers` array + 3. Update VM using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch not easily done via MCP): + ```bash + # This operation is complex for MCP - may need CLI + oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Check for protection labels** (vm-delete skill specific): + + Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. + + If value is "true", the vm-delete skill refuses deletion (this is expected behavior). + +**Verification**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error if deletion successful. + +**CLI Fallback**: +```bash +oc get vm -n +# Should return: Error from server (NotFound) +``` + +--- + + +--- + +### VM Won't Start (Non-Scheduling Issues) + +**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state + +**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. + +**Possible Causes**: +- Guest OS kernel panic or boot failure +- Cloud-init configuration errors +- Missing or corrupted disk image +- Insufficient memory for guest OS to boot +- QEMU/KVM virtualization errors +- VirtualMachineInstance (VMI) creation failures +- virt-launcher pod crashes + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VMI (VirtualMachineInstance) status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and overall VMI state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**2. Check VMI conditions for errors**: + +Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. View virt-launcher pod logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +First, get pod name from step 3, then: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +Look for QEMU errors, memory allocation failures, device errors. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**5. Check virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. + +Alternatively, use `pods_get` to get full pod details: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**6. Access VM console to see guest OS boot messages**: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +Look for kernel panic, initramfs errors, filesystem errors. + +**7. Check VM events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep -A 20 "Events:" +``` + +**Common Error Patterns**: + +1. **Guest OS Boot Failure**: + - Console shows kernel panic + - Guest hangs at GRUB or boot loader + - Cloud-init errors during first boot + +2. **Insufficient Memory**: + - Guest OS kills processes due to OOM + - VMI logs show memory allocation errors + +3. **QEMU Crashes**: + - virt-launcher logs show QEMU segfaults + - VMI repeatedly restarts + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for kernel panic, initramfs errors, filesystem errors. + +2. **Check virt-launcher pod logs for QEMU errors**: + + Use `pods_log` from diagnostic step 4 to view logs. + + Look for: + - "qemu-system-x86_64: ..." errors + - Memory allocation failures + - Device errors + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx + ``` + +3. **Increase memory if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check VMI memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +4. **Verify disk image integrity**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check PVC is bound: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + If using container disk, verify image exists and is accessible (check VMI spec). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get pvc -n + ``` + +5. **Check cloud-init configuration** (if applicable): + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. + + Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq + ``` + +6. **Restart VMI** (soft reset): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete VMI (VM controller will recreate it): + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for new VMI to start (use `resources_get` to check status). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n -w + ``` + +7. **Check virtualization extensions** (KVM): + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + + # In debug shell: + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should eventually return `Running`). + +Check VMI is running: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should show `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should eventually return: Running + +oc get vmi -n +# Should show: Running +``` + +--- + +### VM Won't Stop + +**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state + +**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. + +**Possible Causes**: +- Guest OS not responding to ACPI shutdown signal +- virt-launcher pod stuck and not terminating +- VirtualMachineInstance (VMI) deletion blocked by finalizers +- Guest shutdown scripts hanging +- Filesystem sync issues in guest OS + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (might show `Stopping`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**2. Check VMI status and deletion timestamp**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check VMI events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vmi -n | grep -A 10 "Events:" +``` + +**5. Check VMI finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.metadata.finalizers}' +``` + +**6. Check if guest is responsive** (if VMI still exists): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Patterns**: +- VMI shows `deletionTimestamp` but never actually deletes +- virt-launcher pod in `Terminating` state +- VM runStrategy is `Halted` but printableStatus shows `Stopping` + +**Solutions** (Use MCP Tools First): + +1. **Wait for graceful shutdown** (default: 30 seconds): + + Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. + +2. **Force stop by deleting VMI**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + This is the standard way to force-stop a VM. + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n + # Should return: Error from server (NotFound) + ``` + +3. **Force delete VMI with grace period** (if VMI won't delete): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +4. **Force delete virt-launcher pod**: + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find the pod using `pods_list_in_namespace` from diagnostic step 3. + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + ⚠️ **Note**: For force deletion with grace period, use CLI fallback. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') + oc delete pod $POD_NAME -n --force --grace-period=0 + ``` + +5. **Remove VMI finalizers** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VMI using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update VMI using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Patch VM runStrategy directly** (ensure consistency): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` (diagnostic step 1) + 2. Set `.spec.runStrategy` to `"Halted"` + 3. Update VM using `resources_create_or_update` with modified JSON + + **CLI Fallback** (merge patch easier via CLI): + Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Stopped` or `Halted`). + +Verify VMI is gone: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Verify virt-launcher pod is gone: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for pods matching `virt-launcher-`. Should return no results. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Stopped or Halted + +oc get vmi -n +# Should return: Error from server (NotFound) + +oc get pods -n | grep virt-launcher- +# Should return: No resources found +``` + +**Prevention**: +- Ensure guest OS has ACPI support enabled +- Use proper shutdown commands in guest OS +- Avoid forceful stops unless necessary (can corrupt guest filesystem) + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/network-errors.md new file mode 100644 index 00000000..82fa313c --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/network-errors.md @@ -0,0 +1,429 @@ +--- +title: VM Network Errors +category: kubevirt +sources: + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] +semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] +use_cases: [vm-creation, network-troubleshooting] +related_docs: [INDEX.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Network Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. + +**When to use this document**: +- VM created successfully but secondary networks not attached +- NetworkAttachmentDefinition not found errors +- Multus CNI failures +- SR-IOV device attachment issues + +**Skills that use this**: vm-create + +--- + +### Network Attachment Failures + +**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working + +**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). + +**Possible Causes**: +- NetworkAttachmentDefinition doesn't exist in the namespace +- Multus CNI not installed or not configured on cluster +- Namespace mismatch (NAD in different namespace than VM) +- Interface name conflicts in VM spec +- Bridge/network configuration errors in NAD +- SR-IOV device not available (if using SR-IOV) + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List NetworkAttachmentDefinitions in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "" +} +``` + +Review `.items[].metadata.name` for available NADs. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -n +``` + +**2. List NetworkAttachmentDefinitions in all namespaces**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition" +} +``` + +Omit `namespace` parameter to list across all namespaces. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -A +``` + +**3. Check specific NetworkAttachmentDefinition**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "", + "name": "" +} +``` + +Review `.spec.config` for CNI configuration. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definition -n -o yaml +``` + +**4. Check VM network configuration**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.networks` to see network references. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq +``` + +**5. Check VM domain interfaces**: + +Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq +``` + +**6. Check virt-launcher pod network annotations** (shows actual attachments): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**7. Check Multus is installed**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "openshift-multus" +} +``` + +Should show Multus CNI pods running. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n openshift-multus +``` + +**8. Check for errors in virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n +``` + +**Common Error Messages**: +- `"network-attachment-definition not found"` - NAD doesn't exist in namespace +- `"multus CNI not configured"` - Multus not installed or misconfigured +- `"interface name conflict"` - Duplicate interface names in VM spec +- `"failed to add network"` - CNI plugin error (check NAD config) +- `"no available devices"` - SR-IOV device not available (if using SR-IOV) + +**Solutions** (Use MCP Tools First): + +1. **Verify NetworkAttachmentDefinition exists in correct namespace**: + + Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. + + If NAD is in different namespace, copy it to VM namespace: + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get NAD from source namespace using `resources_get` + 2. Modify `.metadata.namespace` to target namespace + 3. Create NAD in target namespace using `resources_create_or_update` + + **CLI Fallback** (stream processing easier via CLI): + Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" + ```bash + oc get network-attachment-definition -n -o yaml | \ + sed "s/namespace: /namespace: /" | \ + oc apply -f - + ``` + +2. **Create missing NetworkAttachmentDefinition**: + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + Example: Linux bridge network + + **Parameters**: + ```json + { + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": { + "name": "vlan100", + "namespace": "" + }, + "spec": { + "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" + ```bash + cat < + spec: + config: '{ + "cniVersion": "0.3.1", + "type": "bridge", + "bridge": "br1", + "vlan": 100, + "ipam": { + "type": "host-local", + "subnet": "192.168.100.0/24" + } + }' + EOF + ``` + +3. **Check Multus CNI installation**: + + Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. + + To check cluster network operator: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterOperator" + } + ``` + + Filter for `network` operator. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n openshift-multus + oc get clusteroperators network + ``` + +4. **Fix interface name conflicts** (if VM has duplicate names): + + Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. + + Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. + + **CLI Fallback** (interactive edit easier via CLI): + Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' + oc edit vm -n + ``` + +5. **Validate NAD configuration syntax**: + + Use `resources_get` from diagnostic step 3, extract `.spec.config`. + + Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get network-attachment-definition -n -o jsonpath='{.spec.config}' + ``` + +6. **Check SR-IOV device availability** (if using SR-IOV networks): + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + List SR-IOV network node policies: + + **Parameters**: + ```json + { + "apiVersion": "sriovnetwork.openshift.io/v1", + "kind": "SriovNetworkNodePolicy", + "namespace": "openshift-sriov-network-operator" + } + ``` + + Check SR-IOV device plugin pods: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "openshift-sriov-network-operator" + } + ``` + + Filter for pods with "device-plugin" in name. + + Check available SR-IOV devices on node: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "Node", + "name": "" + } + ``` + + Review `.status.allocatable` for SR-IOV resources. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get sriovnetworknodepolicy -n openshift-sriov-network-operator + oc get pods -n openshift-sriov-network-operator | grep device-plugin + oc describe node | grep -A 10 "Allocatable:" + ``` + +7. **Recreate VM with corrected network configuration** (if needed): + + If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +After remediation, check virt-launcher pod network status: + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +Should show all attached networks with status. Example output: +```json +[ + { + "name": "openshift-sdn", + "interface": "eth0", + "ips": ["10.128.2.10"], + "default": true + }, + { + "name": "vlan100", + "interface": "net1", + "ips": ["192.168.100.5"] + } +] +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**Check from inside VM** (via console): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# In guest OS: +ip addr show +# Should show all network interfaces (eth0, net1, etc.) +``` + +**Common Network Types**: +- **Linux Bridge**: Layer 2 bridge for VLAN networks +- **SR-IOV**: High-performance direct device assignment +- **macvlan**: MAC-based VLAN for container networks +- **OVN-Kubernetes**: OpenShift native overlay network + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/runtime-errors.md new file mode 100644 index 00000000..804d3cbc --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/runtime-errors.md @@ -0,0 +1,616 @@ +--- +title: VM Runtime Errors +category: kubevirt +sources: + - title: Kubernetes Pod Lifecycle + url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ + date_accessed: 2026-02-17 +tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] +semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] +use_cases: [vm-creation, vm-lifecycle, diagnostics] +related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Runtime Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. + +**When to use this document**: +- VM or virt-launcher pod shows `CrashLoopBackOff` status +- virt-launcher pod repeatedly restarting +- Guest OS kernel panics on boot + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### CrashLoopBackOff + +**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting + +**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. + +**Possible Causes**: +- Guest OS kernel panic on boot +- Insufficient resources (memory/CPU) for guest OS +- Corrupted disk image or filesystem +- QEMU/libvirt crashes due to configuration errors +- Missing or incompatible device drivers in guest +- Resource limits too low for virt-launcher pod +- Virtualization features (KVM) not available on node + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check virt-launcher pod restart count**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# Look at RESTARTS column +``` + +**2. View recent crash logs** (previous container instance): + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "previous": true, + "tail": 100 +} +``` + +Look for QEMU errors, kernel panics, or segfaults. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n virt-launcher--xxx --previous +``` + +**3. Check current virt-launcher logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. + +**CLI Fallback** (if MCP unavailable or all containers needed): +```bash +oc logs -n virt-launcher--xxx --all-containers +``` + +**4. Check VMI conditions for crash details**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Extract `.status.conditions` for crash details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**5. Check pod events for crash reasons**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" +``` + +**6. Check pod resource limits**: + +**MCP Tool**: `pods_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +Extract `.spec.containers[0].resources` for resource limits. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq +``` + +**7. Check node kubelet logs for OOM kills**: + +⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +oc adm node-logs -u kubelet | grep -i oom +``` + +**8. Access guest console** (if VM briefly starts): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Crash Patterns**: + +1. **Guest Kernel Panic**: + - Console logs show kernel panic messages + - Guest crashes immediately after boot + - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" + +2. **OOM (Out of Memory)**: + - Pod killed with reason: `OOMKilled` + - Guest runs out of memory during boot or operation + - virt-launcher logs show memory allocation failures + +3. **QEMU Crash**: + - virt-launcher logs show QEMU segmentation fault + - Symptoms: "qemu-system-x86_64: terminated by signal" + - Configuration incompatibility or QEMU bug + +4. **Disk Image Corruption**: + - Guest cannot boot from disk + - Filesystem errors in guest console + - DataVolume import failed + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for kernel panic or boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for: + - Kernel panic messages + - Initramfs errors + - Filesystem mounting failures + - Missing device errors + +2. **Review virt-launcher crash logs**: + + Use `pods_log` with `previous: true` from diagnostic step 2. + + Look for: + - QEMU command line errors + - Device initialization failures + - Memory allocation errors + - Signal termination (SIGSEGV, SIGABRT) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx --previous + ``` + +3. **Check for OOM (Out of Memory) kills**: + + **MCP Tool**: `pods_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + Extract `.status.containerStatuses[0].lastState.terminated.reason`. + + If returns `"OOMKilled"`: + - Option 1: Increase virt-launcher memory limits + - Option 2: Decrease guest memory allocation + - Option 3: Use smaller instance type + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' + ``` + +4. **Increase resources if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check current memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +5. **Verify disk image integrity**: + + **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters for list**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" + } + ``` + + **Parameters for specific DV**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check `.status.phase` (should be `Succeeded`). + + If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get datavolume -n -o jsonpath='{.status.phase}' + ``` + +6. **Check virtualization (KVM) availability**: + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +7. **Simplify VM configuration** (eliminate variables): + + Try creating minimal VM using vm-create skill with: + - Small instance type + - No secondary networks + - Simple container disk (e.g., Fedora) + - No cloud-init + + If minimal VM works, add features back one by one. + +8. **Recreate VM with different workload** (test disk image): + + If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) + +After remediation, check pod restart count stops increasing: + +**Parameters for pods**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). + +Check VM reaches Running state: + +**Parameters for VM**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# RESTARTS should stabilize + +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Running +``` + +Verify guest is responsive: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# Should show login prompt or OS console +``` + +**Advanced Debugging**: + +**MCP Tool**: `pods_exec` (from openshift-virtualization) + +Check libvirt domain XML: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["virsh", "dumpxml", "1"] +} +``` + +Check QEMU process: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["ps", "aux"] +} +``` + +Filter output for "qemu" process. + +**CLI Fallback** (if MCP unavailable): +```bash +oc exec -n virt-launcher--xxx -- virsh dumpxml 1 +oc exec -n virt-launcher--xxx -- ps aux | grep qemu +``` + +**Prevention**: +- Start with minimal VM configuration and add complexity gradually +- Use recommended instance types for your workload +- Test disk images before deploying to production +- Ensure nodes have adequate resources and KVM support +- Monitor resource usage and set appropriate limits + +--- + +## Workaround Patterns for MCP Tool Limitations + +### General Pattern: Diagnose → Propose → Confirm → Execute + +When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): + +**1. Diagnose** the root cause using Kubernetes commands +```bash +oc describe vm -n +oc get events -n --field-selector involvedObject.name= +``` + +**2. Propose** a manual workaround to the user with clear explanation +- Explain why the issue occurred +- Show what will be changed +- List alternative options + +**3. Confirm** with user before executing (Human-in-the-Loop) +- Wait for explicit "yes" or "apply workaround" +- Never auto-execute modifications + +**4. Execute** the workaround using oc/kubectl +```bash +oc patch vm -n --type=merge -p '...' +``` + +**5. Verify** the fix was successful +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**6. Document** the limitation and suggest filing enhancement request +- Note this is temporary until MCP tool is enhanced +- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues + +### Example: Adding Tolerations Workaround + +**Diagnostic Output**: +```markdown +## ⚠️ VM Scheduling Issue Detected + +**Root Cause**: Node taints prevent VM scheduling + +**Details**: +- Found 3 nodes with taint: `virtualization=true:NoSchedule` +- VM spec does not include matching tolerations +- This prevents VM scheduling on virtualization-dedicated nodes +``` + +**Proposed Workaround**: +```bash +oc patch vm web-server -n vms --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' +``` + +**User Confirmation Required**: +``` +How would you like to proceed? +- "apply workaround" - I'll patch the VM with tolerations +- "manual" - I'll provide instructions for you to apply manually +- "cancel" - Delete the VM and abort creation +``` + +**After Execution**: +```markdown +## ✓ Workaround Applied Successfully + +**Action**: Added tolerations for taint `virtualization=true:NoSchedule` +**New Status**: Stopped (VM can now be scheduled) + +**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. +Future VMs in this cluster will need the same fix until the tool is enhanced. +``` + +--- + +## VM Status Reference + +### Status Values + +| Status | Meaning | Action Required | +|--------|---------|-----------------| +| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | +| `Running` | VM is running | Normal | +| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | +| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | +| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | +| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | +| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | +| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | +| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | +| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | + +### Checking VM Status + +```bash +# Get printable status +oc get vm -n -o jsonpath='{.status.printableStatus}' + +# Get detailed status and conditions +oc get vm -n -o jsonpath='{.status}' | jq + +# Watch status changes in real-time +oc get vm -n -w +``` + +--- + +## Best Practices for Agents + +When implementing diagnostic workflows: + +1. **Always verify VM status** after creation (wait 5-10 seconds first) +2. **Consult this document** when encountering error status values +3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) +4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) +5. **Respect human-in-the-loop** for all VM modifications +6. **Document temporary workarounds** and their limitations clearly +7. **Suggest filing issues** for missing MCP tool features + +### Document Consultation Pattern + +```markdown +**Document Consultation** (REQUIRED): +1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes +2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." +``` + +--- + +## Known MCP Tool Limitations + +### vm_create tool + +**Currently Supported**: +- ✓ Namespace, name (required) +- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) +- ✓ Size hints (small, medium, large) +- ✓ Storage size +- ✓ Autostart flag +- ✓ Networks (Multus NetworkAttachmentDefinitions) +- ✓ Performance family (u1, o1, c1, m1) +- ✓ Instance type, preference + +**Not Currently Supported** (requires workarounds): +- ✗ Tolerations (for node taints) +- ✗ Node selectors +- ✗ Affinity/anti-affinity rules +- ✗ Resource requests/limits (beyond instance type) +- ✗ Custom labels/annotations +- ✗ SSH keys injection +- ✗ Cloud-init user data + +**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. + +**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues + +--- + +## Additional Resources + +- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) +- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) +- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) +- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) + +--- + +[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/scheduling-errors.md new file mode 100644 index 00000000..e47c8c4f --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/scheduling-errors.md @@ -0,0 +1,417 @@ +--- +title: VM Scheduling Errors +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 +tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] +semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] +use_cases: [vm-creation, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Scheduling Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**When to use this document**: +- VM shows status `ErrorUnschedulable` after creation or start attempt +- VM events mention scheduling failures, taints, resources, or node selectors + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +## ErrorUnschedulable + +**Symptom**: VM shows status `ErrorUnschedulable` after creation + +**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**Possible Causes**: + +### 1. Node Taints (Most Common) + +Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM events for scheduling failures**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `involvedObject.name` == "" and look for messages like: +- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= +``` + +**2. Check node taints in the cluster**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' +``` + +**Common Taint Patterns**: +- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule +- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes +- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads + +**Solution - Add Tolerations to VM**: + +The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. + +**Workaround (post-creation using MCP Tools)**: + +**Step 1**: Get current VM spec + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Step 2**: Modify the returned JSON to add tolerations + +Add to `.spec.template.spec.tolerations`: +```json +{ + "tolerations": [ + { + "key": "virtualization", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule" + } + ] +} +``` + +**Step 3**: Update VM with modified spec + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +```json +{ + "resource": "" +} +``` + +Pass the complete modified VM resource as YAML or JSON string. + +**Step 4**: Verify tolerations were added + +Use `resources_get` again and check `.spec.template.spec.tolerations` in response. + +**Step 5**: Check if VM status improved + +Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. + +**CLI Fallback** (if MCP patch is too complex): +```bash +# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' + +# Verify tolerations +oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq + +# Check status +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**Example - Multiple Tolerations**: +```bash +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: "dedicated" + operator: "Equal" + value: "virt-workloads" + effect: "NoSchedule" +' +``` + +**Toleration Operators**: +- `Equal` - Key and value must match exactly +- `Exists` - Only key must exist (ignores value) + +**Toleration Effects**: +- `NoSchedule` - Don't schedule new pods (existing pods continue) +- `PreferNoSchedule` - Avoid scheduling if possible +- `NoExecute` - Don't schedule AND evict existing pods + +**Alternative Solutions**: +1. **Remove node taints** (if you have cluster-admin access): + ```bash + oc adm taint nodes virtualization=true:NoSchedule- + ``` + +2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits + +3. **File enhancement request** - Request tolerations support in openshift-mcp-server: + https://github.com/openshift/openshift-mcp-server/issues + +--- + +### 2. Insufficient Resources + +Not enough CPU, memory, or storage available on any node. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM resource requests**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' +``` + +**2. Check node resource availability**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. + +Alternatively, use `nodes_top` MCP tool for current resource usage. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe nodes | grep -A 5 "Allocated resources" +``` + +**3. Look for VM events mentioning "Insufficient"**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep "Insufficient" +``` + +**Example Event**: +``` +0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. +``` + +**Solutions** (Use MCP Tools First): + +1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) +2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill +3. **Delete unused VMs** - Use vm-delete skill to free up resources +4. **Check resource quotas**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters for quota**: + ```json + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" + } + ``` + + **Parameters for limit range**: + ```json + { + "apiVersion": "v1", + "kind": "LimitRange", + "namespace": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + oc describe limitrange -n + ``` + +--- + +### 3. Node Selector Mismatch + +VM requires specific node labels that don't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM node selector requirements**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.nodeSelector` to see required node labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' +``` + +**2. List available node labels**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.metadata.labels` for available labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes --show-labels +``` + +**3. Check if any nodes match the selector**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node", + "labelSelector": "=" +} +``` + +Should return at least one node with matching labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -l = +``` + +**Solutions** (Use MCP Tools First): + +**Option 1: Remove node selector from VM** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get current VM using `resources_get` (diagnostic step 1) +2. Remove `.spec.template.spec.nodeSelector` field +3. Update VM using `resources_create_or_update` with modified JSON + +**CLI Fallback** (JSON patch easier via CLI): +Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" +```bash +oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' +``` + +**Option 2: Add label to nodes** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get node using `resources_get` +2. Add label to `.metadata.labels` +3. Update node using `resources_create_or_update` + +⚠️ **Note**: Node labeling typically requires cluster admin privileges. + +**CLI Fallback** (simpler via CLI): +Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" +```bash +oc label node = +``` + +--- + +[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/storage-errors.md new file mode 100644 index 00000000..16ba584d --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/storage-errors.md @@ -0,0 +1,1011 @@ +--- +title: VM Storage Errors +category: kubevirt +sources: + - title: KubeVirt CDI - DataVolumes + url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ + date_accessed: 2026-02-17 + - title: Kubernetes Persistent Volumes + url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ + date_accessed: 2026-02-17 +tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] +semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] +use_cases: [vm-creation, vm-deletion, vm-cloning] +related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] +last_updated: 2026-02-17 +--- + +# VM Storage Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. + +**When to use this document**: +- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` +- Storage deletion fails after VM deletion +- DataVolume cloning operations fail +- PVC provisioning issues + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### ErrorDataVolumeNotReady + +**Symptom**: VM shows status `ErrorDataVolumeNotReady` + +**Description**: The DataVolume (persistent storage) backing the VM is not ready. + +**Possible Causes**: + +#### 1. DataVolume Still Provisioning + +Storage provisioning takes time, especially for large disks or when importing images. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Get detailed DataVolume information**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` and `.status.conditions` for provisioning details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o json +``` + +**3. Check PVC (PersistentVolumeClaim) bound status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Check `.status.phase` for each PVC (should be `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. + +#### 2. Storage Class Not Found + +The requested storage class doesn't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List available storage classes**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" +} +``` + +Review the list of available storage classes (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass +``` + +**2. Check DataVolume's requested storage class**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.spec.pvc.storageClassName` in the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' +``` + +**Solution**: +1. Use a valid storage class from the cluster +2. Recreate VM with correct storage class parameter + +#### 3. Insufficient Storage Quota + +Namespace has insufficient storage quota to provision the PVC. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check resource quotas**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**2. Check storage usage**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase +``` + +**Solution**: +1. Request quota increase from cluster admin +2. Delete unused PVCs to free quota +3. Reduce VM storage size + +--- + +### ErrorPvcNotFound + +**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List PVCs in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review the list of available PVCs (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check VM's PVC references**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' +``` + +**Solution**: +- Wait for DataVolume to create the PVC +- Manually create missing PVC +- Fix VM spec to reference correct PVC name + +--- + + +--- + +### Storage Deletion Failures + +**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace + +**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. + +**Possible Causes**: +- PVC still bound to active PersistentVolume with `Retain` policy +- DataVolume still being referenced by another resource +- CDI (Containerized Data Importer) controller issues +- Storage class retention policy preventing deletion +- Finalizers on PVC/DataVolume blocking cleanup +- PVC still mounted by a pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check PVC status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check specific PVC phase**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Released` or `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o jsonpath='{.status.phase}' +``` + +**3. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**4. Check what's using the PVC**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' +``` + +**5. Check PVC finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. + +**6. Check DataVolume finalizers**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.metadata.finalizers}' +``` + +**7. Check PV reclaim policy**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume" +} +``` + +Filter results for PV where `.spec.claimRef.name` matches ``. + +To get specific PV policy: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume", + "name": "" +} +``` + +Check `.spec.persistentVolumeReclaimPolicy`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pv | grep +oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' +``` + +**Common Finalizer Patterns**: +- `kubernetes.io/pvc-protection` - Protects PVC while in use +- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer + +**Solutions** (Use MCP Tools First): + +1. **Delete DataVolume first, then PVC**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete DataVolume first (often blocks PVC deletion): + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Wait a few seconds, then delete PVC: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete datavolume -n + oc delete pvc -n + ``` + +2. **Check for pods still using PVC**: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "" + } + ``` + + Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. + + Then delete the pods using `pods_delete`: + + **Parameters**: + ```json + { + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' + oc delete pod -n + ``` + +3. **Force delete PVC** (if safe to do so): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete pvc -n --grace-period=0 --force + ``` + +4. **Remove finalizers from PVC** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PVC using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update PVC using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Remove finalizers from DataVolume** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current DataVolume using `resources_get` (diagnostic step 6) + 2. Remove items from `.metadata.finalizers` array + 3. Update DataVolume using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Change PV reclaim policy** (if PV has Retain policy): + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PV policy using `resources_get` (diagnostic step 7) + 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` + 3. Update PV using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" + ```bash + oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' + oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' + ``` + +**Storage Quota Check** (Use MCP Tools First): + +After deletion, verify storage quota is freed: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters for quota check**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.used` to verify storage quota is freed. + +**Parameters for PVC verification**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +oc get pvc -n +``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Confirm PVC is deleted: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Confirm DataVolume is deleted: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +# Should return: Error from server (NotFound) + +oc get datavolume -n +# Should return: Error from server (NotFound) +``` + +--- + +### DataVolume Cloning Failures + +**Symptom**: VM clone created successfully but DataVolume clone operation fails + +**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. + +**Possible Causes**: +- CSI driver doesn't support volume cloning +- Source PVC storage class incompatible with cloning +- Cross-namespace cloning not permitted by storage backend +- Insufficient storage quota in target namespace +- Source PVC not in `Bound` state +- Storage class doesn't have volume cloning enabled +- CDI (Containerized Data Importer) controller issues + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume clone status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and `.metadata.name`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Check DataVolume events for errors**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe datavolume -n +``` + +**3. Check DataVolume phase**: + +Use `resources_get` from step 1, extract `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +``` + +**4. Check if storage class supports cloning**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "name": "" +} +``` + +Review the full YAML output for cloning-related configurations. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass -o yaml | grep -A 5 -i clone +``` + +**5. Check CSI driver capabilities**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "CSIDriver" +} +``` + +Review `.items[].metadata.name` for available CSI drivers. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get csidriver +``` + +**6. Check source PVC status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Bound` for cloning to work). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**7. Check target namespace storage quota**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**8. Check CDI controller logs**: + +**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) + +First, list pods in openshift-cnv namespace: + +**Parameters for pods_list_in_namespace**: +```json +{ + "namespace": "openshift-cnv", + "labelSelector": "app.kubernetes.io/component=cdi-deployment" +} +``` + +Then get logs using `pods_log`: + +**Parameters**: +```json +{ + "namespace": "openshift-cnv", + "name": "", + "tail": 100 +} +``` + +**CLI Fallback** (if MCP unavailable or easier via CLI): +```bash +oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') +``` + +**Common Error Messages**: +- `"volume cloning is not supported"` - CSI driver lacks clone capability +- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage +- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace +- `"insufficient quota"` - Target namespace lacks storage quota +- `"source PVC not bound"` - Source PVC must be in Bound state for cloning +- `"StorageClass does not support cloning"` - Storage class configuration issue + +**Solutions** (Use MCP Tools First): + +1. **Check storage class clone support**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" + } + ``` + + For each storage class in `.items[]`, check: + - `.metadata.name` (storage class name) + - `.provisioner` (CSI driver) + + Storage classes using CSI drivers typically support cloning. Look for provisioners like: + - `csi.ovirt.org` (oVirt CSI) + - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) + - `ebs.csi.aws.com` (AWS EBS CSI) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner + ``` + +2. **Verify source PVC is bound**: + + Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pvc -n -o jsonpath='{.status.phase}' + ``` + +3. **Check target namespace quota**: + + Use `resources_list` from diagnostic step 7 to check quota. + + If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + ``` + +4. **Use snapshot-based cloning** (alternative method): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Step 1**: Create VolumeSnapshot of source PVC + + **Parameters**: + ```json + { + "apiVersion": "snapshot.storage.k8s.io/v1", + "kind": "VolumeSnapshot", + "metadata": { + "name": "-snapshot", + "namespace": "" + }, + "spec": { + "source": { + "persistentVolumeClaimName": "" + } + } + } + ``` + + **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) + + **Step 3**: Create new DataVolume from snapshot + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": "-rootdisk", + "namespace": "" + }, + "spec": { + "source": { + "snapshot": { + "name": "-snapshot", + "namespace": "" + } + }, + "storage": { + "resources": { + "requests": { + "storage": "50Gi" + } + }, + "storageClassName": "" + } + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" + ```bash + cat <-snapshot + namespace: + spec: + source: + persistentVolumeClaimName: + EOF + + oc get volumesnapshot -snapshot -n + + cat <-rootdisk + namespace: + spec: + source: + snapshot: + name: -snapshot + namespace: + storage: + resources: + requests: + storage: 50Gi + storageClassName: + EOF + ``` + +5. **Use "new empty storage" option** (vm-clone skill): + - If cloning isn't supported, create VM with empty storage + - Manually copy data if needed + +6. **Cross-namespace cloning workaround**: + - Some storage backends require snapshot for cross-namespace cloning + - Create snapshot in source namespace, restore in target namespace (see solution 4 above) + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Check DataVolume reached Succeeded phase: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should return `Succeeded`). + +Check PVC was created and bound: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "-rootdisk" +} +``` + +Check `.status.phase` (should return `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +# Should return: Succeeded + +oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' +# Should return: Bound +``` + +**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/mcp-servers/mock-virt-mcp.py b/evaluation/with_skills/rh-virt__vm-rebalance/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..2e083d72 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1458 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/REBALANCE_AUTOMATIC.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/REBALANCE_AUTOMATIC.md new file mode 100644 index 00000000..936a4f3b --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/REBALANCE_AUTOMATIC.md @@ -0,0 +1,760 @@ +# Automatic Rebalancing Strategy + +**Status**: ✅ PRODUCTION READY + +**Purpose**: AI-driven rebalancing where user explains high-level goals (CPU balance, memory optimization, drain node, etc.) and AI generates optimal rebalance plan. User can modify or approve plan before execution. + +--- + +## When to Use Automatic Mode + +Use this mode when the user wants: +- AI to analyze cluster and propose optimal rebalancing +- High-level goal specification (balance CPU, optimize memory, drain node) +- Expert recommendations with ability to customize +- Multi-objective optimization (CPU AND memory simultaneously) +- Intelligent rebalance planning without manual VM-by-node decisions + +**User Request Patterns:** +- "Rebalance VMs based on CPU load" +- "Optimize cluster for CPU and memory" +- "Drain worker-02 for maintenance" +- "Automatically balance the cluster" +- "Help me redistribute VMs to improve performance" +- "Optimize VM placement" + +**Do NOT use Automatic mode when:** +- User specifies exact VM→node mappings → Use Manual mode +- User only wants to see available VMs → Use `/vm-inventory` skill + +--- + +## Workflow + +### Step 1: Gather Cluster State and Determine Optimization Goal + +**1.1 Collect Cluster Information** + +**List all VMs across namespaces:** + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine" +} +``` + +Extract for each VM: +- Name, namespace +- Current node placement (from VirtualMachineInstance if running) +- Resource requests (CPU, memory) +- Storage type (RWX vs RWO) - determines live vs cold migration capability + +**List all nodes:** + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +Extract for each node: +- Name, status (Ready/NotReady) +- Capacity and allocatable resources +- Current utilization +- Taints and labels +- Schedulable status (not cordoned) + +**Gather resource usage metrics:** + +**MCP Tool**: `nodes_top` (from openshift-virtualization) + +**Parameters**: None (lists all nodes) + +Extract current CPU and memory utilization for each node. + +**MCP Tool**: `pods_top` (from openshift-virtualization) + +**Parameters**: +```json +{ + "all_namespaces": true, + "label_selector": "kubevirt.io=virt-launcher" +} +``` + +Extract current CPU and memory usage for each VM. + +--- + +**1.2 Determine Optimization Goal from User Request** + +**Analyze user's language to infer goal:** + +| User Phrase | Optimization Goal | Metrics to Optimize | +|-------------|-------------------|---------------------| +| "balance CPU", "CPU load" | Balance CPU utilization | Minimize CPU variance across nodes | +| "optimize memory", "memory pressure" | Balance memory utilization | Minimize memory variance across nodes | +| "balance cluster", "rebalance", "optimize" | Multi-objective (CPU + memory) | Minimize both CPU and memory variance | +| "drain worker-02", "evacuate node-X" | Drain specific node | Migrate all VMs off target node | +| "optimize performance" | Performance optimization | Balance resources + avoid hotspots | +| "distribute VMs evenly" | VM count distribution | Equal number of VMs per node | + +**If goal is ambiguous**, ask user to clarify: + +``` +I can optimize the cluster for several goals: +1. **CPU load balancing** - Distribute CPU usage evenly across nodes +2. **Memory load balancing** - Distribute memory usage evenly across nodes +3. **Both CPU and memory** - Multi-objective optimization +4. **Drain specific node** - Move all VMs off a node for maintenance +5. **VM count distribution** - Equal number of VMs per node + +Which optimization goal would you like me to pursue? +``` + +**WAIT for user response** before proceeding. + +--- + +**1.3 Support Multi-Objective Optimization** + +When user requests multiple goals (e.g., "balance CPU and memory"): + +**Approach:** +1. **Calculate scoring function** combining all objectives +2. **Weight objectives** (can ask user for priorities or use defaults) +3. **Find rebalance plan** that optimizes combined score + +**Example Scoring:** +``` +Score = (0.5 × CPU_variance_reduction) + (0.5 × Memory_variance_reduction) +``` + +**User can adjust weights** if AI proposes alternative approach: + +``` +I can optimize for: +- Equal priority: CPU 50%, Memory 50% +- CPU-focused: CPU 70%, Memory 30% +- Memory-focused: CPU 30%, Memory 70% + +Would you like to adjust priorities, or proceed with equal weighting? +``` + +--- + +### Step 2: Analyze and Generate Optimal Migration Plan + +**2.1 Identify Migration Candidates** + +For each optimization goal: + +**CPU Balancing:** +- Identify overloaded nodes (>80% CPU) +- Identify underloaded nodes (<50% CPU) +- Select VMs to migrate from overloaded to underloaded nodes + +**Memory Balancing:** +- Identify nodes with high memory pressure (>85%) +- Identify nodes with low memory usage (<50%) +- Select VMs to migrate for better distribution + +**Node Drain:** +- Select ALL VMs currently on target node +- Find suitable destination nodes with capacity + +**Performance Optimization:** +- Identify VMs with high resource variance (bursty workloads) +- Distribute high-performance VMs across different nodes +- Avoid co-locating resource-intensive VMs + +**2.2 Apply Constraints and Validation** + +For each candidate migration, check: + +**Storage Compatibility** (see SKILL.md - Common Validation Logic): +- RWX storage → Live migration possible +- RWO storage → Cold migration required +- Stopped VM → Cold migration required + +**Target Node Capacity:** +- Verify target has sufficient CPU and memory +- Account for VM resource requests +- Ensure node is Ready and schedulable + +**Taints and Tolerations:** +- Check target node for taints +- Verify VM has matching tolerations +- If mismatch, skip that target or propose adding tolerations + +**Concurrency Limits** (see [references/performance-tuning.md](./references/performance-tuning.md)): +- Cluster limit: 5 concurrent migrations (default) +- Per-node limit: 2 outbound migrations (default) +- Plan migration batches respecting limits + +**Network Bandwidth:** +- Avoid saturating network with too many concurrent large VM migrations +- Consider VM memory size when scheduling concurrent migrations + +**2.3 Optimize Migration Plan** + +**Migration Ordering Strategy:** + +1. **Smallest VMs first** - Faster migrations, higher success rate +2. **Live migrations before cold** - Minimize total downtime +3. **Group by source node** - Efficient for node draining +4. **Respect dependencies** - Avoid migrating related VMs simultaneously (e.g., database + app tier) + +**Expected Improvement Calculation:** + +**Before migration:** +``` +CPU variance = StdDev([worker-01: 85%, worker-02: 78%, worker-03: 42%, worker-04: 38%]) + = 22.1% +``` + +**After migration:** +``` +CPU variance = StdDev([worker-01: 65%, worker-02: 58%, worker-03: 62%, worker-04: 55%]) + = 4.2% +``` + +**Improvement:** +``` +Variance reduction: 22.1% → 4.2% = 81% improvement +``` + +--- + +### Step 3: Present Plan to User with Modification Options + +**CRITICAL**: Use the **Common Plan Visualization** format defined in SKILL.md. + +**Reference**: [SKILL.md - Common Plan Visualization](./SKILL.md#common-plan-visualization) + +**Present the plan with:** +1. **Optimization Goal** statement +2. **VM Rebalance Plan** table (from SKILL.md common format) +3. **Node State: Before → After** table (from SKILL.md common format) +4. **Key Improvement** summary line +5. **Migration Summary** (for batch operations) +6. **Risks & Considerations** (optional) +7. **User options** (approve / modify / explain / cancel) + +**Additional context for automatic mode:** +- State the optimization goal clearly (CPU balance, memory balance, multi-objective, node drain, etc.) +- Include expected improvements with metrics (variance reduction, capacity headroom, etc.) +- Offer modification options (skip, change target, reorder) +- Allow user to explain reasoning for specific migrations + +**CRITICAL: WAIT for user response before executing.** + +--- + +**3.1 Handle User Modifications** + +If user selects **"modify"**, enter interactive modification mode: + +**Example Interaction:** + +``` +User: "modify" + +Agent: "Which migrations would you like to modify?" + +Options: +- "skip <#>" - Remove migration from plan (e.g., "skip 5") +- "change target <#> to " - Change destination (e.g., "change target 1 to worker-04") +- "reorder" - Change execution order +- "done" - Finish modifications and execute + +User: "skip 5" + +Agent: "Migration #5 (app-vm-05) removed from plan. Updated plan:" + +## Updated Rebalancing Plan + +**Proposed Migrations:** (Now 4 migrations) + +| # | VM | Current Node | Target Node | Type | Impact | +|---|----|--------------|-------------|------|--------| +| 1 | database-01 | worker-01 | worker-03 | live | <1s | +| 2 | web-server-02 | worker-01 | worker-04 | live | <1s | +| 3 | cache-vm | worker-02 | worker-04 | live | <1s | +| 4 | api-gateway | worker-02 | worker-03 | live | <1s | + +**Updated Expected Improvements:** +- CPU variance: 22.1% → 5.8% (74% improvement) +- Memory variance: 11.8% → 8.1% (31% improvement) + +**Ready to execute?** (approve / modify more / cancel) + +User: "approve" + +Agent: [Proceeds to Step 4 - Execution] +``` + +--- + +**3.2 AI Proposes Better Approach (If Detected)** + +If AI detects a potentially better strategy, propose alternative: + +``` +💡 **Alternative Approach Detected** + +I notice that migrating app-vm-05 via cold migration will cause 40s downtime. However, I found: +- app-vm-05 is currently stopped (not running) +- Moving it now via cold migration has **zero additional downtime** since it's already offline + +**Alternative proposal:** +Include app-vm-05 in the plan (no additional impact vs current state) + +Would you like to: +- **accept alternative** - Include app-vm-05 in plan +- **keep original** - Proceed with current plan +- **explain more** - Tell me more about this alternative +``` + +**User has final word** - If user prefers original plan, execute original plan. + +--- + +### Step 4: Validate All VMs Before Execution + +**BEFORE executing any migration**, validate ALL VMs in the plan: + +**For each VM**, perform **Common Validation Logic from SKILL.md**: + +1. **Verify VM exists** (see SKILL.md - Validation 1) +2. **Check current location** (see SKILL.md - Validation 2) +3. **Validate storage compatibility** (see SKILL.md - Validation 3) +4. **Verify target node exists** (see SKILL.md - Validation 4) + +**If any VM fails validation:** +- Remove from rebalance plan +- Warn user: "Migration #X (vm-name) failed validation: [reason]. Proceeding with remaining migrations." +- Continue with other migrations + +**Reference**: [SKILL.md - Common Validation Logic](./SKILL.md#common-validation-logic) + +--- + +### Step 5: Execute Migrations with Progress Reporting + +**5.1 Group Migrations by Type** + +**Live Migrations** (execute first): +- Can run concurrently (up to cluster limits) +- Lower risk, zero downtime +- Follow live migration workflow from REBALANCE_MANUAL.md + +**Cold Migrations** (execute after live migrations): +- Run sequentially (to prevent cascading failures) +- Higher risk, has downtime +- Follow cold migration workflow from REBALANCE_MANUAL.md + +**5.2 Respect Concurrency Limits** + +**Cluster-wide limit**: 5 concurrent migrations (default) +**Per-node limit**: 2 outbound migrations per source node (default) + +**Monitor current migrations:** + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration" +} +``` + +Count migrations where `.status.phase` is NOT "Succeeded" or "Failed". + +**Wait if at limit** before starting new migrations. + +**Reference**: [references/performance-tuning.md](./references/performance-tuning.md#concurrency-limits-tuning) + +--- + +**5.3 Execute Each Migration** + +**For Live Migrations:** + +**Create VirtualMachineInstanceMigration:** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +```json +{ + "resource": "apiVersion: kubevirt.io/v1\nkind: VirtualMachineInstanceMigration\nmetadata:\n name: migrate--\n namespace: \nspec:\n vmiName: " +} +``` + +**Monitor migration progress:** + +Poll using `resources_get` for VirtualMachineInstanceMigration, checking `.status.phase`: +- Pending → Scheduling → PreparingTarget → Running → Succeeded + +**For Cold Migrations:** + +Follow cold migration workflow from REBALANCE_MANUAL.md: +1. Stop VM using `vm_lifecycle` (action: stop) and wait for completion +2. Re-read VM using `resources_get` for fresh resourceVersion +3. Update VM nodeAffinity to target node +4. Start VM using `vm_lifecycle` (action: start) +5. Verify VM reached target node + +**Reference**: [REBALANCE_MANUAL.md - Cold Migration Workflow](./REBALANCE_MANUAL.md) + +--- + +**5.4 Report Progress Incrementally** + +After each migration completes: + +```markdown +## Automatic Rebalancing in Progress + +**Status:** 2/4 migrations complete + +✓ **database-01**: Migrated to worker-03 (live, 42s) +✓ **web-server-02**: Migrated to worker-04 (live, 38s) +⏳ **cache-vm**: Migrating to worker-04 (live, in progress - 15s elapsed) +⏸️ **api-gateway**: Pending (waiting for cache-vm to complete) + +**Estimated time remaining:** 2-3 minutes +``` + +Update after each completion/start. + +--- + +**5.5 Error Handling During Execution** + +**On first failure:** + +1. **Stop remaining migrations** (do not continue blindly) +2. **Report detailed status**: + +```markdown +## ⚠️ Rebalancing Paused - Migration Failed + +**Status:** 2/4 successful, 1 failed, 1 not attempted + +**Successful:** +- ✓ database-01: Migrated to worker-03 (live, 42s) +- ✓ web-server-02: Migrated to worker-04 (live, 38s) + +**Failed:** +- ❌ cache-vm: Migration timeout (VM memory write rate too high) + +**Not Attempted:** +- ⏸️ api-gateway: Skipped due to previous failure + +**Troubleshooting:** + +Migration timeout typically occurs when: +- VM has high memory write rate (database, caching workload) +- Network bandwidth insufficient for transfer speed +- VM memory size very large (>32GB) + +**Recommendations:** +1. Reduce workload on cache-vm and retry +2. Use cold migration for cache-vm (will have ~30-60s downtime) +3. Check network bandwidth availability + +**How would you like to proceed?** +- **retry** - Retry failed migration with same settings +- **retry cold** - Retry using cold migration instead +- **skip** - Skip cache-vm and continue with api-gateway +- **abort** - Stop rebalancing, leave cluster in current state +``` + +3. **Wait for user decision** before proceeding + +**Reference**: [SKILL.md - Common Error Handling](./SKILL.md#common-error-handling) + +--- + +### Step 6: Report Final Results + +**On Complete Success:** + +```markdown +## ✓ Automatic Rebalancing Complete + +**All migrations successful!** + +**Executed Migrations:** + +| VM | From | To | Type | Duration | Status | +|----|------|----|----|----------|--------| +| database-01 | worker-01 | worker-03 | live | 42s | ✓ Success | +| web-server-02 | worker-01 | worker-04 | live | 38s | ✓ Success | +| cache-vm | worker-02 | worker-04 | live | 35s | ✓ Success | +| api-gateway | worker-02 | worker-03 | live | 41s | ✓ Success | + +**Cluster State: Before → After** + +| Node | CPU Before | CPU After | Change | Memory Before | Memory After | Change | +|------|------------|-----------|--------|---------------|--------------|--------| +| worker-01 | 85% | 68% | -17% ✓ | 72% | 59% | -13% ✓ | +| worker-02 | 78% | 58% | -20% ✓ | 65% | 52% | -13% ✓ | +| worker-03 | 42% | 62% | +20% | 48% | 61% | +13% | +| worker-04 | 38% | 55% | +17% | 51% | 63% | +12% | + +**Improvements Achieved:** +- ✓ **CPU load balanced**: All nodes within 10% variance (was 22.1%) +- ✓ **Memory balanced**: All nodes within 8% variance (was 11.8%) +- ✓ **No node exceeding 70% utilization** (was 85% max) +- ✓ **Cluster capacity headroom**: 41% average (was 28%) +- ✓ **Total execution time**: 2 minutes 36 seconds + +**Next Steps:** +- Monitor cluster for 24-48 hours to ensure sustained improvement +- Consider removing nodeAffinity constraints (if added) for long-term flexibility +- Use `/vm-inventory` to verify all VMs are healthy + +Cluster is now optimally balanced. No further action needed. +``` + +**On Partial Success:** + +Display similar format but include: +- Which migrations succeeded +- Which failed (with error details and troubleshooting) +- Which were not attempted (and why) +- Current cluster state vs target +- Recommendations for completing rebalancing + +--- + +## Advanced Features + +### Intelligent Workload Analysis + +**Categorize VMs by workload type** (see [references/production-considerations.md](./references/production-considerations.md)): + +- **Database** (high dirty page rate) → Schedule during low-activity window, consider cold migration +- **Web servers** (low dirty page rate) → Safe for concurrent live migration +- **Caching** (very high dirty page rate) → Migrate during idle or use cold migration +- **Batch processing** → Migrate during job idle periods + +**Use workload characteristics** to optimize migration scheduling. + +### Network Bandwidth Awareness + +**Monitor network saturation:** + +**MCP Tool**: `nodes_stats_summary` (from openshift-virtualization) + +**Parameters**: +```json +{ + "name": "" +} +``` + +Review `.network.interfaces[].rxBytes` and `.network.interfaces[].txBytes`. + +**If saturation detected** (>80% utilization): +- Reduce concurrent migrations +- Set bandwidth limits per migration +- Suggest dedicated migration network + +**Reference**: [references/live-migration-best-practices.md#dedicated-migration-network](./references/live-migration-best-practices.md#dedicated-migration-network-production-best-practice) + +### Multi-Constraint Optimization + +**Consider additional constraints:** +- **Anti-affinity rules**: Don't co-locate VMs with same label +- **Topology spread**: Distribute VMs across zones/racks +- **Resource quotas**: Respect namespace limits +- **Custom scheduling**: Honor existing nodeSelector/tolerations + +**If conflicts detected**, explain to user and suggest resolution. + +--- + +## Human-in-the-Loop Requirements + +**CRITICAL: This mode requires user approval at multiple points.** + +### 1. Goal Clarification (if ambiguous) +- Present optimization options +- Wait for user to select goal +- Do NOT proceed with assumptions + +### 2. Plan Approval (MANDATORY) +- Display complete rebalance plan +- Show expected impact and improvements +- Offer modification options +- **REQUIRE explicit approval** ("approve", "yes", "execute") +- **NEVER execute without approval** + +### 3. Plan Modification (if requested) +- Allow user to skip migrations +- Allow changing target nodes +- Allow reordering +- Recalculate expected improvements +- Present updated plan for approval + +### 4. Error Handling (on failure) +- Stop execution immediately +- Report failure details +- Ask user how to proceed +- **Do NOT continue without user decision** + +### 5. Alternative Proposals (if AI detects better approach) +- Present alternative with rationale +- Show comparison vs original plan +- **User chooses** - execute user's preference + +**Rationale**: User maintains control over cluster changes while benefiting from AI's analytical capabilities. + +--- + +## Safety Considerations + +**Automatic mode involves batch operations with higher complexity:** + +**Risks:** +- ⚠️ Multiple concurrent migrations can saturate network +- ⚠️ Batch failures can compound +- ⚠️ Wrong optimization goal can worsen cluster state +- ⚠️ User must understand multi-VM impact + +**Mitigations:** +- ✓ Require explicit plan approval before execution +- ✓ Allow plan modification before execution +- ✓ Stop on first failure (no blind continuation) +- ✓ Respect cluster concurrency limits +- ✓ Validate all VMs before starting any migration +- ✓ Detailed progress reporting with errors +- ✓ Clear troubleshooting guidance + +**Best Practices:** +- Start with dev/test clusters to learn behavior +- Use during maintenance windows for production +- Monitor network bandwidth during migrations +- Have rollback plan for critical workloads +- Consider snapshots before risky migrations (use `/vm-snapshot-create`) + +**Reference**: [references/anti-patterns.md](./references/anti-patterns.md) + +--- + +## Example Usage + +### Example 1: Multi-Objective Optimization + +``` +User: "Rebalance VMs to optimize both CPU and memory" + +Agent: [Loads REBALANCE_AUTOMATIC.md] + [Gathers cluster state: VMs, nodes, metrics] + [Determines goal: Multi-objective (CPU + memory)] + [Analyzes and generates plan] + +## Automatic Rebalancing Plan + +**Optimization Goal:** Balance CPU and memory load across cluster + +[Displays plan with 5 migrations] +[Shows current state, proposed migrations, expected improvements] + +**How would you like to proceed?** +Options: approve | modify | explain <#> | cancel + +User: "approve" + +Agent: [Validates all VMs] + [Executes migrations with progress reporting] + +## ✓ Automatic Rebalancing Complete + +All 5 migrations successful. +- CPU variance: 22.1% → 4.2% (81% improvement) +- Memory variance: 11.8% → 7.3% (38% improvement) + +Cluster is now balanced. +``` + +### Example 2: With User Modification + +``` +User: "Balance cluster CPU load" + +Agent: [Generates plan with 4 migrations] + +## Automatic Rebalancing Plan + +[Shows plan including cold migration of app-vm-05] + +User: "modify - skip 4" + +Agent: "Migration #4 (app-vm-05) removed from plan." + +[Shows updated plan with 3 migrations] +[Recalculates expected improvements] + +**Ready to execute?** (approve / modify more / cancel) + +User: "approve" + +Agent: [Executes 3 migrations] + +## ✓ Rebalancing Complete + +3/3 migrations successful. +CPU variance: 22.1% → 6.8% (69% improvement) +``` + +### Example 3: AI Proposes Alternative + +``` +User: "Drain worker-02 for maintenance" + +Agent: [Generates plan to migrate all 5 VMs from worker-02] + +💡 **Alternative Approach Detected** + +I notice 2 VMs on worker-02 use RWO storage (cold migration required). +Current plan has ~80s total downtime (2 VMs × 40s each). + +**Alternative:** +Migrate VMs sequentially instead of concurrently to reduce risk: +- Same total time +- Lower network impact +- Easier to abort if issues + +Would you like to: accept alternative | keep original | explain more + +User: "accept alternative" + +Agent: [Updates plan to sequential execution] + [Presents updated plan for approval] +``` + +--- + +**Sources:** +- [Live Migration - KubeVirt User Guide](https://kubevirt.io/user-guide/compute/live_migration/) +- [Node Assignment - KubeVirt User Guide](https://kubevirt.io/user-guide/compute/node_assignment/) +- [Kubernetes Descheduler](https://github.com/kubernetes-sigs/descheduler) +- [Best Practices for Virtual Machine Deployments](https://learn.microsoft.com/en-us/azure/openshift/best-practices-openshift-virtualization) + +**Last Updated**: 2026-02-24 +**Status**: Production Ready diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/REBALANCE_MANUAL.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/REBALANCE_MANUAL.md new file mode 100644 index 00000000..f1e9d2c6 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/REBALANCE_MANUAL.md @@ -0,0 +1,848 @@ +# Manual Rebalancing Strategy + +**Status**: ✅ FULLY IMPLEMENTED + +**Purpose**: Execute VM migrations where the user specifies which VM(s) to move and the target node(s). + +## When to Use Manual Mode + +Use this mode when the user specifies: +- Exact VM name(s) to migrate +- Target node(s) for placement +- Migration type (live or cold) + +**User Request Patterns:** +- "Move VM database-01 to worker-03" +- "Live migrate web-server from worker-01 to worker-05" +- "Cold migrate app-vm to worker-02" +- "Migrate VMs web-01, web-02, web-03 to worker-05" + +## Manual Rebalancing Workflow + +### Step 1: Gather Migration Parameters + +**Collect the following information from the user:** + +**Required Parameters:** +1. **VM Name(s)** - Name of virtual machine(s) to migrate + - Example: "database-01" or ["web-01", "web-02", "web-03"] + +2. **Source Namespace** - Namespace where VM(s) exist + - Example: "production", "vms", "dev-environment" + +3. **Target Node** (optional for live migration, required for cold migration) + - Example: "worker-03", "worker-node-1.example.com" + - For live migration: Can be omitted (scheduler chooses) + - For cold migration: Required (set via nodeAffinity) + +4. **Migration Type** - Choose migration strategy + - `live` - Zero downtime, VM stays running (requires RWX storage) + - `cold` - Brief downtime, stop → move → start (always works) + +**If user doesn't specify migration type, determine automatically:** +- Check VM's PVC access mode using `resources_get` +- If PVC has ReadWriteMany (RWX) → Suggest live migration +- If PVC has ReadWriteOnce (RWO) → Require cold migration +- If uncertain → Ask user to choose + +### Step 2: Validate Migration Feasibility + +**CRITICAL**: Before proceeding with migration execution, perform the **Common Validation Logic** defined in SKILL.md. + +**Reference**: See [SKILL.md - Common Validation Logic](./SKILL.md#common-validation-logic) for complete validation steps. + +**The common validation performs these checks** (defined in SKILL.md): +1. **Validation 1**: Verify VM Exists +2. **Validation 2**: Check Current VM Location +3. **Validation 3**: Validate Storage Compatibility (RWX vs RWO) +4. **Validation 4**: Verify Target Node Exists + +**Only proceed to Step 3 after all validations pass.** + +### Step 3: Present Rebalance Plan for Confirmation + +**After validation, present the rebalance plan to the user:** + +**For Live Migration:** + +```markdown +## VM Rebalance Plan + +**Please review and confirm the migration:** + +| Parameter | Value | Details | +|-----------|-------|---------| +| VM Name | `` | [from user input] | +| Namespace | `` | [from user input] | +| Current Node | `` | [detected from VMI status] | +| Target Node | `` | [from user input or "Scheduler decides"] | +| Migration Type | `live` | Zero downtime, VM stays running | +| Storage Type | `RWX (ReadWriteMany)` | Live migration supported | +| Current Status | `Running` | [from VMI phase] | + +**Migration Strategy: Live Migration** + +**What will happen:** +1. Create VirtualMachineInstanceMigration resource +2. KubeVirt will: + - Create new virt-launcher pod on target node + - Transfer VM memory and state (live) + - Switch network traffic to new pod + - Terminate old virt-launcher pod +3. VM continues running throughout (brief network pause <1s) + +**Impact:** +- ✓ Zero downtime (VM stays running) +- ✓ Applications remain accessible +- ⚠️ Brief network pause during cutover (<1 second) +- ⚠️ Requires network bandwidth for memory transfer +- ⚠️ Migration duration depends on VM memory size + +**Estimated Duration:** ~30-60 seconds for typical VM (depends on memory size) + +**Reference**: See [references/live-migration-best-practices.md](./references/live-migration-best-practices.md) for configuration parameters, storage requirements, and network optimization + +**Rollback:** If migration fails, VM continues running on current node + +Confirm this migration or tell me what to change: +- yes - Proceed with migration +- no - Cancel migration +- modify - Change parameters +``` + +**For Cold Migration:** + +```markdown +## VM Rebalance Plan + +| Parameter | Value | Details | +|-----------|-------|---------| +| VM Name | `` | [from user input] | +| Namespace | `` | [from user input] | +| Current Node | `` | [detected] | +| Target Node | `` | [from user input] | +| Migration Type | `cold` | VM will stop and restart | +| Storage Type | `RWO (ReadWriteOnce)` | Live migration not supported | +| Current Status | `Running` | Will be stopped then restarted | + +**Migration Strategy: Cold Migration** + +**What will happen:** +1. Update VM spec with **required nodeAffinity** (forces placement on target node) +2. Stop VM using vm_lifecycle +3. VM pod terminates on current node +4. Start VM using vm_lifecycle +5. Kubernetes scheduler places VM on target node (due to nodeAffinity) +6. **After migration:** You'll choose whether to remove the affinity for long-term flexibility + +**Impact:** +- ⚠️ **Downtime**: ~30-60 seconds while VM restarts +- ⚠️ Applications will be interrupted +- ⚠️ Network connections will be dropped +- ✓ Works with any storage type (RWO or RWX) +- ✓ Less resource-intensive than live migration +- ℹ️ **Long-term**: After migration, you can remove node affinity to allow flexible rescheduling if target node fails + +**Estimated Downtime:** ~30-60 seconds + +**Rollback:** Can start VM again if migration fails + +**Note on Node Affinity:** +We use nodeAffinity (not nodeSelector) for better long-term resilience. After migration, you'll have the option to remove it, allowing the VM to reschedule to other nodes if fails in the future. + +**Reference**: See [references/production-considerations.md](./references/production-considerations.md) for workload categorization and downtime planning strategies + +Confirm this migration or tell me what to change: +- yes - Proceed with cold migration +- no - Cancel +- modify - Change parameters +``` + +**WAIT for user confirmation before proceeding to Step 4.** Do NOT continue until user explicitly confirms with "yes". + +### Step 4a: Execute Live Migration (if migration type = live) + +**ONLY PROCEED AFTER USER CONFIRMATION IN STEP 3.** + +**Create a VirtualMachineInstanceMigration resource to trigger live migration:** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +- `resource`: (JSON object as string) - REQUIRED + - Complete VirtualMachineInstanceMigration resource definition as a JSON-formatted string + - Must follow KubeVirt API specification + +**Resource Structure** (for live migration): + +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": "migration--", + "namespace": "" + }, + "spec": { + "vmiName": "" + } +} +``` + +**Example tool invocation:** +```json +{ + "resource": "{\"apiVersion\":\"kubevirt.io/v1\",\"kind\":\"VirtualMachineInstanceMigration\",\"metadata\":{\"name\":\"migration-database-01-20260223\",\"namespace\":\"production\"},\"spec\":{\"vmiName\":\"database-01\"}}" +} +``` + +**Note**: The `resource` parameter accepts the resource definition as a JSON-formatted string. The MCP tool will parse and apply this resource to the cluster. + +**Note on Target Node Selection:** +- KubeVirt scheduler automatically selects target node +- To influence target node, update VM's nodeAffinity BEFORE creating migration +- For manual target node selection, combine with nodeAffinity update first + +**Expected Output**: VirtualMachineInstanceMigration resource created successfully + +**Error Handling:** +- If creation fails → Check RBAC permissions, report error to user +- If VMI not found → Verify VM is running, report error +- If VMI not migratable → Check storage access mode, suggest cold migration + +#### Monitor Migration Progress + +**After creating migration, monitor progress:** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters** (to monitor migration status): +- `apiVersion`: "kubevirt.io/v1" - REQUIRED +- `kind`: "VirtualMachineInstanceMigration" - REQUIRED +- `name`: "migration--" - REQUIRED +- `namespace`: "" - REQUIRED + +**Poll every 5-10 seconds until migration completes or fails.** + +**Timeout**: Stop polling after 10 minutes (600 seconds) and report timeout error to user. Most migrations complete within 1-5 minutes depending on VM memory size. + +**Reference**: See [references/performance-tuning.md](./references/performance-tuning.md) for timeout configuration and migration optimization strategies + +**Expected Output**: VMIM resource with status field + +**Extract Information:** +- `status.phase` - Migration phase: + - `Scheduling` - Finding target node + - `PreparingTarget` - Setting up target pod + - `TargetReady` - Target pod ready + - `Running` - Transferring memory + - `Succeeded` - Migration completed + - `Failed` - Migration failed +- `status.migrationState.completed` - Boolean, true when done +- `status.migrationState.targetNode` - Destination node +- `status.migrationState.sourceNode` - Origin node +- `status.migrationState.startTimestamp` - When migration began +- `status.migrationState.endTimestamp` - When migration completed + +**When status.phase = "Succeeded":** +- Migration completed successfully +- Proceed to Step 5 (Report Results) + +**When status.phase = "Failed":** +- Extract failure reason from status +- Consult troubleshooting documentation (see Step 5 failure handling) +- Report detailed error to user + +### Step 4b: Execute Cold Migration (if migration type = cold) + +**ONLY PROCEED AFTER USER CONFIRMATION IN STEP 3.** + +**Cold migration workflow: Stop VM → Re-read VM → Update node placement → Start VM** + +#### Sub-step 4b.1: Stop the VM + +**MCP Tool**: `vm_lifecycle` (from openshift-virtualization) + +**Parameters**: +- `namespace`: "" - REQUIRED +- `name`: "" - REQUIRED +- `action`: "stop" - REQUIRED + +**Expected Output**: VM stopped successfully, VMI terminates + +**Wait for VM to fully stop:** +1. Wait 10 seconds +2. Check VM status using `resources_get` (VirtualMachine) +3. If `status.printableStatus` is not "Stopped", wait another 10 seconds and check again +4. Repeat until VM is fully stopped + +**Error Handling:** +- If stop fails → Report error, check if VM is already stopped +- If VM stuck in Terminating after 60 seconds → Report to user + +#### Sub-step 4b.2: Re-read VM for Fresh ResourceVersion + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +- `apiVersion`: "kubevirt.io/v1" - REQUIRED +- `kind`: "VirtualMachine" - REQUIRED +- `name`: "" - REQUIRED +- `namespace`: "" - REQUIRED + +**Why**: `vm_lifecycle` modified the VM. Re-reading gets fresh resourceVersion to prevent conflicts when updating nodeAffinity. + +**Use this fresh VM spec for nodeAffinity update in next step.** + +#### Sub-step 4b.3: Update VM nodeAffinity + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +- `resource`: (JSON object as string) - REQUIRED + - Complete VirtualMachine resource from Sub-step 4b.2 with updated nodeAffinity + +**Resource Modification** (add required nodeAffinity to VM spec): + +Preserve all existing VM fields and only modify the affinity section. + +**Critical Affinity Structure:** +```json +{ + "spec": { + "template": { + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "kubernetes.io/hostname", + "operator": "In", + "values": [""] + } + ] + } + ] + } + } + } + } + } + } +} +``` + +**Why nodeAffinity instead of nodeSelector?** +- Provides flexibility for long-term resilience +- If target node fails later, user can remove affinity to allow rescheduling +- More powerful than nodeSelector (supports multiple nodes, preferences) + +**Expected Output**: VirtualMachine resource updated successfully + +**Error Handling:** +- If update fails → Check RBAC permissions, explain to user with exact error +- If VM not found → Report error with VM name and namespace +- If conflicts with existing affinity → Ask user: "VM already has node affinity rules. Should I replace them with new affinity to ?" + +#### Sub-step 4b.4: Start the VM + +**MCP Tool**: `vm_lifecycle` (from openshift-virtualization) + +**Parameters**: +- `namespace`: "" - REQUIRED +- `name`: "" - REQUIRED +- `action`: "start" - REQUIRED + - Example: `"start"` (power on the VM) + +**Expected Output**: VM starts successfully + +**After starting, verify VM scheduled on target node:** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters** (get VMI to verify node placement): +- `apiVersion`: "kubevirt.io/v1" - REQUIRED +- `kind`: "VirtualMachineInstance" - REQUIRED +- `name`: "" - REQUIRED +- `namespace`: "" - REQUIRED + +**Expected Output**: VMI resource with node placement + +**Extract Information:** +- `status.nodeName` - Should match target node +- `status.phase` - Should be "Running" + +**Verification:** +- If `status.nodeName` matches target node → Success, proceed to Step 4b.5 +- If `status.nodeName` does NOT match target node → Migration failed, report error with explanation + +**Error Handling with Clear Explanations:** +- If VM fails to start → Explain: "VM failed to start on . This usually means the node doesn't have enough resources (CPU/memory) or has scheduling constraints preventing this VM." + - Consult [scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) + - Suggest: Check node capacity, verify node is Ready, check for taints +- If VM scheduled on wrong node → Explain: "VM started but Kubernetes placed it on instead of . This means couldn't accommodate the VM due to resource constraints, taints, or other scheduling rules." + - Report actual vs expected node + - Ask user if they want to retry with different target node +- If VM stuck in Scheduling phase → Explain: "VM cannot be scheduled. The target node likely lacks sufficient CPU/memory resources, or has taints that prevent scheduling." + - Show node constraints + - Suggest checking node details using `resources_get` with kind="Node" + +#### Sub-step 4b.5: Post-Migration Cleanup (Optional - Long-term Resilience) + +**After successful cold migration, ask user about removing node affinity for long-term flexibility.** + +**Present cleanup option to user:** + +```markdown +## ✓ Cold Migration Successful - VM Running on Target Node + +**VM Details:** +- **Name**: `` +- **Namespace**: `` +- **Current Node**: `` ✓ + +**Important: Long-term Resilience** + +The VM now has a **required node affinity** to ``. This means: +- ✓ VM will stay on `` (as you requested) +- ⚠️ If `` fails or needs maintenance later, the VM **cannot reschedule** to other nodes + +**Recommendation:** Remove the node affinity to allow flexible rescheduling in the future. + +Would you like to remove the node affinity now? +- **yes** - Remove affinity (VM can reschedule to any healthy node if fails) +- **no** - Keep affinity (VM stays pinned to permanently) +- **later** - Keep for now, I'll remove it manually when needed +``` + +**WAIT for user decision.** + +**If user says "yes" (remove affinity):** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters** (get current VM spec): +- `apiVersion`: "kubevirt.io/v1" - REQUIRED +- `kind`: "VirtualMachine" - REQUIRED +- `name`: "" - REQUIRED +- `namespace`: "" - REQUIRED + +**Expected Output**: Full VirtualMachine resource with current affinity + +**Modify the VM spec to remove nodeAffinity:** + +Extract the full VM resource, then remove `spec.template.spec.affinity.nodeAffinity` (or set to null/empty). + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +- `resource`: (JSON object as string) - REQUIRED + - Complete VirtualMachine resource with affinity removed as a JSON-formatted string + +**Important**: +- Preserve all other VM spec fields +- Only remove the affinity section +- VM continues running on current node (doesn't trigger immediate reschedule) +- Future restarts will allow flexible scheduling + +**Expected Output**: VirtualMachine resource updated successfully + +**Error Handling:** +- If update fails → Explain: "Failed to remove node affinity. RBAC permissions issue or API error. VM will remain pinned to . You can remove manually using `resources_create_or_update` with the VM spec after removing the affinity field." + +**If user says "no" or "later" (keep affinity):** +- Report: "Node affinity kept. VM will remain on . To remove manually later, use `resources_create_or_update` with the VM spec after removing the affinity field." +- Proceed to Step 5 + +### Step 5: Report Migration Results + +**On Success (Live Migration):** + +```markdown +## ✓ Live Migration Successful + +**VM Details:** +- **Name**: `` +- **Namespace**: `` +- **Migration Type**: Live (zero downtime) + +**Migration Summary:** +- **Source Node**: `` +- **Target Node**: `` +- **Duration**: seconds +- **Status**: Succeeded + +**Impact:** +- ✓ VM remained running throughout migration +- ✓ Applications stayed accessible +- ✓ Brief network pause during cutover (<1 second) + +**Current VM Status:** +- **Running on**: `` +- **Phase**: Running +- **Ready**: True + +### Next Steps + +**Verify application health:** +Use `/vm-inventory` to check VM status + +**View migration details:** +Use `resources_get` with kind="VirtualMachineInstanceMigration" and name="migration--" + +**Cleanup:** +The VirtualMachineInstanceMigration resource can be deleted if no longer needed. +``` + +**On Success (Cold Migration) - If affinity was REMOVED:** + +```markdown +## ✓ Cold Migration Successful + +**VM Details:** +- **Name**: `` +- **Namespace**: `` +- **Migration Type**: Cold (with downtime) + +**Migration Summary:** +- **Source Node**: `` +- **Target Node**: `` +- **Downtime**: ~ seconds +- **Status**: Succeeded + +**Steps Completed:** +1. ✓ Updated VM with required nodeAffinity to `` +2. ✓ Stopped VM on `` +3. ✓ Started VM on `` +4. ✓ Verified VM running on target node +5. ✓ Removed node affinity for flexible rescheduling + +**Current VM Status:** +- **Running on**: `` +- **Phase**: Running +- **Ready**: True +- **Node Affinity**: None (can reschedule to any node if needed) + +**Long-term Resilience:** +✓ VM can now reschedule to other nodes if `` fails or needs maintenance. + +### Next Steps + +**Verify application health:** +Use `/vm-inventory` to check VM status + +**Test application connectivity:** +VM has restarted, verify services are healthy. +``` + +**On Success (Cold Migration) - If affinity was KEPT:** + +```markdown +## ✓ Cold Migration Successful + +**VM Details:** +- **Name**: `` +- **Namespace**: `` +- **Migration Type**: Cold (with downtime) + +**Migration Summary:** +- **Source Node**: `` +- **Target Node**: `` +- **Downtime**: ~ seconds +- **Status**: Succeeded + +**Steps Completed:** +1. ✓ Updated VM with required nodeAffinity to `` +2. ✓ Stopped VM on `` +3. ✓ Started VM on `` +4. ✓ Verified VM running on target node +5. ℹ️ Kept node affinity (as requested) + +**Current VM Status:** +- **Running on**: `` +- **Phase**: Running +- **Ready**: True +- **Node Affinity**: Required on `` (VM will stay on this node) + +**Important:** +⚠️ VM is pinned to ``. If this node fails, the VM cannot reschedule to other nodes. + +**To remove affinity later:** +Use `resources_create_or_update` with the VM spec after removing the `spec.template.spec.affinity` field + +### Next Steps + +**Verify application health:** +Use `/vm-inventory` to check VM status + +**Test application connectivity:** +VM has restarted, verify services are healthy. +``` + +**On Failure (with Troubleshooting):** + +**OPTIONAL**: If migration fails, consult documentation for common failure scenarios. + +**Document Consultation** (OPTIONAL - when migration fails): +1. **Action**: Read relevant troubleshooting guides to understand VM migration failure scenarios: + - [scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) - For ErrorUnschedulable, node taints, resource constraints + - [lifecycle-errors.md](../../docs/troubleshooting/lifecycle-errors.md) - For VM start/stop failures during cold migration + - [storage-errors.md](../../docs/troubleshooting/storage-errors.md) - For PVC access mode issues affecting live migration +2. **Output to user**: "I consulted [scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) to understand potential causes for the migration failure." + +**When to consult**: +- Live migration fails (check storage, network, resource constraints) +- Cold migration fails during start (likely scheduling or resource issues) +- VM stuck in ErrorUnschedulable state +- Unexpected error messages from migration tools + +**When NOT to consult**: +- RBAC permission errors (clear cause) +- VM not found errors (clear cause) +- PVC access mode incompatibility (already validated in Step 2) + +```markdown +## ❌ Migration Failed + +**Error**: + +**VM Details:** +- **Name**: `` +- **Namespace**: `` +- **Migration Type**: +- **Source Node**: `` +- **Target Node**: `` + +**Common Causes:** + +**For Live Migration Failures:** +- **Insufficient resources** - Target node lacks CPU/memory for VM +- **Network bandwidth** - Slow network prevents migration convergence +- **PVC access mode** - Storage is not ReadWriteMany (RWX) +- **Timeout exceeded** - Migration took longer than allowed (default 150s/GiB) +- **High memory write rate** - VM writes to memory faster than transfer rate + +**For Cold Migration Failures:** +- **Node not schedulable** - Target node is cordoned, has taints, or lacks resources +- **NodeSelector conflict** - VM has other scheduling constraints preventing placement +- **VM failed to start** - Check scheduling errors on target node +- **Storage issues** - PVC not accessible from target node + +**Troubleshooting Steps:** + +1. **Check node availability:** + Use `resources_list` with kind="Node" to verify target node is Ready and schedulable + +2. **Check VM events:** + Use `resources_get` with kind="VirtualMachine" to see VM status and conditions + +3. **Check migration status** (for live migration): + Use `resources_list` with kind="VirtualMachineInstanceMigration" to see migration resource status + +4. **Verify storage access:** + Use `resources_get` with kind="PersistentVolumeClaim" to check PVC access modes + +5. **Check resource capacity:** + Use `nodes_top` or `resources_get` with kind="Node" to verify available CPU/memory + +**Recommended Actions:** + +- **If timeout**: Retry with slower workload or enable auto-converge +- **If resources**: Choose different target node with more capacity +- **If storage**: Use cold migration instead (works with RWO) +- **If node unavailable**: Verify node is Ready and untainted + +**Rollback (if migration succeeded but VM has issues on target node):** + +To move VM back to original node, perform reverse migration: +- **For live migration**: Create new migration from current node → original node +- **For cold migration**: Update nodeAffinity to original node, restart VM +- Use same migration type as original migration + +Example: "Migrate from back to " + +Would you like help troubleshooting this error? +``` + +## Batch VM Migration + +**For migrating multiple VMs in a single operation:** + +**User Request:** "Migrate VMs web-01, web-02, web-03 to worker-05" + +**Safety Limits:** +- ⚠️ **Batches >5 VMs**: Warn user about potential network saturation and suggest staged execution +- ⚠️ **Batches >10 VMs**: Strongly recommend splitting into multiple batches (e.g., 5 VMs at a time) +- ⚠️ **Multiple cold migrations**: Warn about cumulative downtime impact + +**Reference**: See [references/anti-patterns.md](./references/anti-patterns.md) for common mistakes when performing batch migrations and network saturation issues + +**Workflow:** +1. Validate all VMs exist and are migratable +2. Determine migration type for each (based on storage) +3. **Check batch size** - If >5 VMs, warn user and offer to stage migrations +4. Present batch rebalance plan: + ```markdown + ## Batch Rebalance Plan + + Migrating 3 VMs to worker-05: + + | VM Name | Current Node | Migration Type | Impact | + |---------|--------------|----------------|--------| + | web-01 | worker-01 | live | <1s pause | + | web-02 | worker-02 | live | <1s pause | + | web-03 | worker-03 | cold | ~40s downtime | + + Total: 2 live migrations + 1 cold migration + + Proceed with batch migration? + ``` +4. Execute migrations sequentially (avoid saturating network) +5. Report results for each VM + +## Example Usage + +### Example 1: Successful Live Migration + +``` +User: "Live migrate VM database-01 from production namespace to worker-03" + +Agent: [Validates VM exists, checks storage is RWX, verifies target node] + +## VM Rebalance Plan + +Please review and confirm the migration: + +| Parameter | Value | Details | +|-----------|-------|---------| +| VM Name | `database-01` | from user input | +| Namespace | `production` | from user input | +| Current Node | `worker-01` | detected from VMI status | +| Target Node | `worker-03` | from user input | +| Migration Type | `live` | Zero downtime, VM stays running | +| Storage Type | `RWX (ReadWriteMany)` | Live migration supported | + +**What will happen:** +1. Create VirtualMachineInstanceMigration resource +2. VM memory and state transferred live to worker-03 +3. Brief network pause during cutover (<1 second) + +**Impact:** Zero downtime, brief network pause + +Confirm this migration? + +User: "yes" + +Agent: [Creates VirtualMachineInstanceMigration resource] + [Monitors migration status] + +## ✓ Live Migration Successful + +**VM Details:** +- Name: `database-01` +- Namespace: `production` +- Migration Type: Live (zero downtime) + +**Migration Summary:** +- Source Node: `worker-01` +- Target Node: `worker-03` +- Duration: 45 seconds +- Status: Succeeded + +VM is now running on `worker-03`. +``` + +### Example 2: Successful Cold Migration + +``` +User: "Move VM web-server to worker-05 using cold migration" + +Agent: [Validates VM exists, checks current node, verifies target node] + +## VM Rebalance Plan + +| Parameter | Value | Details | +|-----------|-------|---------| +| VM Name | `web-server` | from user input | +| Namespace | `vms` | detected/default | +| Current Node | `worker-02` | detected | +| Target Node | `worker-05` | from user input | +| Migration Type | `cold` | VM will stop and restart | +| Storage Type | `RWO (ReadWriteOnce)` | Live migration not supported | + +**What will happen:** +1. Update VM nodeAffinity to target worker-05 +2. Stop VM (applications will be interrupted) +3. Start VM (Kubernetes places on worker-05) + +**Impact:** Downtime ~30-60 seconds + +Confirm this migration? + +User: "yes" + +Agent: [Updates VM nodeAffinity] + [Calls vm_lifecycle stop] + [Calls vm_lifecycle start] + [Verifies VM on target node] + +## ✓ Cold Migration Successful + +**VM Details:** +- Name: `web-server` +- Namespace: `vms` +- Migration Type: Cold (with downtime) + +**Migration Summary:** +- Source Node: `worker-02` +- Target Node: `worker-05` +- Downtime: ~40 seconds +- Status: Succeeded + +VM is now running on `worker-05`. Verify application health. +``` + +### Example 3: Migration Failure - Storage Incompatible + +``` +User: "Live migrate app-vm to worker-03" + +Agent: [Validates VM] + [Checks PVC access mode - discovers RWO] + +## ❌ Live Migration Not Supported + +**VM Details:** +- Name: `app-vm` +- Namespace: `default` +- Storage Type: `RWO (ReadWriteOnce)` + +**Issue:** Live migration requires ReadWriteMany (RWX) storage. This VM uses ReadWriteOnce (RWO), which only allows attachment to one node at a time. + +**Recommended Solution:** Use cold migration instead. + +Would you like to proceed with cold migration? (Brief downtime ~30-60 seconds) + +User: "yes, do cold migration" + +Agent: [Switches to cold migration workflow] + [Continues with cold migration process...] +``` + +--- + +**Reference Documentation:** + +**Internal Skill Documentation**: +- [references/live-migration-best-practices.md](./references/live-migration-best-practices.md) - Configuration, requirements, dedicated networks +- [references/performance-tuning.md](./references/performance-tuning.md) - Right-sizing, overcommit, bandwidth tuning +- [references/anti-patterns.md](./references/anti-patterns.md) - Common mistakes to avoid +- [references/production-considerations.md](./references/production-considerations.md) - HA strategies, capacity planning + +**Official KubeVirt Documentation**: +- [Live Migration - KubeVirt User Guide](https://kubevirt.io/user-guide/compute/live_migration/) +- [Node Assignment - KubeVirt User Guide](https://kubevirt.io/user-guide/compute/node_assignment/) +- [VirtualMachineInstanceMigration API](https://kubevirt.io/api-reference/main/definitions.html#_v1_virtualmachineinstancemigration) diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/SKILL.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/SKILL.md new file mode 100644 index 00000000..4ee0bb9c --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/SKILL.md @@ -0,0 +1,391 @@ +--- +name: vm-rebalance +description: | + Orchestrate VM migrations across cluster nodes for load balancing, maintenance, and resource optimization. + + Use when: + - "Move VM database-01 to worker-03" + - "Rebalance VMs to optimize CPU load" + - "Drain worker-02 for maintenance" + - "Automatically rebalance the cluster" + + Supports Manual (user-driven) and Automatic (AI-driven) modes. + + NOT for creating VMs (use vm-create) or lifecycle only (use vm-lifecycle-manager). + +model: inherit +color: yellow +--- + +# /vm-rebalance Skill + +Orchestrate VM migrations across OpenShift cluster nodes for load balancing, maintenance, and resource optimization. Supports manual and automatic rebalancing with live migration (zero downtime) and cold migration (brief downtime) strategies. + +**Implementation**: Uses KubeVirt's VirtualMachineInstanceMigration API for live migrations and node affinity for cold migrations. + +## Prerequisites + +**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools**: +- `resources_list` - List VMs and nodes +- `resources_get` - Get VM and node details +- `resources_create_or_update` - Create migrations and update VM specs +- `vm_lifecycle` - Start/stop VMs for cold migration +- `nodes_top` - Monitor node resource usage +- `pods_top` - Monitor VM resource consumption + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file + +**Required Cluster Setup**: +- OpenShift cluster (>= 4.17) +- OpenShift Virtualization operator installed +- ServiceAccount with permissions: get/list/update for VMs, create for VirtualMachineInstanceMigration +- For live migration: RWX storage and sufficient network bandwidth + +### Prerequisite Verification + +**Before executing:** + +1. Check `openshift-virtualization` exists in `.mcp.json` → If missing, report setup +2. Verify `KUBECONFIG` is set (presence only, never expose value) → If missing, report +3. For live migration: Check PVC access mode is ReadWriteMany (RWX) via `resources_get` + +**Human Notification Protocol:** `❌ Cannot execute vm-rebalance: MCP server not available. Setup: Add to .mcp.json, set KUBECONFIG, restart Claude Code. Docs: https://github.com/openshift/openshift-mcp-server` + +⚠️ **SECURITY**: Never display KUBECONFIG path or credential values. + +## When to Use This Skill + +**Trigger when:** +- User explicitly invokes `/vm-rebalance` +- User requests moving VM(s) to specific node(s) +- User wants to drain node for maintenance +- User requests load balancing or resource optimization + +**User phrases:** +- "Move VM database-01 to worker-03" +- "Live migrate web-server to worker-05" +- "Drain worker-02 for maintenance" +- "Balance CPU load across nodes" +- "Automatically rebalance the cluster" + +**Do NOT use when:** +- Creating VMs → `/vm-create` +- Start/stop only → `/vm-lifecycle-manager` +- Cloning VMs → `/vm-clone` +- Deleting VMs → `/vm-delete` + +## Workflow + +### Step 1: Determine Rebalancing Mode + +**Manual Mode**: User specifies VM name(s) and target node(s). Example: "Move VM database-01 to worker-03" + +**Automatic Mode**: User requests AI-driven rebalancing. Example: "Rebalance VMs based on CPU" + +### Step 2: Load Strategy File and Execute + +**For Manual Mode:** + +**Document Consultation** (REQUIRED - Execute FIRST): +1. Read [REBALANCE_MANUAL.md](./REBALANCE_MANUAL.md) using Read tool +2. Output: "I consulted [REBALANCE_MANUAL.md](./REBALANCE_MANUAL.md) to understand the manual migration workflow." +3. **Then execute**: Follow workflow in REBALANCE_MANUAL.md + +--- + +**For Automatic Mode:** + +**Document Consultation** (REQUIRED - Execute FIRST): +1. Read [REBALANCE_AUTOMATIC.md](./REBALANCE_AUTOMATIC.md) using Read tool +2. Output: "I consulted [REBALANCE_AUTOMATIC.md](./REBALANCE_AUTOMATIC.md) to understand the automatic rebalancing workflow." +3. **Then execute**: Follow workflow in REBALANCE_AUTOMATIC.md + +## Common Validation Logic + +**Shared by ALL migration strategies. Execute before any VM migration:** + +### Validation 1: Verify VM Exists + +**MCP Tool**: `resources_get` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", name=``, namespace=``) + +**Extract**: `spec.template.spec.volumes[].persistentVolumeClaim.claimName`, `status.ready` + +**Errors**: VM not found → Use vm-inventory | Namespace not found → Verify name | Permission denied → Check RBAC + +### Validation 2: Check Current VM Location + +**MCP Tool** (if VM running): `resources_get` (apiVersion="kubevirt.io/v1", kind="VirtualMachineInstance", name=``, namespace=``) + +**Extract**: `status.nodeName`, `status.phase` + +**Validation**: If already on target → "VM already on target node. No migration needed." + +### Validation 3: Validate Storage Compatibility + +**MCP Tool**: `resources_get` (apiVersion="v1", kind="PersistentVolumeClaim", name=``, namespace=``) + +**Extract**: `spec.accessModes` +- ReadWriteMany (RWX) → Live migration supported +- ReadWriteOnce (RWO) → Live migration NOT supported + +**Error for live migration**: If RWO → "Cannot live migrate. Use cold migration (brief downtime ~30-60s)." + +**Reference**: [references/live-migration-best-practices.md](./references/live-migration-best-practices.md) + +### Validation 4: Verify Target Node Exists + +**MCP Tool**: `resources_list` (apiVersion="v1", kind="Node") + +**Validation**: Verify target exists, `status.conditions[]` shows Ready=True, not cordoned + +**Errors**: Not found → "Node doesn't exist" | Not Ready → "Choose different target" | Cordoned → "Uncordon or choose different target" + +**Reference**: [../../docs/troubleshooting/scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) + +## Node Selection for Automatic Rebalancing + +**Applies to Automatic Mode only.** + +**Use** `resources_list` **(apiVersion="v1", kind="Node")** + +Filter where ALL true: +1. `metadata.labels["kubevirt.io/schedulable"] == "true"` +2. `status.capacity["devices.kubevirt.io/kvm"]` > "0" +3. No `node-role.kubernetes.io/control-plane` or `node-role.kubernetes.io/master` label + +**If no nodes**: "No suitable nodes. Check OpenShift Virtualization operator and hardware virtualization support." + +**Note**: Ignore custom taints. Use official KubeVirt labels. + +## Common Migration Types + +**Live Migration**: Zero downtime, <1s pause during cutover. Requires RWX storage. Memory transferred while VM runs. + +**Cold Migration**: Brief downtime (~30-60s). Works with any storage. Stop VM → Update placement → Start on target. + +**Reference**: [references/live-migration-best-practices.md](./references/live-migration-best-practices.md) + +## Common Plan Visualization + +**ALL strategies MUST use this standardized format for consistency.** + +### Information Relevance Principle + +Show only what matters: +- ✅ Include: Deviations from defaults, user-specified criteria, non-obvious context +- ❌ Exclude: Standard procedures, default settings, info already visible in tables + +### Standard Plan Format + +**Table 1: VM Rebalance Plan** + +```markdown +## 📋 VM Rebalance Plan + +| VM | Instance Type | Current Node | → | New Node | Type | Downtime | Notes | +|----|---------------|--------------|---|----------|------|----------|-------| +| vm-1 | u1.xlarge | worker-01 | → | worker-03 | Live | <1s pause | ContainerDisk | +| vm-2 | u1.2xmedium | worker-01 | → | worker-02 | Cold | ~40s | RWO storage | +| vm-3 | u1.medium | worker-02 | - | *stays* | - | - | Already balanced | +``` + +**Column Definitions**: VM name | Instance type | Current node | Movement indicator | Target node (or *stays*) | Migration type (Live/Cold/-) | Downtime (<1s/~30-60s/-) | Brief explanation + +**Table 2: Node State Before → After** + +```markdown +## 📊 Node State: Before → After + +| Node | VMs Now | CPU Now | Memory Now | → | VMs After | CPU After | Memory After | Change | +|------|---------|---------|------------|---|-----------|-----------|--------------|--------| +| worker-01 | 5 | 85% | 72% | → | 3 | 68% | 59% | ✓ Reduced load | +| worker-02 | 2 | 42% | 48% | → | 3 | 58% | 61% | ← Receiving VMs | +| worker-03 | 3 | 38% | 51% | → | 4 | 55% | 63% | ← Receiving VMs | +``` + +**CRITICAL - Capacity Calculation Method:** + +CPU/Memory percentages MUST be calculated based on **allocated capacity**, not actual runtime usage: + +**CPU Percentage Calculation**: +1. Get node total CPU capacity from `resources_get` Node → `status.capacity.cpu` (e.g., "32" = 32 vCPUs) +2. For each VM on node, get allocated vCPUs from VMI → `spec.domain.cpu.sockets × spec.domain.cpu.cores × spec.domain.cpu.threads` +3. Sum all VM vCPUs on the node +4. Calculate: (Sum of VM vCPUs / Node CPU capacity) × 100 + +**Memory Percentage Calculation**: +1. Get node total memory capacity from `resources_get` Node → `status.capacity.memory` (e.g., "128Gi") +2. For each VM on node, get allocated memory from VMI → `spec.domain.memory.guest` +3. Sum all VM memory allocations on the node (convert to same units) +4. Calculate: (Sum of VM memory / Node memory capacity) × 100 + +**Example**: Node with 32 vCPUs hosting VMs with 2+4+8+4+2 = 20 vCPUs → CPU = 62.5% (20/32), NOT the actual runtime usage which might be 0% if VMs are idle. + +**Rationale**: Shows **capacity planning** (how much is reserved) rather than runtime utilization, which is more useful for rebalancing decisions. + +**Overcommit Detection and Warning**: + +If any node's CPU or Memory percentage **exceeds 100%** after rebalancing: + +```markdown +⚠️ **OVERCOMMIT WARNING** + +**Node(s) will be overcommitted after this rebalance:** +- **worker-02**: CPU 125% (40 vCPUs allocated / 32 vCPUs capacity) - **25% overcommit** +- **worker-03**: Memory 110% (88Gi allocated / 80Gi capacity) - **10% overcommit** + +**Impact:** +- **CPU overcommit**: VMs may experience CPU throttling and reduced performance when all VMs are active simultaneously +- **Memory overcommit**: Risk of VM eviction or OOM (Out of Memory) if total memory demand exceeds node capacity + +**Recommendations:** +- Consider distributing VMs across more nodes to avoid overcommit +- Review VM instance types to ensure they match actual workload requirements +- Monitor node resource usage closely after rebalancing + +**Proceed with overcommit?** (yes/cancel) +``` + +**When NOT to warn**: If percentages ≤ 100%, overcommit is not present. Omit this warning section. + +**After tables, include:** + +**Key Improvement**: `"Distribution from 1 node to 4 nodes hosting VMs"` or `"CPU variance reduced from 22% to 4% (81% improvement)"` + +**Rebalance Summary** (batch operations): +```markdown +- Total VMs: 5 | Live: 4 | Cold: 1 | Staying: 2 +- Total Downtime: ~40s | Duration: 1-2min (parallel) +``` + +**Execution Mode**: `**Parallel** (default) - all VMs rebalance simultaneously` OR `**Sequential** (user requested)` + +**Terminology Standards**: +- ✅ "VM Rebalance Plan", "Rebalancing", "Live/Cold migration", "Current Node/New Node", "VMs Now/VMs After" +- ❌ "VM Migration Plan" (reserved for future migration skill) + +## Common Error Handling + +### Error 1: Live Migration Fails - Storage Not RWX +**Symptom**: "Cannot live migrate: PVC access mode is ReadWriteOnce" +**Solution**: Use cold migration OR convert PVC to RWX +**Reference**: [../../docs/troubleshooting/storage-errors.md](../../docs/troubleshooting/storage-errors.md) + +### Error 2: VM Stuck ErrorUnschedulable After Cold Migration +**Symptom**: "VM cannot be scheduled: ErrorUnschedulable" +**Solution**: Check node capacity (`nodes_top`), verify no blocking taints (`resources_get` Node), add tolerations, choose different target, remove nodeSelector +**Reference**: [../../docs/troubleshooting/scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) + +### Error 3: Live Migration Times Out +**Symptom**: "Migration exceeded timeout: 150s per GiB" +**Solution**: Retry migration, reduce VM workload, use cold migration, increase timeout in HyperConverged CR +**Reference**: [references/performance-tuning.md](./references/performance-tuning.md) + +### Error 4: Migration Rejected - Cluster Limit Reached +**Symptom**: "Migration rejected: cluster limit reached (5 concurrent)" +**Solution**: Wait for migrations to complete (`resources_list` VirtualMachineInstanceMigration), retry, migrate sequentially, increase limit +**Reference**: [references/performance-tuning.md](./references/performance-tuning.md) + +### Error 5: RBAC Permission Denied +**Symptom**: "Forbidden: User cannot create VirtualMachineInstanceMigration" +**Solution**: Verify RBAC permissions (`create` on VirtualMachineInstanceMigration, `update` on VirtualMachine), contact admin + +### Error 6: Network Saturation +**Symptom**: Multiple migrations slow/fail, high network utilization +**Solution**: Reduce concurrent migrations, set bandwidth limit, use dedicated migration network +**Reference**: [references/performance-tuning.md](./references/performance-tuning.md) + +### Error 7: Resource Version Conflict During Cold Migration +**Symptom**: "Apply failed: conflict with 'kubernetes-mcp-server' using .spec.runStrategy" +**Solution**: After `vm_lifecycle` stop, re-read VM using `resources_get` before updating nodeAffinity (gets fresh resourceVersion) +**Workflow**: Stop → Wait → Re-read → Update nodeAffinity → Start +**Reference**: [REBALANCE_MANUAL.md - Sub-step 4b.2.5](./REBALANCE_MANUAL.md) + +## Dependencies + +### Required MCP Servers +- `openshift-virtualization` - OpenShift MCP server (https://github.com/openshift/openshift-mcp-server) + +### Required MCP Tools +- `resources_list`, `resources_get`, `resources_create_or_update`, `vm_lifecycle`, `nodes_top`, `pods_top`, `nodes_stats_summary` + +### Related Skills +- `vm-inventory` - List VMs and check placement +- `vm-lifecycle-manager` - Simple start/stop +- `vm-create` - Create VMs with placement +- `vm-snapshot-create` - Backup before risky migrations + +### Reference Documentation + +**Skill Strategy Files**: +- [REBALANCE_MANUAL.md](./REBALANCE_MANUAL.md) - User-driven migration +- [REBALANCE_AUTOMATIC.md](./REBALANCE_AUTOMATIC.md) - AI-driven rebalancing + +**Performance and Best Practices**: +- [references/live-migration-best-practices.md](./references/live-migration-best-practices.md) - Configuration, requirements, networks +- [references/performance-tuning.md](./references/performance-tuning.md) - Right-sizing, overcommit, bandwidth +- [references/anti-patterns.md](./references/anti-patterns.md) - Common mistakes +- [references/production-considerations.md](./references/production-considerations.md) - HA, capacity, security + +**Troubleshooting**: +- [../../docs/troubleshooting/INDEX.md](../../docs/troubleshooting/INDEX.md) - Master index +- [../../docs/troubleshooting/scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) - ErrorUnschedulable, taints +- [../../docs/troubleshooting/storage-errors.md](../../docs/troubleshooting/storage-errors.md) - PVC access modes +- [../../docs/troubleshooting/lifecycle-errors.md](../../docs/troubleshooting/lifecycle-errors.md) - VM start/stop + +**Official Documentation**: +- [OpenShift Virt - Live Migration](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt-live-migration) +- [OpenShift Virt - Node Placement](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt-node-placement) +- [KubeVirt - Live Migration](https://kubevirt.io/user-guide/compute/live_migration/) +- [KubeVirt - Node Assignment](https://kubevirt.io/user-guide/compute/node_assignment/) +- [VirtualMachineInstanceMigration API](https://kubevirt.io/api-reference/main/definitions.html#_v1_virtualmachineinstancemigration) + +## Critical: Human-in-the-Loop Requirements + +**IMPORTANT**: This skill performs VM migrations affecting placement and availability. You MUST: + +1. **Before Initiating Migration** + - Present complete rebalance plan (VM, nodes, type, impact) + - Explain downtime (live = <1s pause, cold = 30-60s) + - Show current vs target placement + - Ask: "Confirm this migration?" + - Wait for explicit confirmation + +2. **Never Auto-Execute** + - **NEVER migrate without confirmation** + - **NEVER assume live vs cold** - ask or infer from storage + - **NEVER skip impact explanation** + - **NEVER proceed if validation fails** + +3. **For Batch Operations** + - Present all VMs to migrate + - Show total impact (e.g., "3 VMs, 2 live + 1 cold") + - Confirm entire batch before starting + - Report progress for each + - Stop on first failure + +**Why**: Live migration (brief pause, bandwidth, performance impact), Cold migration (downtime, dropped connections), Wrong node (performance degradation), Batch (network saturation) + +**Rationale**: Prevents unintended disruption; maintains user control. + +## Security Considerations + +- **RBAC Enforcement**: Requires specific permissions (create/update/list) +- **Node Access**: Respects node taints and RBAC policies +- **Storage Security**: Data remains encrypted if using encrypted storage classes +- **Network Isolation**: Migrations respect NetworkPolicies +- **Audit Trail**: All operations logged in Kubernetes API audit logs +- **KUBECONFIG Security**: Credentials never exposed +- **Resource Quotas**: Respects namespace quotas +- **Tenant Isolation**: Cannot migrate across namespaces without RBAC + +--- + +**Strategy Implementation**: ✅ REBALANCE_MANUAL.md | ✅ REBALANCE_AUTOMATIC.md + +**Reference Documentation**: ✅ live-migration-best-practices.md | ✅ performance-tuning.md | ✅ anti-patterns.md | ✅ production-considerations.md + +**Last Updated**: 2026-02-24 | **OpenShift Virtualization**: 4.17, 4.18, 4.19, 4.20 diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/anti-patterns.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/anti-patterns.md new file mode 100644 index 00000000..f1321fdc --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/anti-patterns.md @@ -0,0 +1,869 @@ +# Anti-Patterns: What NOT to Do + +**Purpose**: Common mistakes, anti-patterns, and pitfalls to avoid when rebalancing VMs in OpenShift Virtualization. + +**When to consult this document**: Before planning rebalancing operations, when troubleshooting failures, or when designing cluster architecture. + +--- + +## Official Sources + +This document is compiled from official Red Hat documentation and community best practices: + +- [Best Practices for Virtual Machine Deployments on OpenShift Virtualization](https://learn.microsoft.com/en-us/azure/openshift/best-practices-openshift-virtualization) - Microsoft Azure Red Hat OpenShift (2026-02-16) +- [Best Practices to Deploy VMs in Red Hat OpenShift Virtualization](https://docs.netapp.com/us-en/netapp-solutions-virtualization/openshift/os-osv-bpg.html) - NetApp Solutions +- [OpenShift Virtualization Best Practices](https://www.tigera.io/learn/guides/kubernetes-networking/openshift-virtualization/) - Tigera +- [Troubleshooting OpenShift Virtualization](https://access.redhat.com/articles/6256861) - Red Hat Customer Portal + +--- + +## Storage Anti-Patterns + +### ❌ Anti-Pattern 1: Using RWO Storage for Live Migration + +**What NOT to Do:** +```yaml +# BAD: VM using ReadWriteOnce storage +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: vm-disk +spec: + accessModes: + - ReadWriteOnce # Cannot live migrate! + storageClassName: gp3 +``` + +**Why It Fails:** + +From Red Hat documentation: +> "Live migration requires the use of a shared storage solution that provides ReadWriteMany (RWX) access mode. The VM disks should be backed by storage option that provides RWX access mode." + +**Error Message:** +``` +cannot migrate VMI: PVC vm-disk is not shared, live migration requires +that all PVCs must be shared (using ReadWriteMany access mode) +``` + +**Correct Approach:** + +**Before Planning Live Migration**, verify storage using MCP tools: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.spec.accessModes` includes `"ReadWriteMany"`. + +**If RWO storage**, use **cold migration** instead (see REBALANCE_MANUAL.md). + +**Storage Types Supporting RWX:** +- ✅ NFS (ontap-nas driver) +- ✅ SMB/CIFS (ontap-nas driver) +- ✅ iSCSI/FC (ontap-san driver, **raw block mode only**) +- ❌ AWS EBS gp3 (RWO only) +- ❌ Local storage (RWO only) + +--- + +### ❌ Anti-Pattern 2: Not Setting SVM Volume Limits + +**What NOT to Do:** + +Deploy Trident without configuring SVM (Storage Virtual Machine) volume limits, allowing unchecked resource consumption. + +**Why It's Dangerous:** + +From NetApp documentation: +> "Set volume limits to prevent Trident from consuming all storage" + +**Impact:** +- Trident creates unlimited volumes +- Storage backend exhausted +- Other workloads starved of storage +- Production outages + +**Correct Approach:** + +Configure limits at multiple levels: + +**1. SVM-level volume limit:** +```bash +vserver modify -vserver -max-volumes +``` + +**2. Storage limits on SVM:** +```bash +vserver create -vserver vserver_name -aggregate aggregate_name -storage-limit value +``` + +**3. Trident backend parameters:** +- `limitVolumeSize`: Maximum volume size created by Trident (e.g., "100Gi") +- `limitVolumePoolSize`: Maximum FlexVol size for economy drivers (e.g., "500Gi") + +--- + +### ❌ Anti-Pattern 3: Enabling showmount Without Justification + +**What NOT to Do:** + +Leave `showmount` enabled on NFS SVMs, exposing volume information to unauthorized clients. + +**Why It's a Security Risk:** + +From NetApp documentation: +> "Disable showmount to prevent unauthorized volume discovery" + +**Correct Approach:** + +Disable showmount unless specifically required: + +```bash +vserver nfs modify -vserver -showmount disabled +``` + +Implement separate export policies for infrastructure vs application nodes for granular access control. + +--- + +## Scheduling and Node Placement Anti-Patterns + +### ❌ Anti-Pattern 4: Excessive Affinity Rules + +**What NOT to Do:** + +Apply too many complex affinity, anti-affinity, node selector, and toleration rules to VMs. + +**Why It's Problematic:** + +From community best practices: +> "Too many rules make scheduling slow and hard to reason about." + +From Red Hat documentation: +> "Affinity rules only apply during scheduling. OpenShift Container Platform does not reschedule running workloads if the constraints are no longer met." + +**Impact:** +- Slow VM scheduling (scheduler overhead) +- Impossible-to-satisfy constraints (VM stuck in Pending) +- Difficult troubleshooting (complex rule interactions) +- No automatic rebalancing when constraints violated + +**Correct Approach:** + +**Keep rules simple and minimal:** + +```yaml +# GOOD: Simple, clear node selector +spec: + template: + spec: + nodeSelector: + workload-type: virtualization +``` + +```yaml +# BAD: Too many overlapping constraints +spec: + template: + spec: + nodeSelector: + node-role.kubernetes.io/worker: "" + workload-type: virtualization + zone: us-east-1a + instance-type: m5.4xlarge + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: [web, database, cache] + topologyKey: kubernetes.io/hostname + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: tier + operator: In + values: [frontend] + topologyKey: failure-domain.beta.kubernetes.io/zone + tolerations: + - key: dedicated + operator: Equal + value: virtualization + effect: NoSchedule + - key: high-performance + operator: Exists + effect: NoSchedule +``` + +**Recommendation:** +- Use **one** primary constraint (nodeSelector OR affinity) +- Add tolerations only when nodes have taints +- Avoid mixing required and preferred affinity rules +- Document the intent of each rule + +--- + +### ❌ Anti-Pattern 5: Not Planning for Node Failures + +**What NOT to Do:** + +Rely on automatic VM failover without configuring machine health checks. + +**Why It Fails:** + +From Red Hat documentation: +> "If a node fails and machine health checks are not deployed on your cluster, virtual machines (VMs) with RunStrategy: Always configured are not automatically relocated to healthy nodes. To trigger VM failover, you must manually delete the Node object." + +**Impact:** +- VMs remain assigned to failed node +- Manual intervention required for recovery +- Extended downtime during node failures + +**Correct Approach:** + +**1. Deploy Machine Health Checks:** + +Configure cluster-level machine health checks to detect and remediate node failures automatically. + +**2. Use RunStrategy: Always for HA VMs:** + +```yaml +spec: + runStrategy: Always # Ensures VM restarts after node recovery +``` + +**3. Implement VM Replication:** + +For critical VMs, create replicas with anti-affinity rules to ensure distribution across different nodes/zones. + +**4. Monitor Node Health:** + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +Filter for nodes where `.status.conditions[]` shows `Ready=False` or other unhealthy states. + +--- + +## Resource Management Anti-Patterns + +### ❌ Anti-Pattern 6: Exceeding CPU Overcommit Limits + +**What NOT to Do:** + +Configure CPU overcommit ratio >1.8x physical cores. + +**Why It's Dangerous:** + +From Red Hat documentation: +> "CPU over-commitment ratio must not exceed 1.8x of the number of physical cores while memory usage may not exceed 0.9x of the physical memory available in a cluster. CPU over-commitment leads to throttling, causing slowness of all workloads on the impacted node." + +**Impact:** +- CPU throttling across ALL VMs on node +- Unpredictable performance degradation +- Cascading slowness affecting entire cluster +- User-facing application latency + +**Correct Approach:** + +**Check Current Overcommit:** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "hco.kubevirt.io/v1beta1", + "kind": "HyperConverged", + "namespace": "openshift-cnv", + "name": "kubevirt-hyperconverged" +} +``` + +Review `.spec.resourceRequirements.vmiCPUAllocationRatio`. + +**Safe Limits:** +- **Production**: 1.0-1.2x (no/minimal overcommit) +- **Dev/Test**: 1.2-1.5x (moderate overcommit) +- **Absolute Maximum**: 1.8x (with careful monitoring) + +**Never Exceed**: 1.8x CPU or 0.9x memory limits. + +--- + +### ❌ Anti-Pattern 7: Applying Strict Resource Limits to VMs + +**What NOT to Do:** + +Set both resource requests **and** limits on VMs without specific governance requirements. + +**Why It's Problematic:** + +From Microsoft Azure Red Hat OpenShift guidance: +> "Avoid strict resource limits: Set only guest memory for VMs; avoid strict resource limits unless required for governance." + +**Impact:** +- CPU throttling even when node has spare capacity +- Reduced VM performance +- Wasted cluster resources +- Difficult troubleshooting (invisible throttling) + +**Correct Approach:** + +**Set requests only:** + +```yaml +# GOOD: Requests only (allows bursting) +spec: + template: + spec: + domain: + resources: + requests: + memory: 16Gi + cpu: 4 +``` + +```yaml +# BAD: Requests + limits (strict throttling) +spec: + template: + spec: + domain: + resources: + requests: + memory: 16Gi + cpu: 4 + limits: # Avoid unless required + memory: 16Gi + cpu: 4 +``` + +**Only set limits when:** +- Governance policies mandate strict resource boundaries +- Multi-tenant environments require isolation +- Preventing one VM from starving others + +--- + +### ❌ Anti-Pattern 8: Relying on On-Premises Sizing References + +**What NOT to Do:** + +Size VMs in OpenShift Virtualization based on on-premises VM sizes without testing. + +**Why It Fails:** + +From Microsoft Azure Red Hat OpenShift guidance: +> "Avoid relying solely on on-premises sizing references; benchmark your own workloads to inform right sizing." + +**Impact:** +- Overprovisioned VMs (wasted resources) +- Underprovisioned VMs (performance issues) +- Unexpected architectural overhead (VMs != native pods) +- Incorrect migration time estimates + +**Correct Approach:** + +**1. Benchmark workloads in OpenShift Virtualization:** +- Deploy test VMs with various sizes +- Run representative workload tests +- Measure actual performance vs requirements + +**2. Account for architectural overhead:** + +Expect 4-56% performance overhead vs bare metal (see performance-tuning.md for details). + +**3. Monitor and adjust:** + +**MCP Tool**: `pods_top` (from openshift-virtualization) + +**Parameters**: +```json +{ + "all_namespaces": true, + "label_selector": "kubevirt.io=virt-launcher" +} +``` + +Track actual resource usage and resize VMs accordingly. + +--- + +## Network Anti-Patterns + +### ❌ Anti-Pattern 9: Using OVN-Kubernetes with Linux Bridge on Default Interface + +**What NOT to Do:** + +Attempt to attach a Linux bridge or bonding device to the host's default interface when using OVN-Kubernetes CNI. + +**Why It Fails:** + +From Red Hat documentation: +> "If your OpenShift Container Platform cluster uses OVN-Kubernetes as the default CNI provider, you cannot attach a Linux bridge or bonding device to a host's default interface." + +**Impact:** +- Network configuration failures +- VM networking broken +- Migration network setup fails + +**Correct Approach:** + +**Option 1: Use secondary network interface** + +Attach Linux bridge to a different physical interface (not the default). + +**Option 2: Switch to OpenShift SDN CNI** + +If Linux bridge on default interface is required, reconfigure cluster to use OpenShift SDN instead of OVN-Kubernetes. + +**Option 3: Use OVS bridge instead** + +For migration networks, use Open vSwitch bridge (compatible with OVN-Kubernetes). + +--- + +### ❌ Anti-Pattern 10: Ignoring MTU Mismatches + +**What NOT to Do:** + +Mix network types with different default MTUs without explicit configuration. + +**Why It's Problematic:** + +From Red Hat documentation: +> "When a virtual machine interface is connected to an OVS bridge, the default MTU is 1400, but when connected to a Linux bridge, the default MTU is 1500." + +**Impact:** +- Packet fragmentation +- Reduced network performance +- Subtle communication failures +- Migration slowdowns + +**Correct Approach:** + +**Explicitly set MTU in NetworkAttachmentDefinition:** + +```json +{ + "cniVersion": "0.3.1", + "name": "migration-bridge", + "type": "macvlan", + "master": "eth1", + "mode": "bridge", + "mtu": 9000, # Explicit MTU setting + "ipam": {...} +} +``` + +**Validate MTU consistency across all interfaces involved in migration.** + +--- + +## Architecture and Platform Anti-Patterns + +### ❌ Anti-Pattern 11: Using RHEL Compute Nodes + +**What NOT to Do:** + +Deploy OpenShift Virtualization on Red Hat Enterprise Linux (RHEL) compute nodes. + +**Why It Fails:** + +From Red Hat documentation: +> "OpenShift Virtualization requires Red Hat Enterprise Linux CoreOS (RHCOS) compute nodes. Even though it is possible to deploy Red Hat Enterprise Linux (RHEL) compute nodes, they are incompatible with OpenShift Virtualization." + +**Impact:** +- VM scheduling failures +- Unsupported configuration +- Migration failures +- No Red Hat support + +**Correct Approach:** + +**Verify all nodes are RHCOS:** + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node, check `.status.nodeInfo.osImage` contains "CoreOS". + +**If RHEL nodes detected:** +- Replace with RHCOS nodes +- Do NOT schedule VMs on RHEL nodes +- Remove RHEL nodes from cluster before deploying virtualization workloads + +--- + +### ❌ Anti-Pattern 12: Placing Master Nodes on Same VMware Host + +**What NOT to Do:** + +In VMware-based deployments, place multiple OpenShift master nodes on the same VMware ESXi host. + +**Why It's Dangerous:** + +From VMware best practices: +> "Critical best practices include: distributing the 3 virtual master nodes across different VMware hosts, placing each master node on a separate datastore, and avoiding hosting master nodes on datastores with high I/O workloads." + +**Impact:** +- Single point of failure (host failure kills multiple masters) +- etcd performance degradation (etcd is latency-sensitive) +- Cluster control plane outage +- Violates high-availability principles + +**Correct Approach:** + +**1. Distribute master nodes across different VMware hosts** + +Use VM anti-affinity rules to enforce separation. + +**2. Use separate datastores for each master node** + +Prevents storage failure from affecting multiple masters. + +**3. Avoid high I/O datastores for master nodes** + +etcd is sensitive to disk latency; use low-latency storage. + +--- + +### ❌ Anti-Pattern 13: Ignoring etcd Latency Sensitivity + +**What NOT to Do:** + +Place etcd (control plane) on high-latency storage or overloaded nodes. + +**Why It's Critical:** + +From best practices: +> "The etcd component hosted on control-plane nodes is usually the component most sensitive to latency issues." + +**Impact:** +- Cluster control plane slowness +- API server timeouts +- Failed VM operations +- Cluster instability + +**Correct Approach:** + +**1. Use low-latency storage for control plane nodes:** +- SSD-backed storage (not HDD) +- Local NVMe if available +- Avoid shared storage with high I/O contention + +**2. Monitor etcd latency:** + +Prometheus metrics: `etcd_disk_wal_fsync_duration_seconds` + +**Target**: <10ms for WAL fsync + +**3. Isolate control plane from VM workloads:** + +Use taints on master nodes to prevent VM scheduling. + +--- + +## Migration Operation Anti-Patterns + +### ❌ Anti-Pattern 14: Not Reducing VM Workload During Migration + +**What NOT to Do:** + +Attempt live migration of write-intensive VMs (databases, caches) under full load. + +**Why It's Problematic:** + +High memory write rate (dirty page rate) can exceed network transfer rate, preventing migration convergence. + +**Impact:** +- Migration timeouts +- Failed migrations +- Extended migration duration +- Network saturation + +**Correct Approach:** + +**Before migrating write-intensive VMs:** + +1. **Schedule migration during low-activity window** (off-hours, maintenance window) + +2. **Temporarily reduce workload:** + - Stop non-critical background processes + - Scale down application traffic + - Pause batch jobs + +3. **Consider cold migration instead** for extremely write-heavy workloads (guaranteed completion) + +4. **Increase timeouts if load cannot be reduced:** + +Modify `.spec.liveMigrationConfig.completionTimeoutPerGiB` in HyperConverged CR (see performance-tuning.md). + +--- + +### ❌ Anti-Pattern 15: Parallel Migrations Without Dedicated Network + +**What NOT to Do:** + +Run many concurrent migrations on shared application network without bandwidth limits. + +**Why It's Dangerous:** + +- Saturates network bandwidth +- Degrades application performance +- Migration failures due to slow transfers +- Cascading performance impact + +**Impact Observed:** + +From search results: +> "Network saturation risk with concurrent migrations" + +**Correct Approach:** + +**Option 1: Use dedicated migration network** (see live-migration-best-practices.md) + +**Option 2: Limit concurrent migrations:** + +Modify `.spec.liveMigrationConfig.parallelMigrationsPerCluster` in HyperConverged CR: + +```yaml +spec: + liveMigrationConfig: + parallelMigrationsPerCluster: 3 # Conservative limit + bandwidthPerMigration: 64Mi # Bandwidth cap per migration +``` + +**Option 3: Migrate sequentially** + +Migrate VMs one at a time instead of batch operations. + +--- + +### ❌ Anti-Pattern 16: Not Validating Migration Prerequisites + +**What NOT to Do:** + +Attempt migration without verifying storage, network, and capacity prerequisites. + +**Why It Fails:** + +Common failures: +- RWO storage → "PVC is not shared" error +- VM not running → "cannot migrate stopped VM" +- Node at capacity → ErrorUnschedulable +- Network issues → Migration timeout + +**Correct Approach:** + +**Always run pre-migration validation** (see live-migration-best-practices.md for complete checklist): + +1. ✅ Verify PVC access modes (RWX required) +2. ✅ Check VM is running (VMI exists) +3. ✅ Validate target node capacity +4. ✅ Confirm virt-handler pods healthy +5. ✅ Check cluster migration limits + +**Use Common Validation Logic from SKILL.md before every migration.** + +--- + +## Production Deployment Anti-Patterns + +### ❌ Anti-Pattern 17: Deploying to Production Without Testing + +**What NOT to Do:** + +Deploy VMs directly to production without dev/test validation. + +**Why It's Risky:** + +From best practices: +> "Begin with non-critical or dev/test workloads before moving production systems - this phased approach allows teams to gain hands-on experience while minimizing risk." + +**Impact:** +- Unexpected performance issues +- Migration failures affecting production +- Learning curve impacts critical systems +- Difficult rollback + +**Correct Approach:** + +**Phased Rollout:** + +**Phase 1: Development/Test** +- Deploy test VMs +- Validate performance and functionality +- Test migration workflows +- Gain operational experience + +**Phase 2: Non-Critical Production** +- Migrate non-critical workloads +- Monitor performance and stability +- Refine sizing and configurations +- Build confidence + +**Phase 3: Critical Production** +- Migrate critical workloads +- Ensure HA and DR configured +- 24/7 monitoring in place +- Rollback plan ready + +--- + +### ❌ Anti-Pattern 18: Starting Big Instead of Small + +**What NOT to Do:** + +Provision large VM fleet from day one without iterative growth. + +**Why It's Problematic:** + +From best practices: +> "When starting with OpenShift Virtualization, it's essential to start small and scale up as needed to avoid over-provisioning and wasting resources." + +**Impact:** +- Overprovisioned cluster (wasted costs) +- Underutilized resources +- Difficult rightsizing later +- Commitment to suboptimal architecture + +**Correct Approach:** + +**Start small:** +1. Deploy 5-10 VMs initially +2. Monitor resource usage patterns +3. Adjust sizing based on actual metrics +4. Gradually add VMs as needs grow + +**Validate assumptions:** +- Test architectural overhead +- Measure actual performance +- Refine resource allocation +- Iterate on configuration + +--- + +### ❌ Anti-Pattern 19: Not Monitoring After Rebalancing + +**What NOT to Do:** + +Execute rebalancing operations and assume everything is optimal without validation. + +**Why It's Risky:** + +- May not achieve intended load distribution +- Hidden performance degradation +- VMs scheduled suboptimally +- Resource contention not detected + +**Correct Approach:** + +**Post-Rebalancing Validation:** + +**1. Verify VM placement:** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +For each migrated VM, confirm `.status.nodeName` matches expected target node. + +**2. Monitor node resource usage:** + +**MCP Tool**: `nodes_top` (from openshift-virtualization) + +**Before vs After Comparison:** + +| Node | CPU Before | CPU After | Improvement | +|------|------------|-----------|-------------| +| worker-01 | 85% | 65% | -20% ✓ | +| worker-02 | 78% | 64% | -14% ✓ | +| worker-03 | 42% | 58% | +16% | +| worker-04 | 38% | 53% | +15% | + +**3. Validate application performance:** + +Check application-specific metrics (response time, throughput, error rates). + +**4. Monitor for 24-48 hours:** + +Ensure sustained improvement without unexpected side effects. + +--- + +## Summary: Anti-Pattern Checklist + +Before rebalancing VMs, avoid these critical mistakes: + +**Storage:** +- ❌ Using RWO storage for live migration +- ❌ Not setting SVM volume limits +- ❌ Leaving showmount enabled + +**Scheduling:** +- ❌ Too many complex affinity rules +- ❌ Not configuring machine health checks + +**Resources:** +- ❌ CPU overcommit >1.8x +- ❌ Strict resource limits without justification +- ❌ Using on-premises sizing without testing + +**Network:** +- ❌ Linux bridge on default interface with OVN-Kubernetes +- ❌ Ignoring MTU mismatches +- ❌ Parallel migrations without dedicated network + +**Platform:** +- ❌ Using RHEL compute nodes +- ❌ Master nodes on same VMware host +- ❌ Ignoring etcd latency sensitivity + +**Operations:** +- ❌ Not reducing VM workload during migration +- ❌ Skipping pre-migration validation +- ❌ No post-rebalancing monitoring + +**Production:** +- ❌ Deploying to production without testing +- ❌ Starting big instead of small + +--- + +## Related Documentation + +- [Live Migration Best Practices](./live-migration-best-practices.md) - What TO do for successful migrations +- [Performance Tuning](./performance-tuning.md) - Optimization strategies +- [Production Considerations](./production-considerations.md) - Right-sizing, workload planning, HA strategies + +--- + +**Last Updated**: 2026-02-24 +**OpenShift Virtualization Versions**: 4.17, 4.18, 4.19, 4.20 +**Status**: Curated from official Red Hat sources and production experience diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/live-migration-best-practices.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/live-migration-best-practices.md new file mode 100644 index 00000000..7df97527 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/live-migration-best-practices.md @@ -0,0 +1,794 @@ +# Live Migration Best Practices + +**Purpose**: Configuration parameters, requirements, and best practices for VM live migration in OpenShift Virtualization. + +**When to consult this document**: Before executing live migrations, when configuring cluster-wide migration settings, or when troubleshooting migration performance issues. + +--- + +## Official Sources + +This document is compiled from official Red Hat documentation: + +- [Live Migrating VMs with OpenShift Virtualization](https://developers.redhat.com/articles/2025/07/14/live-migrating-vms-openshift-virtualization) - Red Hat Developer (2025-07-14) +- [How OpenShift Virtualization Supports VM Live Migration](https://developers.redhat.com/articles/2025/06/05/how-openshift-virtualization-supports-vm-live-migration) - Red Hat Developer (2025-06-05) +- [Chapter 12. Live Migration - OpenShift Container Platform 4.18](https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/virtualization/live-migration) - Red Hat Documentation +- [Best Practices for Virtual Machine Deployments on OpenShift Virtualization](https://learn.microsoft.com/en-us/azure/openshift/best-practices-openshift-virtualization) - Microsoft Azure Red Hat OpenShift (2026-02-16) +- [Best Practices to Deploy VMs in Red Hat OpenShift Virtualization](https://docs.netapp.com/us-en/netapp-solutions-virtualization/openshift/os-osv-bpg.html) - NetApp Solutions + +--- + +## Configuration Parameters + +### HyperConverged CR Live Migration Settings + +All live migration settings are configured in the `HyperConverged` custom resource located in the `openshift-cnv` namespace. + +**Default Configuration:** +```yaml +apiVersion: hco.kubevirt.io/v1beta1 +kind: HyperConverged +metadata: + name: kubevirt-hyperconverged + namespace: openshift-cnv +spec: + liveMigrationConfig: + completionTimeoutPerGiB: 800 # Seconds per GiB for migration completion + parallelMigrationsPerCluster: 5 # Max concurrent migrations cluster-wide + parallelOutboundMigrationsPerNode: 2 # Max concurrent migrations per source node + progressTimeout: 150 # Max seconds without progress before cancellation + bandwidthPerMigration: 64Mi # (Optional) Bandwidth limit per migration + network: "" # (Optional) Dedicated secondary network for migration +``` + +**Parameter Explanations:** + +| Parameter | Default | Description | Tuning Guidance | +|-----------|---------|-------------|-----------------| +| `completionTimeoutPerGiB` | 800s | Migration completion duration per gigabyte of VM memory | Increase for high memory write rate (dirty page) workloads | +| `progressTimeout` | 150s | Maximum seconds without migration progress before cancellation | Increase for large VMs (>100GB) or slow networks | +| `parallelMigrationsPerCluster` | 5 | Cluster-wide concurrent migration limit | Increase if network bandwidth allows; decrease if saturation occurs | +| `parallelOutboundMigrationsPerNode` | 2 | Per-node concurrent outbound migration limit | Keep at 2 to prevent single-node overload | +| `bandwidthPerMigration` | 64Mi | (Optional) Bandwidth limit per migration | Set to prevent network saturation; omit for unlimited | +| `network` | "" | (Optional) NetworkAttachmentDefinition for dedicated migration network | Highly recommended for production; see Dedicated Networks section | + +**How to Update Configuration Using MCP Tools:** + +**Step 1: Get current HyperConverged resource** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "hco.kubevirt.io/v1beta1", + "kind": "HyperConverged", + "namespace": "openshift-cnv", + "name": "kubevirt-hyperconverged" +} +``` + +**Step 2: Modify the returned JSON to update liveMigrationConfig** + +Add or update the `.spec.liveMigrationConfig` section: +```json +{ + "spec": { + "liveMigrationConfig": { + "completionTimeoutPerGiB": 1200, + "parallelMigrationsPerCluster": 10, + "progressTimeout": 300, + "bandwidthPerMigration": "32Mi" + } + } +} +``` + +**Step 3: Apply the updated configuration** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +```json +{ + "resource": "" +} +``` + +Pass the complete modified HyperConverged resource as YAML or JSON string. + +--- + +## Prerequisites and Requirements + +### Storage Requirements + +**CRITICAL**: Live migration requires **ReadWriteMany (RWX)** access mode storage. + +**Supported Storage Types for Live Migration:** + +| Storage Type | Access Mode | Live Migration Support | Notes | +|--------------|-------------|------------------------|-------| +| NFS (ontap-nas driver) | RWX | ✅ Supported | Recommended for general use | +| SMB/CIFS (ontap-nas driver) | RWX | ✅ Supported | Windows-compatible | +| iSCSI/FC (ontap-san driver) | RWX (raw block mode only) | ✅ Supported | High performance; requires raw block volumes | +| Local storage / AWS EBS (gp3) | RWO | ❌ NOT Supported | Use cold migration instead | + +**Validation Using MCP Tools:** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +**Expected Output**: Check `.spec.accessModes` in the returned PVC resource. + +For live migration, access modes must include `"ReadWriteMany"`. + +**What Happens with RWO Storage:** + +When attempting live migration with ReadWriteOnce (RWO) storage: +``` +Error: cannot migrate VMI: PVC is not shared, live migration requires +that all PVCs must be shared (using ReadWriteMany access mode) +``` + +**Solution**: Use cold migration workflow for VMs with RWO storage (see REBALANCE_MANUAL.md). + +--- + +### Hardware and Network Requirements + +**Minimum Requirements:** + +- **Nodes**: Red Hat Enterprise Linux CoreOS (RHCOS) compute nodes (RHEL nodes are incompatible) +- **Network**: All nodes must be on the same L2 network or have routable connectivity +- **CPU**: Sufficient CPU headroom on target node for incoming VM workload +- **Memory**: Sufficient free memory on target node (>= VM memory allocation) + +**Recommended for Production:** + +- **Network Cards**: 100Gbps NICs for large VM migrations (>500GB memory) +- **Dedicated Migration Network**: Secondary physical network or VLAN for isolation +- **Storage Backend**: SSD-backed shared storage (NFS-CSI, OpenShift Data Foundation, Azure NetApp Files) +- **MTU Configuration**: Set to 9000 for migration networks to improve efficiency + +--- + +## Dedicated Migration Network (Production Best Practice) + +### Why Use a Dedicated Network? + +**Benefits:** +- Isolates migration traffic from application workloads +- Prevents network contention and performance degradation +- Enables higher bandwidth allocation (e.g., 100Gbps dedicated) +- Improves security and manageability +- Reduces migration time for large VMs + +**When to Use:** +- Production environments with large VMs (>100GB memory) +- Clusters with high application network traffic +- Environments requiring strict network isolation +- High-availability requirements with frequent migrations + +### Configuration Example + +**Step 1: Create NodeNetworkConfigurationPolicy (NNCP)** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +```json +{ + "resource": "apiVersion: nmstate.io/v1\nkind: NodeNetworkConfigurationPolicy\nmetadata:\n name: migration-network-policy\nspec:\n desiredState:\n interfaces:\n - name: br-lm\n description: OVS bridge for live migration\n type: ovs-bridge\n state: up\n bridge:\n allow-extra-patch-ports: true\n port:\n - name: enp4s0\n vlan:\n mode: access\n tag: 3030\n options:\n stp: false" +} +``` + +**Step 2: Create NetworkAttachmentDefinition (NAD)** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +```json +{ + "resource": "apiVersion: k8s.cni.cncf.io/v1\nkind: NetworkAttachmentDefinition\nmetadata:\n name: migration-network\n namespace: openshift-cnv\nspec:\n config: '{\n \"cniVersion\": \"0.3.1\",\n \"name\": \"migration-bridge\",\n \"type\": \"macvlan\",\n \"master\": \"eth1\",\n \"mode\": \"bridge\",\n \"ipam\": {\n \"type\": \"whereabouts\",\n \"range\": \"10.200.5.0/24\",\n \"excludeSubnets\": \"10.200.5.0/30\"\n }\n }'" +} +``` + +**Step 3: Configure HyperConverged CR to Use Network** + +**MCP Tool**: Get current HyperConverged, modify, and update using `resources_create_or_update` + +Add to `.spec.liveMigrationConfig`: +```json +{ + "network": "migration-network" +} +``` + +**Step 4: Verify virt-handler Pods Restarted** + +**MCP Tool**: `pods_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "labelSelector": "kubevirt.io=virt-handler" +} +``` + +**Expected**: All pods show READY status and recent start time (AGE). + +Filter results where `status.containerStatuses[0].ready == true` and `status.containerStatuses[0].restartCount` is recent. + +**Verification After Migration:** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "namespace": "", + "name": "" +} +``` + +Check `.status.migrationState.targetNodeAddress` - should be an IP from the dedicated subnet (e.g., 10.200.5.15). + +--- + +## Migration Process and Technologies + +### Pre-copy Migration + +Live migration uses **pre-copy** strategy: + +1. **Initial Copy**: VM continues running on source node while memory is copied to target +2. **Iterative Copy**: Pages modified during copy (dirty pages) are re-copied +3. **Cutover**: Brief pause (<1 second) to copy final dirty pages and switch execution +4. **Cleanup**: Source VM instance is terminated + +**Multi-fd Technology** (for high-load scenarios): + +- Sends data over multiple network streams in parallel +- Maximizes bandwidth utilization during migration +- Handles high dirty page rates (e.g., SAP HANA, databases with high write rates) +- Automatically enabled by KubeVirt when beneficial + +**Migration Phases:** + +``` +Pending → Scheduling → PreparingTarget → Running → Succeeded +``` + +**Monitor with MCP Tools:** + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration" +} +``` + +Filter results by `.status.phase` to see current migration status. + +--- + +## Best Practices + +### 1. VM Resource Optimization + +**Enable Dedicated Resources:** + +Configure VMs with dedicated CPU and memory isolation for performance-sensitive workloads: + +```yaml +spec: + template: + spec: + domain: + cpu: + dedicatedCpuPlacement: true + resources: + requests: + memory: 16Gi +``` + +**Benefits:** +- Improves VM performance and latency predictability +- Reduces migration time (less CPU contention) +- Better accuracy for latency predictions + +**When to Use:** +- Database workloads (PostgreSQL, MySQL, SAP HANA) +- Real-time analytics applications +- Low-latency requirements + +### 2. Hugepage Configuration + +For large VMs (>100GB memory), configure hugepages to reduce memory page overhead: + +**Node Configuration Using MCP Tools:** + +**MCP Tool**: `resources_get` then `resources_create_or_update` (from openshift-virtualization) + +**Step 1: Get Node resource** + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node", + "name": "" +} +``` + +**Step 2: Add label to node** + +Modify returned JSON to add `.metadata.labels.cpumanager = "true"`, then update with `resources_create_or_update`. + +**VM Configuration:** +```yaml +spec: + template: + spec: + domain: + memory: + hugepages: + pageSize: 1Gi +``` + +**Benefits:** +- Reduces page-dirtying overhead during migration +- Improves memory access performance +- Faster migration completion for very large VMs (tested with 1TB VMs) + +### 3. Network Optimization + +**Set Network MTU to 9000** (jumbo frames): + +Configure in the NetworkAttachmentDefinition used for migration: + +```yaml +spec: + liveMigrationConfig: + network: migration-network # NetworkAttachmentDefinition with MTU 9000 +``` + +**Benefits:** +- Significantly improves network efficiency +- Reduces packet overhead +- Faster data transfer for large VM migrations + +**Validate MTU Setting Using MCP Tools:** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "openshift-cnv", + "name": "migration-network" +} +``` + +Check `.spec.config` for MTU setting in the JSON configuration. + +### 4. Storage Configuration + +**For Testing/Development:** +- NFS-CSI with SSD backend storage +- Shared storage accessible from all nodes + +**For Production:** +- OpenShift Data Foundation (ODF) with SSD-backed storage +- Azure NetApp Files with appropriate performance tier +- NetApp ONTAP with dedicated SVM for virtualization workloads + +**Storage Validation Before Migration Using MCP Tools:** + +For each VM in rebalance plan: + +**Step 1: Get VM resource** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Step 2: Extract PVC names from `.spec.template.spec.volumes[].persistentVolumeClaim.claimName`** + +**Step 3: For each PVC, verify access mode** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +**Expected**: `.spec.accessModes` must include `"ReadWriteMany"`. + +### 5. Concurrency Management + +**Default Limits:** +- **Cluster-wide**: 5 concurrent migrations +- **Per-node outbound**: 2 concurrent migrations + +**When to Increase:** +- Cluster has sufficient network bandwidth (100Gbps+ NICs) +- Dedicated migration network is configured +- Routine maintenance window with many VMs to migrate + +**When to Decrease:** +- Network saturation detected (monitor with Prometheus) +- Migration failures due to timeouts +- Shared application network (no dedicated migration network) + +**Monitoring Network Saturation Using MCP Tools:** + +**MCP Tool**: `nodes_stats_summary` (from openshift-virtualization) + +**Parameters**: +```json +{ + "name": "" +} +``` + +Review `.network.interfaces[].rxBytes` and `.network.interfaces[].txBytes` for throughput metrics. + +Alternatively, use `nodes_top` for current resource usage: + +**MCP Tool**: `nodes_top` (from openshift-virtualization) + +**Parameters**: +```json +{ + "name": "" +} +``` + +### 6. Pre-Migration Validation Checklist + +Before initiating migration: + +1. ✅ **Storage**: Verify all PVCs use ReadWriteMany (RWX) access mode +2. ✅ **Network**: Confirm all nodes are network-accessible +3. ✅ **Capacity**: Verify target node has sufficient CPU and memory +4. ✅ **Health**: Check `virt-handler` pods are Running (1/1) on all nodes +5. ✅ **Workload**: Consider VM workload intensity (reduce load if possible) +6. ✅ **Limits**: Check current cluster migration count < `parallelMigrationsPerCluster` + +**Validation Using MCP Tools:** + +**1. Check PVC Access Modes:** + +For each VM, use `resources_get` to get VirtualMachine, extract PVC names, then `resources_get` for each PVC and verify `.spec.accessModes` includes `"ReadWriteMany"`. + +**2. Check virt-handler Health:** + +**MCP Tool**: `pods_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "labelSelector": "kubevirt.io=virt-handler" +} +``` + +Filter results where `status.containerStatuses[0].ready == true`. All pods must show ready status. + +**3. Check Current Migration Count:** + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration" +} +``` + +Count results where `.status.phase` is NOT "Succeeded" or "Failed". Compare to `parallelMigrationsPerCluster` limit from HyperConverged CR. + +**4. Check Target Node Capacity:** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node", + "name": "" +} +``` + +Review `.status.allocatable` and `.status.capacity` for available CPU and memory. + +Alternatively use `nodes_top`: + +**MCP Tool**: `nodes_top` (from openshift-virtualization) + +**Parameters**: +```json +{ + "name": "" +} +``` + +--- + +## Test Results and Validation + +### SAP HANA 1TB VM Live Migration (Red Hat Developer Article 2025-07-14) + +**Test Environment:** +- **Hardware**: 8-socket Intel Xeon Platinum, 12TB memory, 100Gbps NICs +- **OpenShift**: 4.17.15 +- **VM Size**: 1TB memory (SAP HANA 2.00.081.00.1733303410) +- **Storage**: NFS-CSI with SSD backend +- **Network**: Dedicated 100Gbps secondary network, MTU 9000 + +**Results:** +- ✅ **Idle/Cooled-off**: Live migration completed successfully with **zero data loss or corruption** +- ✅ **High-load**: Migrations progressed as expected even with large volumes of dirty pages +- ✅ **Integrity**: Full VM and data integrity maintained; failed migrations safely canceled +- ⏱️ **Duration**: ~30-60 seconds for typical VMs; longer for 1TB VM under load + +**Key Findings:** +- Multi-fd technology enabled migrations to continue transferring data quickly while dirty pages were being generated +- Dedicated 100Gbps network critical for large VM migrations +- 1GB hugepages reduced page-dirtying overhead + +--- + +## Limitations and Constraints + +### Migration Requirements + +**MUST HAVE for Live Migration:** +- ReadWriteMany (RWX) storage on all VM volumes +- VM currently running (VirtualMachineInstance exists) +- Target node has sufficient capacity (CPU, memory) +- All nodes have RHCOS (not RHEL) + +**CANNOT Live Migrate When:** +- VM uses ReadWriteOnce (RWO) storage → Use cold migration +- VM is stopped (no VirtualMachineInstance) → Use cold migration or start VM first +- Target node is cordoned or NotReady → Choose different target +- Cluster at `parallelMigrationsPerCluster` limit → Wait for completion + +### Known Issues + +**Single Node OpenShift (SNO):** +- VMs created from common templates with `evictionStrategy: LiveMigrate` trigger `VMCannotBeEvicted` alert +- **Workaround**: Use `evictionStrategy: None` for SNO clusters + +**OVN-Kubernetes CNI:** +- Cannot attach Linux bridge or bonding device to host's default interface +- **Workaround**: Use secondary network interface or switch to OpenShift SDN CNI + +**MTU Differences:** +- OVS bridge default MTU: 1400 +- Linux bridge default MTU: 1500 +- **Impact**: May cause fragmentation; configure MTU explicitly + +--- + +## Troubleshooting Common Issues + +### Issue 1: Migration Timeout + +**Symptom:** +``` +Migration exceeded timeout: 150 seconds per GiB +``` + +**Causes:** +- High memory write rate (dirty page rate exceeds transfer rate) +- Insufficient network bandwidth +- Large VM memory size + +**Solutions:** + +**1. Increase timeout (temporary):** + +Use `resources_get` to fetch HyperConverged CR, modify `.spec.liveMigrationConfig`, then update with `resources_create_or_update`: + +```json +{ + "spec": { + "liveMigrationConfig": { + "completionTimeoutPerGiB": 1200, + "progressTimeout": 300 + } + } +} +``` + +**2. Reduce VM workload** during migration: +- Stop write-intensive processes temporarily +- Schedule migration during low-activity window + +**3. Use cold migration** instead (guaranteed completion - see REBALANCE_MANUAL.md) + +**4. Configure auto-converge** (cluster-level KubeVirt setting): +- Throttles vCPU to reduce dirty page rate +- Enables migration convergence for high write-rate VMs + +### Issue 2: Network Saturation + +**Symptom:** +- Multiple concurrent migrations slow or fail +- High network utilization on migration network + +**Solutions:** + +**1. Reduce concurrent migrations:** + +Use `resources_get` to fetch HyperConverged CR, modify `.spec.liveMigrationConfig.parallelMigrationsPerCluster`, then update with `resources_create_or_update`: + +```json +{ + "spec": { + "liveMigrationConfig": { + "parallelMigrationsPerCluster": 3 + } + } +} +``` + +**2. Set bandwidth limit per migration:** + +Modify `.spec.liveMigrationConfig.bandwidthPerMigration`: + +```json +{ + "spec": { + "liveMigrationConfig": { + "bandwidthPerMigration": "32Mi" + } + } +} +``` + +**3. Use dedicated migration network** (see Dedicated Migration Network section) + +### Issue 3: virt-handler Pods Not Ready + +**Symptom:** + +Using `pods_list` with `labelSelector: "kubevirt.io=virt-handler"`, some pods show `status.containerStatuses[0].ready == false`. + +**Causes:** +- Recent HyperConverged configuration change +- Network configuration error +- Node connectivity issue + +**Solutions:** + +**1. Wait for pod restart** (after config change): + +Pods restart automatically after HyperConverged update. Monitor using `pods_list` until all show ready status. + +**2. Check pod logs:** + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "name": "", + "namespace": "openshift-cnv", + "tail": 100 +} +``` + +**3. Verify node network configuration** (if using dedicated network): + +Use `resources_list` to check NodeNetworkConfigurationPolicy: + +**Parameters**: +```json +{ + "apiVersion": "nmstate.io/v1", + "kind": "NodeNetworkConfigurationPolicy" +} +``` + +And NetworkAttachmentDefinition: + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "openshift-cnv" +} +``` + +### Issue 4: Migration Rejected - Cluster Limit Reached + +**Symptom:** +``` +Migration rejected: cluster migration limit reached (5 concurrent) +``` + +**Solutions:** + +**1. Wait for ongoing migrations** to complete: + +Monitor using `resources_list`: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration" +} +``` + +Filter for migrations where `.status.phase` is NOT "Succeeded" or "Failed". + +**2. Increase cluster limit** (if network allows): + +Use `resources_get` to fetch HyperConverged CR, modify `.spec.liveMigrationConfig.parallelMigrationsPerCluster`, then update with `resources_create_or_update`: + +```json +{ + "spec": { + "liveMigrationConfig": { + "parallelMigrationsPerCluster": 10 + } + } +} +``` + +**3. Migrate VMs sequentially** instead of batch operation + +--- + +## Related Documentation + +- [Performance Tuning Guide](./performance-tuning.md) - Advanced tuning for migration performance +- [Anti-Patterns](./anti-patterns.md) - Common mistakes to avoid +- [Production Considerations](./production-considerations.md) - Right-sizing, workload planning, HA strategies +- [Troubleshooting: Scheduling Errors](../../../docs/troubleshooting/scheduling-errors.md) - ErrorUnschedulable after cold migration + +--- + +**Last Updated**: 2026-02-24 +**OpenShift Virtualization Versions**: 4.17, 4.18, 4.19, 4.20 +**Status**: Production-ready guidance from official Red Hat sources diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/performance-tuning.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/performance-tuning.md new file mode 100644 index 00000000..f1af24e9 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/performance-tuning.md @@ -0,0 +1,719 @@ +# Performance Tuning for VM Rebalancing + +**Purpose**: Advanced performance tuning parameters, optimization strategies, and monitoring guidance for VM live migration and rebalancing operations. + +**When to consult this document**: When migrations are slow, when planning large-scale rebalancing, or when optimizing cluster performance for frequent migrations. + +--- + +## Official Sources + +This document is compiled from official Red Hat documentation: + +- [Live Migrating VMs with OpenShift Virtualization](https://developers.redhat.com/articles/2025/07/14/live-migrating-vms-openshift-virtualization) - Red Hat Developer (2025-07-14) +- [Best Practices for Virtual Machine Deployments on OpenShift Virtualization](https://learn.microsoft.com/en-us/azure/openshift/best-practices-openshift-virtualization) - Microsoft Azure Red Hat OpenShift (2026-02-16) +- [Announcing Right-Sizing for OpenShift Virtualization](https://developers.redhat.com/articles/2025/04/28/announcing-right-sizing-openshift-virtualization) - Red Hat Developer (2025-04-28) +- [Best Practices to Deploy VMs in Red Hat OpenShift Virtualization](https://docs.netapp.com/us-en/netapp-solutions-virtualization/openshift/os-osv-bpg.html) - NetApp Solutions + +--- + +## Right-Sizing Virtual Machines + +### Why Right-Sizing Matters for Rebalancing + +Properly sized VMs: +- Migrate faster (smaller memory footprint) +- Reduce network bandwidth requirements +- Improve cluster resource utilization +- Enable more efficient load balancing +- Prevent resource contention during migrations + +### Right-Sizing Methodology + +**Step 1: Define Health Metrics** + +Target healthy resource utilization ranges: + +| Resource | Target Range | Warning Threshold | Critical Threshold | +|----------|--------------|-------------------|-------------------| +| CPU Utilization | 60-70% average | >80% | >90% | +| Memory Pressure | <80% | >85% | >95% | +| Disk I/O Latency | <10ms | >50ms | >100ms | +| Network Throughput | <70% capacity | >80% | >90% | + +**Step 2: Monitor VM Resource Usage** + +**Using MCP Tools:** + +**MCP Tool**: `pods_top` (from openshift-virtualization) + +**Parameters**: +```json +{ + "all_namespaces": true, + "label_selector": "kubevirt.io=virt-launcher" +} +``` + +This returns CPU and memory usage for all VM launcher pods. Filter by specific namespace or VM name as needed. + +**For detailed metrics:** + +**MCP Tool**: `nodes_stats_summary` (from openshift-virtualization) + +**Parameters**: +```json +{ + "name": "" +} +``` + +Review `.pods[].containers[]` metrics for specific VM resource consumption including: +- `cpu.usageNanoCores` - Current CPU usage +- `memory.workingSetBytes` - Active memory usage +- `rootfs.usedBytes` - Disk usage + +**Step 3: Analyze Historical Data** + +Collect metrics over time (minimum 7 days for meaningful patterns): + +- Peak usage periods +- Resource saturation events +- Correlation between workload and resource consumption +- Trending (growing vs stable resource needs) + +**Step 4: Adjust VM Specifications** + +Based on observed metrics, resize VMs using `resources_get` and `resources_create_or_update`: + +**Example: Resize VM Memory** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Modify `.spec.template.spec.domain.resources.requests.memory` based on usage analysis, then update with `resources_create_or_update`. + +**Avoid Overprovisioning:** +- Don't rely on on-premises sizing references +- Benchmark your actual workloads +- Consider OpenShift Virtualization architectural overhead (see Architectural Overhead section) + +--- + +## Architectural Overhead + +### OpenShift Virtualization Performance Characteristics + +Running VMs in OpenShift Virtualization introduces architectural overhead compared to bare metal or native pods: + +**Observed Performance (Azure Red Hat OpenShift with Standard_D96ds_v5 nodes, OpenShift 4.20, Virtualization 4.20):** + +| Workload Type | VM Performance | Pod Performance | Overhead | +|---------------|----------------|-----------------|----------| +| **Compute** (events/sec) | 525,022 | 546,997 | ~4% slower | +| **Compute** (latency ms) | 0.70 | 0.65 | ~8% higher latency | +| **Storage** (1 thread TPM) | 4,332 | 6,303 | ~31% slower | +| **Storage** (32 threads TPM) | 64,294 | 103,359 | ~38% slower | +| **Network** (64B, 1 thread Gbps) | 0.4 | 0.9 | ~56% slower | +| **Network** (1024B, 8 threads Gbps) | 24.7 | 28.9 | ~15% slower | + +**Key Takeaways:** +- Compute overhead is minimal (~4-8%) +- Storage and network have higher overhead (15-56% depending on workload) +- Multi-threaded workloads show better relative performance + +**Implications for Rebalancing:** +- VMs require more time to migrate than equivalent containerized workloads +- Plan capacity with overhead in mind (don't fill nodes to 100%) +- Network-intensive VMs benefit most from dedicated migration networks +- Consider workload characteristics when planning concurrent migrations + +--- + +## Tuned Configuration for High-Performance VMs + +### SAP HANA Tuning Example + +For database and high-performance workloads, apply tuned profiles to guest OS: + +**Tuned Profile (RHEL Guest):** + +```ini +[main] +summary=Optimize for SAP HANA and high-performance VMs + +[cpu] +force_latency=cstate.id:3|70 +governor=performance +energy_perf_bias=performance +min_perf_pct=100 + +[vm] +transparent_hugepages=never + +[sysctl] +# Semaphore limits +kernel.sem = 32000 1024000000 500 32000 + +# Disable NUMA balancing for predictable performance +kernel.numa_balancing = 0 + +# Scheduler tuning for low latency +kernel.sched_min_granularity_ns = 3000000 +kernel.sched_wakeup_granularity_ns = 4000000 + +# Memory management +vm.dirty_ratio = 40 +vm.dirty_background_ratio = 10 +vm.swappiness = 10 +``` + +**When to Apply:** +- Database VMs (PostgreSQL, MySQL, Oracle, SAP HANA) +- Real-time analytics workloads +- Low-latency trading platforms +- High-performance computing (HPC) VMs + +**Impact on Migration:** +- Reduces dirty page rate (faster convergence) +- More predictable migration times +- Better performance during and after migration + +--- + +## CPU and Memory Overcommit + +### Understanding Overcommit Ratios + +OpenShift Virtualization allows overcommit of CPU and memory resources, enabling higher VM density per node. + +**Default Overcommit Ratios:** +- CPU: No overcommit (1:1 mapping) +- Memory: No overcommit (1:1 mapping) + +**Recommended Production Limits (Red Hat Guidance):** +- **CPU Overcommit**: Maximum 1.8x physical cores +- **Memory Overcommit**: Maximum 0.9x physical memory + +**Consequences of Exceeding Limits:** +- **CPU**: Throttling causes slowness across all workloads on affected node +- **Memory**: OOM (Out of Memory) kills, VM crashes, data loss + +### Configuring Overcommit + +**Update HyperConverged CR using MCP Tools:** + +**Step 1: Get HyperConverged resource** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "hco.kubevirt.io/v1beta1", + "kind": "HyperConverged", + "namespace": "openshift-cnv", + "name": "kubevirt-hyperconverged" +} +``` + +**Step 2: Modify overcommit configuration** + +Add to `.spec.resourceRequirements`: + +```yaml +spec: + resourceRequirements: + vmiCPUAllocationRatio: 1.5 # Allow 1.5x CPU overcommit + vmiMemoryOvercommitPercent: 20 # Allow 20% memory overcommit +``` + +**Step 3: Update using `resources_create_or_update`** + +**Best Practices:** +- Use conservative overcommit for production (1.2x CPU max, 10% memory max) +- Use higher overcommit for dev/test (1.8x CPU, 20% memory acceptable) +- Monitor node resource usage closely after enabling overcommit +- Adjust based on actual VM behavior patterns + +**Impact on Rebalancing:** +- Higher overcommit = more VMs per node = longer migration times +- Rebalancing may be needed more frequently with overcommit +- Target node capacity calculations must account for overcommit ratios + +--- + +## Network Performance Tuning + +### MTU Configuration + +**Why MTU Matters:** +- Default MTU (1500 bytes) causes fragmentation for large data transfers +- Jumbo frames (MTU 9000) significantly improve network efficiency +- Critical for large VM migrations (>100GB memory) + +**Set MTU in NetworkAttachmentDefinition:** + +When creating dedicated migration network, include MTU setting: + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters** (excerpt): +```json +{ + "resource": "apiVersion: k8s.cni.cncf.io/v1\nkind: NetworkAttachmentDefinition\nmetadata:\n name: migration-network\n namespace: openshift-cnv\nspec:\n config: '{\n \"cniVersion\": \"0.3.1\",\n \"name\": \"migration-bridge\",\n \"type\": \"macvlan\",\n \"master\": \"eth1\",\n \"mode\": \"bridge\",\n \"mtu\": 9000,\n \"ipam\": {...}\n }'" +} +``` + +**Validate MTU:** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "openshift-cnv", + "name": "migration-network" +} +``` + +Check `.spec.config` for `"mtu": 9000`. + +### NAPI and Multiqueue Tuning + +For network-intensive workloads, enable multiqueue virtio-net: + +**VM Configuration:** +```yaml +spec: + template: + spec: + domain: + devices: + interfaces: + - name: default + model: virtio + masquerade: {} + ports: + - port: 80 + networkInterfaceMultiqueue: true # Enable multiqueue +``` + +**Benefits:** +- Parallelizes network processing across multiple vCPUs +- Improves throughput for high-bandwidth workloads +- Reduces latency for network-intensive applications + +**When to Use:** +- VMs with >4 vCPUs +- High network throughput requirements (>10Gbps) +- Web servers, load balancers, network appliances + +--- + +## Storage Performance Optimization + +### Storage Class Selection + +Different storage backends have different performance characteristics: + +| Storage Backend | IOPS | Latency | Throughput | Best For | +|-----------------|------|---------|------------|----------| +| OpenShift Data Foundation (ODF) | High | Low (<5ms) | Very High | General purpose, production | +| Azure NetApp Files (ANF) Premium | Very High | Very Low (<1ms) | Very High | Database, high-performance | +| NFS-CSI (SSD-backed) | Medium | Medium (5-10ms) | High | Dev/test, general use | +| AWS EBS gp3 | Medium | Medium (10-20ms) | Medium | Cost-effective, RWO only | + +**Check Storage Class Performance:** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "name": "" +} +``` + +Review `.parameters` for performance tier, provisioning type, and backend configuration. + +### Storage Limits (NetApp ONTAP) + +When using NetApp storage backends, configure limits to prevent resource exhaustion: + +**SVM Volume Limits:** + +Set maximum volumes per SVM to prevent Trident from consuming all storage capacity. + +**Storage Quotas:** + +Implement storage limits on SVMs to enforce resource boundaries. + +**Trident Backend Parameters:** + +Configure in Trident backend definition: +- `limitVolumeSize`: Maximum individual volume size (e.g., "100Gi") +- `limitVolumePoolSize`: Maximum FlexVol size for economy drivers (e.g., "500Gi") + +**Impact on Rebalancing:** +- Storage limits prevent VMs from growing unbounded +- Predictable storage capacity aids in target node selection +- Quota enforcement ensures fair resource distribution + +--- + +## Migration Bandwidth Management + +### Bandwidth Per Migration + +**Purpose**: Limit bandwidth consumption per migration to prevent network saturation. + +**Default**: Unlimited (no bandwidth limit) + +**When to Set:** +- Shared application network (no dedicated migration network) +- Multiple concurrent migrations planned +- Network capacity constraints + +**Configuration Using MCP Tools:** + +**Step 1: Get HyperConverged resource** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "hco.kubevirt.io/v1beta1", + "kind": "HyperConverged", + "namespace": "openshift-cnv", + "name": "kubevirt-hyperconverged" +} +``` + +**Step 2: Set bandwidth limit** + +Modify `.spec.liveMigrationConfig.bandwidthPerMigration`: + +```yaml +spec: + liveMigrationConfig: + bandwidthPerMigration: 64Mi # 64 MiB/s per migration +``` + +Common values: +- `32Mi` - Conservative (256 Mbps) +- `64Mi` - Default (512 Mbps) +- `128Mi` - High bandwidth (1 Gbps) +- Omit field for unlimited + +**Step 3: Update using `resources_create_or_update`** + +**Monitoring Bandwidth Usage:** + +**MCP Tool**: `nodes_stats_summary` (from openshift-virtualization) + +**Parameters**: +```json +{ + "name": "" +} +``` + +Review `.network.interfaces[].rxBytes` and `.network.interfaces[].txBytes` for current throughput. + +**Tuning Guidance:** +- Start conservative (32-64Mi) and increase if migrations are slow +- Monitor network utilization during migrations +- Unlimited bandwidth is acceptable with dedicated migration network + +--- + +## Concurrency Limits Tuning + +### Parallel Migrations Per Cluster + +**Default**: 5 concurrent migrations cluster-wide + +**When to Increase:** +- Dedicated migration network with high bandwidth (100Gbps) +- Routine maintenance windows requiring many migrations +- Cluster has >20 nodes + +**When to Decrease:** +- Network saturation observed +- Migration failures due to timeouts +- Shared application network + +**Configuration:** + +Modify `.spec.liveMigrationConfig.parallelMigrationsPerCluster` in HyperConverged CR: + +```yaml +spec: + liveMigrationConfig: + parallelMigrationsPerCluster: 10 # Increase from default 5 +``` + +**Conservative**: 3-5 migrations +**Moderate**: 5-10 migrations +**Aggressive**: 10-20 migrations (requires dedicated network) + +### Parallel Outbound Migrations Per Node + +**Default**: 2 concurrent outbound migrations per source node + +**Recommendation**: Keep at 2 to prevent single-node overload. + +**Why 2 is Optimal:** +- Prevents source node CPU/memory saturation +- Limits network bandwidth consumption per node +- Avoids cascading performance degradation +- Tested and validated by Red Hat + +**Only increase to 3-4 if:** +- Node has very high CPU/memory headroom (>50% free) +- Dedicated high-bandwidth migration network +- Extensive testing validates stability + +--- + +## Timeout Configuration + +### Completion Timeout Per GiB + +**Default**: 800 seconds per GiB of VM memory + +**Calculation**: For a 16GB VM, timeout = 16 * 800 = 12,800 seconds (~3.5 hours) + +**When to Increase:** +- High dirty page rate workloads (databases, caching systems) +- VMs with >100GB memory +- Network bandwidth constraints + +**When to Decrease:** +- Fast dedicated migration network (100Gbps) +- Low dirty page rate (mostly idle VMs) +- Want faster failure detection + +**Configuration:** + +Modify `.spec.liveMigrationConfig.completionTimeoutPerGiB` in HyperConverged CR: + +```yaml +spec: + liveMigrationConfig: + completionTimeoutPerGiB: 1200 # Increase for large/busy VMs +``` + +**Tuning by Workload Type:** + +| Workload Type | Recommended Timeout | Rationale | +|---------------|---------------------|-----------| +| Database (write-heavy) | 1200-1600s | High dirty page rate | +| Web server (mostly read) | 600-800s | Low dirty page rate | +| Caching (Redis/Memcached) | 1600-2000s | Very high dirty page rate | +| General purpose | 800s (default) | Balanced | + +### Progress Timeout + +**Default**: 150 seconds without progress before cancellation + +**Purpose**: Detects stuck migrations and fails fast rather than hanging indefinitely. + +**When to Increase:** +- Very large VMs (>500GB memory) +- Slow networks (<1Gbps) +- Initial memory copy takes >2 minutes + +**When to Decrease:** +- Want faster failure detection +- Prefer to retry quickly rather than wait + +**Configuration:** + +Modify `.spec.liveMigrationConfig.progressTimeout` in HyperConverged CR: + +```yaml +spec: + liveMigrationConfig: + progressTimeout: 300 # 5 minutes without progress +``` + +**Recommended Values:** +- Small VMs (<50GB): 150s (default) +- Medium VMs (50-200GB): 200-300s +- Large VMs (>200GB): 300-600s + +--- + +## Monitoring and Observability + +### Key Metrics to Monitor + +**During Rebalancing Operations:** + +1. **Migration Progress** + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration" +} +``` + +Monitor `.status.phase` for each migration (Pending → Scheduling → PreparingTarget → Running → Succeeded). + +2. **Node Resource Usage** + +**MCP Tool**: `nodes_top` (from openshift-virtualization) + +**Parameters**: +```json +{ + "name": "" +} +``` + +Track CPU and memory utilization before, during, and after migrations. + +3. **Network Throughput** + +**MCP Tool**: `nodes_stats_summary` (from openshift-virtualization) + +**Parameters**: +```json +{ + "name": "" +} +``` + +Review `.network.interfaces[]` metrics for bandwidth usage. + +4. **VM Health** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.status.conditions[]` for VM health status. + +### Performance Benchmarking + +**Before Rebalancing:** +- Establish baseline performance metrics +- Document current resource utilization +- Identify performance-sensitive VMs + +**During Rebalancing:** +- Monitor migration duration +- Track network bandwidth consumption +- Watch for resource contention + +**After Rebalancing:** +- Validate improved load distribution +- Confirm no performance degradation +- Document improvements achieved + +**Tools for Benchmarking:** +- Apache JMeter (web application load testing) +- stress-ng (CPU/memory stress testing) +- fio (storage I/O benchmarking) +- iperf3 (network throughput testing) + +--- + +## Scaling Strategies + +### Scale Out vs Scale Up + +**Scale Out** (add more nodes): +- **Pros**: Better fault tolerance, more migration targets, horizontal capacity growth +- **Cons**: Higher complexity, more licensing costs, requires cluster expansion + +**Scale Up** (larger node sizes): +- **Pros**: Simpler management, fewer migration hops, better resource consolidation +- **Cons**: Larger blast radius, limited by maximum instance size, single point of failure risk + +**For Demanding Workloads:** + +From Microsoft Azure Red Hat OpenShift guidance: +> "Scale out or up for demanding workloads: Add more nodes or upsize the nodes in your Azure Red Hat OpenShift cluster for high concurrency or resource-intensive applications." + +**Recommendation:** +- Start with scale-up to minimum 8-core Azure VMs (per OpenShift Virtualization requirements) +- Scale-out when individual nodes exceed 70-80% sustained utilization +- Balance between node size and cluster size for optimal resilience + +### Node Pool Strategy + +**Workload-Specific Node Pools:** + +Create dedicated node pools for different VM workload types using labels, taints, and tolerations: + +**Example Node Pool Configuration:** + +**Pool 1: General VMs** +- Node labels: `workload-type=general` +- Node taints: None +- VM tolerations: Not required + +**Pool 2: High-Performance VMs** +- Node labels: `workload-type=high-performance` +- Node taints: `performance=dedicated:NoSchedule` +- VM tolerations: Match taint + +**Pool 3: GPU Workloads** +- Node labels: `workload-type=gpu` +- Node taints: `nvidia.com/gpu=present:NoSchedule` +- VM tolerations: Match taint + +**Apply Labels to Nodes Using MCP Tools:** + +**MCP Tool**: `resources_get` then `resources_create_or_update` (from openshift-virtualization) + +Get node, modify `.metadata.labels`, then update. + +**Benefits:** +- Simplifies maintenance (drain entire pool) +- Limits blast radius (failures contained to pool) +- Improves resource efficiency (right-sized pools) +- Enables topology spread rules (VMs across zones/pools) + +--- + +## Related Documentation + +- [Live Migration Best Practices](./live-migration-best-practices.md) - Configuration parameters and requirements +- [Anti-Patterns](./anti-patterns.md) - Common mistakes to avoid +- [Production Considerations](./production-considerations.md) - Right-sizing, workload planning, HA strategies + +--- + +**Last Updated**: 2026-02-24 +**OpenShift Virtualization Versions**: 4.17, 4.18, 4.19, 4.20 +**Status**: Production-ready guidance from official Red Hat sources diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/production-considerations.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/production-considerations.md new file mode 100644 index 00000000..919280a8 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/production-considerations.md @@ -0,0 +1,868 @@ +# Production Considerations for VM Rebalancing + +**Purpose**: Production deployment guidance, workload planning, high availability strategies, and operational best practices for VM rebalancing in OpenShift Virtualization. + +**When to consult this document**: Before deploying to production, when planning capacity, or when designing HA/DR strategies. + +--- + +## Official Sources + +This document is compiled from official Red Hat documentation: + +- [Best Practices for Virtual Machine Deployments on OpenShift Virtualization](https://learn.microsoft.com/en-us/azure/openshift/best-practices-openshift-virtualization) - Microsoft Azure Red Hat OpenShift (2026-02-16) +- [Announcing Right-Sizing for OpenShift Virtualization](https://developers.redhat.com/articles/2025/04/28/announcing-right-sizing-openshift-virtualization) - Red Hat Developer (2025-04-28) +- [Best Practices to Deploy VMs in Red Hat OpenShift Virtualization](https://docs.netapp.com/us-en/netapp-solutions-virtualization/openshift/os-osv-bpg.html) - NetApp Solutions +- [OpenShift Virtualization Best Practices](https://trilio.io/openshift-virtualization/) - Trilio + +--- + +## Workload Identification and Categorization + +### Common Workload Types + +Before provisioning VMs, categorize workloads to determine performance and resource requirements: + +| Workload Type | Characteristics | Resource Profile | Migration Considerations | +|---------------|----------------|------------------|--------------------------| +| **General Purpose** | Web servers, app servers, CMS | Moderate CPU/memory | Easy to migrate, low dirty page rate | +| **Database** | RDBMS, NoSQL | High CPU, memory, consistent IOPS | High dirty page rate; schedule migrations carefully | +| **Real-time Analytics** | Operational dashboards | Low latency, high throughput | Sensitive to migration pause; use dedicated network | +| **AI/ML** | Training, inference | Very high CPU/GPU, memory | Large memory footprint; long migration times | +| **Data Streaming** | Event-driven architectures | High throughput, low latency | Network-intensive; avoid concurrent migrations | +| **Batch Processing** | Periodic jobs | Variable resources | Migrate during job idle periods | +| **HPC** | Scientific simulations | Very high CPU, memory | Extremely long migrations; consider cold migration | +| **Edge/IoT** | Sensor aggregation | Low resources | Easy to migrate, scale horizontally | +| **Media Processing** | Encoding, streaming | High CPU, network | High dirty page rate during processing | +| **Dev/Test** | Development environments | Variable | Higher overcommit acceptable | + +### Workload Assessment Using MCP Tools + +**Step 1: Inventory Current VMs** + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine" +} +``` + +**Step 2: Analyze Resource Usage** + +**MCP Tool**: `pods_top` (from openshift-virtualization) + +**Parameters**: +```json +{ + "all_namespaces": true, + "label_selector": "kubevirt.io=virt-launcher" +} +``` + +**Step 3: Categorize by Usage Pattern** + +Group VMs by observed characteristics: +- CPU-intensive: >70% CPU utilization +- Memory-intensive: >80% memory utilization +- I/O-intensive: High storage throughput +- Network-intensive: High network bandwidth + +**Step 4: Plan Rebalancing Strategy** + +Based on workload type: +- **CPU-intensive**: Balance CPU across nodes +- **Memory-intensive**: Balance memory across nodes +- **I/O-intensive**: Distribute across different storage backends +- **Network-intensive**: Stagger migrations to avoid saturation + +--- + +## Right-Sizing Virtual Machines for Production + +### Health Metrics Definition + +Establish target ranges for healthy resource utilization: + +**CPU Utilization:** +- **Target**: 60-70% average usage +- **Warning**: >80% sustained +- **Critical**: >90% sustained +- **Action**: Scale up VM or rebalance to less loaded node + +**Memory Pressure:** +- **Target**: 70-80% utilization +- **Warning**: >85% with swap activity +- **Critical**: >95% or OOM events +- **Action**: Increase VM memory or reduce workload + +**Disk I/O:** +- **Target**: <10ms latency, <70% queue depth +- **Warning**: >50ms latency +- **Critical**: >100ms latency or queue saturation +- **Action**: Move to faster storage tier or distribute workload + +**Network Throughput:** +- **Target**: <70% interface capacity +- **Warning**: >80% sustained +- **Critical**: >90% or packet loss +- **Action**: Enable multiqueue, use faster NICs, rebalance + +### Monitoring Setup Using MCP Tools + +**VM-Level Metrics:** + +**MCP Tool**: `pods_top` (from openshift-virtualization) + +Provides current CPU and memory usage for VM launcher pods. + +**Node-Level Metrics:** + +**MCP Tool**: `nodes_top` (from openshift-virtualization) + +Shows aggregate node resource consumption. + +**Detailed Statistics:** + +**MCP Tool**: `nodes_stats_summary` (from openshift-virtualization) + +**Parameters**: +```json +{ + "name": "" +} +``` + +Provides comprehensive metrics including: +- Per-pod resource usage +- Container-level metrics +- Filesystem usage +- Network interface statistics +- PSI (Pressure Stall Information) metrics on cgroup v2 systems + +### Sizing Recommendations by Workload + +**Database Workloads:** +- Start with: 4-8 vCPU, 16-32Gi memory +- Storage: Premium SSD or NVMe with RWX support +- Network: Enable multiqueue virtio-net +- Special: Consider dedicated CPU placement (`dedicatedCpuPlacement: true`) + +**Web Servers:** +- Start with: 2-4 vCPU, 4-8Gi memory +- Storage: Standard SSD acceptable +- Network: Standard configuration sufficient +- Special: Scale horizontally rather than vertically + +**AI/ML Workloads:** +- Start with: 8-16 vCPU, 32-64Gi memory +- Storage: High-performance SSD +- Network: High bandwidth (consider dedicated migration network) +- Special: GPU support currently not available (plan accordingly) + +**Dev/Test Environments:** +- Start with: 1-2 vCPU, 2-4Gi memory +- Storage: Standard tier acceptable +- Network: Standard configuration +- Special: Higher overcommit ratios acceptable (1.5-1.8x CPU) + +### Minimum Requirements + +From Microsoft Azure Red Hat OpenShift documentation: +> "Minimum core requirement: OpenShift Virtualization requires a minimum of eight (8) core Azure VMs for OpenShift worker nodes." + +**Implications for Rebalancing:** +- Worker nodes must have ≥8 cores +- Plan VM placement considering this minimum +- Avoid creating nodes smaller than this threshold + +--- + +## High Availability Strategies + +### VM-Level High Availability + +**RunStrategy Configuration:** + +```yaml +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + name: critical-app +spec: + runStrategy: Always # Ensures VM restarts after failures +``` + +**RunStrategy Options:** + +| Strategy | Behavior | Use Case | +|----------|----------|----------| +| `Always` | VM runs continuously; restarts on failure | Production VMs requiring HA | +| `RerunOnFailure` | Restarts only if VM crashes | Batch workloads | +| `Manual` | User controls start/stop | Dev/test VMs | +| `Halted` | VM stays stopped | Maintenance, cold storage | + +**Eviction Strategy:** + +OpenShift Virtualization automatically sets `evictionStrategy` to `LiveMigrate` for VMs with RWX storage: + +```yaml +spec: + template: + spec: + evictionStrategy: LiveMigrate # Automatically set for RWX VMs +``` + +**Note for Single Node OpenShift (SNO):** + +From known issues: +> "In a Single Node OpenShift (SNO) cluster, a VMCannotBeEvicted alert occurs on virtual machines created from common templates that have the eviction strategy set to LiveMigrate." + +**Workaround**: Use `evictionStrategy: None` for SNO clusters. + +### Pod Anti-Affinity for VM Replicas + +For critical applications, deploy multiple VM replicas with anti-affinity: + +```yaml +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + name: web-server-replica-1 + labels: + app: web-server +spec: + template: + metadata: + labels: + app: web-server + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - web-server + topologyKey: kubernetes.io/hostname # Different nodes +``` + +**Benefits:** +- VMs distributed across different failure domains +- Node failure affects only one replica +- Improves overall availability + +**Verify Distribution Using MCP Tools:** + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "labelSelector": "app=web-server" +} +``` + +Check `.status.nodeName` for each instance to confirm distribution. + +### Machine Health Checks + +**Critical for Automatic Failover:** + +From Red Hat documentation: +> "If a node fails and machine health checks are not deployed on your cluster, virtual machines (VMs) with RunStrategy: Always configured are not automatically relocated to healthy nodes." + +**Deploy Machine Health Checks:** + +Configure at cluster level to detect and remediate node failures. This enables automatic VM recovery without manual intervention. + +**Monitor Node Health:** + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +Filter for nodes where `.status.conditions[]` shows unhealthy states (`Ready=False`, `DiskPressure=True`, `MemoryPressure=True`). + +### Topology Spread for Zone Resilience + +For multi-zone clusters, use topology spread rules: + +```yaml +spec: + template: + spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: critical-app +``` + +**Benefits:** +- VMs spread across availability zones +- Zone failure doesn't affect all replicas +- Improved disaster recovery + +--- + +## Capacity Planning + +### Cluster Sizing Approach + +**Scale Out vs Scale Up:** + +From Microsoft Azure Red Hat OpenShift guidance: +> "Scale out or up for demanding workloads: Add more nodes or upsize the nodes in your Azure Red Hat OpenShift cluster for high concurrency or resource-intensive applications." + +**Scale Out (add more nodes):** +- **Pros**: Better fault tolerance, horizontal growth, more migration targets +- **Cons**: Higher complexity, more license costs, requires cluster expansion + +**When to Scale Out:** +- Current nodes consistently >70-80% utilized +- Need more fault isolation +- Planning for growth +- HA requirements mandate distribution + +**Scale Up (larger node sizes):** +- **Pros**: Simpler management, better resource consolidation, fewer migration hops +- **Cons**: Larger blast radius, limited by max instance size, single point of failure risk + +**When to Scale Up:** +- VMs don't fit on existing nodes +- Few large VMs rather than many small VMs +- Simplicity valued over distribution + +**Recommended Approach:** +1. Start with moderate node sizes (8-16 cores) +2. Scale out to 3-5 nodes minimum for HA +3. Scale up only when specific VMs require larger nodes +4. Maintain headroom (30-40% free capacity) for migrations and failures + +### Node Pool Strategy + +**Create workload-specific pools using labels and taints:** + +**Pool Configuration Example:** + +**General VM Pool:** +- Node size: 8-16 cores, 32-64GB RAM +- Labels: `workload-type=general` +- No taints (default scheduling) + +**High-Performance Pool:** +- Node size: 16-32 cores, 64-128GB RAM +- Labels: `workload-type=high-performance`, `cpumanager=true` +- Taints: `performance=dedicated:NoSchedule` + +**GPU Pool (future):** +- Node size: GPU-enabled instances +- Labels: `workload-type=gpu` +- Taints: `nvidia.com/gpu=present:NoSchedule` + +**Configure Labels Using MCP Tools:** + +**MCP Tool**: `resources_get` then `resources_create_or_update` (from openshift-virtualization) + +**Step 1: Get Node** + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node", + "name": "" +} +``` + +**Step 2: Add Labels** + +Modify `.metadata.labels`: +```json +{ + "workload-type": "high-performance", + "cpumanager": "true" +} +``` + +**Step 3: Update Node** + +Use `resources_create_or_update` with modified resource. + +**Benefits of Node Pools:** +- Simplifies maintenance (drain entire pool) +- Limits blast radius (failures contained) +- Improves efficiency (right-sized for workload) +- Enables topology spread rules + +### Capacity Headroom + +**Reserve capacity for:** +- Node failures (n-1 redundancy minimum) +- VM migrations (target nodes need free resources) +- Burst workloads (temporary spikes) +- New VM deployments + +**Recommended Headroom:** +- **Production**: 30-40% free capacity cluster-wide +- **Dev/Test**: 20-30% free capacity +- **Minimum**: 20% free capacity (below this, rebalancing becomes difficult) + +**Monitor Capacity Using MCP Tools:** + +**MCP Tool**: `nodes_top` (from openshift-virtualization) + +Calculate cluster-wide utilization: +``` +Total CPU Used / Total CPU Capacity = Cluster CPU Utilization +Total Memory Used / Total Memory Capacity = Cluster Memory Utilization +``` + +**Action Thresholds:** +- <70%: Healthy headroom +- 70-80%: Plan for expansion +- >80%: Add nodes urgently +- >90%: Emergency capacity issue + +--- + +## Storage Planning for Production + +### Storage Backend Selection + +**OpenShift Data Foundation (ODF):** +- **Best for**: General purpose, production workloads +- **Performance**: High IOPS, low latency (<5ms) +- **RWX Support**: Yes +- **Considerations**: Requires dedicated storage nodes; use taints/tolerations to isolate ODF workload + +**Azure NetApp Files (ANF):** +- **Best for**: High-performance databases, latency-sensitive apps +- **Performance**: Very high IOPS, very low latency (<1ms) +- **RWX Support**: Yes +- **Considerations**: Choose performance tier based on workload requirements + +**NFS-CSI (SSD-backed):** +- **Best for**: Dev/test, general use +- **Performance**: Medium IOPS, medium latency (5-10ms) +- **RWX Support**: Yes +- **Considerations**: Cost-effective, sufficient for non-critical workloads + +**AWS EBS gp3:** +- **Best for**: Cost-effective storage +- **Performance**: Medium IOPS, medium latency (10-20ms) +- **RWX Support**: No (RWO only) +- **Considerations**: Cannot use live migration; cold migration only + +### Storage QoS and Limits + +**NetApp ONTAP QoS:** + +From NetApp documentation: +> "Apply QoS policies to SVMs to limit the number of IOPS consumable by the Trident provisioned volumes." + +**Why QoS Matters:** +- Prevents one VM from starving others +- Protects non-Trident workloads from VM I/O impact +- Enforces fair resource distribution +- Predictable performance for all VMs + +**SVM Isolation:** + +From NetApp documentation: +> "Establish dedicated Storage Virtual Machines (SVMs) to provide isolation and administrative separation between tenants." + +**Benefits:** +- Tenant isolation +- Privilege delegation +- Resource quota enforcement +- Security boundary + +### Storage Validation Before Rebalancing + +**For Live Migration, verify RWX storage:** + +For each VM: + +**Step 1: Get VM** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Step 2: Extract PVC Names** + +From `.spec.template.spec.volumes[].persistentVolumeClaim.claimName`. + +**Step 3: Check PVC Access Mode** + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +**Required**: `.spec.accessModes` must include `"ReadWriteMany"`. + +--- + +## Network Planning + +### Dedicated Migration Network + +**Production Requirement:** + +For production clusters with large VMs or frequent migrations, a dedicated migration network is **highly recommended**. + +**Benefits:** +- Isolates migration traffic from applications +- Enables 100Gbps bandwidth for large VM migrations +- Prevents network contention +- Improves security (separate VLAN) + +**Implementation:** + +See [live-migration-best-practices.md](./live-migration-best-practices.md) for complete configuration steps. + +**Key Components:** +1. Secondary physical NIC or VLAN +2. NodeNetworkConfigurationPolicy (NNCP) +3. NetworkAttachmentDefinition (NAD) +4. HyperConverged CR configuration + +### Network Performance Tuning + +**MTU Configuration:** + +Set to 9000 (jumbo frames) for migration networks: + +```json +{ + "cniVersion": "0.3.1", + "name": "migration-bridge", + "type": "macvlan", + "mtu": 9000, + "ipam": {...} +} +``` + +**Multiqueue virtio-net:** + +Enable for VMs with >4 vCPUs and high network throughput: + +```yaml +spec: + template: + spec: + domain: + devices: + interfaces: + - name: default + model: virtio + networkInterfaceMultiqueue: true +``` + +### Network Isolation + +**Namespace Separation:** + +From best practices: +> "Use Namespaces to provide logical boundary for resources." + +**Pod Security Policies:** + +Disable privileged container capabilities for VM launcher pods to enhance security. + +**Separate Export Policies:** + +For NFS storage, implement separate export policies for infrastructure nodes vs application nodes. + +--- + +## Operational Best Practices + +### Phased Production Rollout + +From best practices: +> "Begin with non-critical or dev/test workloads before moving production systems - this phased approach allows teams to gain hands-on experience while minimizing risk." + +**Recommended Phases:** + +**Phase 1: Development/Test (2-4 weeks)** +- Deploy 5-10 test VMs +- Validate performance vs expectations +- Test live and cold migration workflows +- Benchmark resource overhead +- Train operations team + +**Phase 2: Non-Critical Production (4-8 weeks)** +- Migrate non-critical workloads (internal tools, QA environments) +- Monitor stability and performance +- Refine sizing and configurations +- Build runbooks and procedures +- Establish monitoring and alerting + +**Phase 3: Critical Production (Ongoing)** +- Migrate critical workloads in prioritized order +- Ensure HA and DR fully configured +- 24/7 monitoring and on-call support +- Document rollback procedures +- Conduct regular DR tests + +### Start Small, Scale Gradually + +From best practices: +> "When starting with OpenShift Virtualization, it's essential to start small and scale up as needed to avoid over-provisioning and wasting resources." + +**Growth Strategy:** + +**Month 1-2: Pilot** +- 5-10 VMs +- Single workload type +- Limited users +- Focus on learning + +**Month 3-6: Expansion** +- 20-50 VMs +- Multiple workload types +- Broader user base +- Refine processes + +**Month 6-12: Production Scale** +- 50-200+ VMs +- All workload types +- Organization-wide +- Mature operations + +**Benefits:** +- Avoids over-provisioning +- Iterative learning +- Cost-effective growth +- Risk mitigation + +### Monitoring and Alerting + +**Key Metrics to Monitor:** + +**VM Health:** +- Status (Running, Stopped, Error) +- Resource utilization (CPU, memory, disk, network) +- Guest agent connectivity +- Migration status + +**Node Health:** +- Resource utilization +- virt-handler pod status +- Network connectivity +- Storage backend health + +**Cluster Health:** +- Current migration count vs limits +- HyperConverged CR status +- Storage capacity and performance +- Network saturation + +**Migration Operations:** +- Success/failure rate +- Average migration duration +- Timeout occurrences +- Concurrent migration count + +**Alert Thresholds:** + +| Metric | Warning | Critical | +|--------|---------|----------| +| Node CPU | >80% | >90% | +| Node Memory | >85% | >95% | +| Migration Failures | >10% | >25% | +| virt-handler Pods Not Ready | Any | >1 | +| Cluster Migration Limit | >80% (4/5) | At limit (5/5) | + +### Backup and Disaster Recovery + +**VM Snapshots:** + +Use vm-snapshot skills for point-in-time backups before risky operations: +- Before major migrations +- Before configuration changes +- Before OS upgrades in guest +- Regular backup schedule (daily/weekly) + +**Disaster Recovery Planning:** + +**Multi-Zone Deployment:** +- Distribute VMs across availability zones +- Use topology spread constraints +- Configure zone-resilient storage + +**Backup Strategy:** +- Regular VM snapshots +- Export critical VM definitions +- Document restore procedures +- Test DR scenarios quarterly + +**RTO/RPO Targets:** + +Define recovery objectives: +- **RTO** (Recovery Time Objective): How quickly must VMs be recovered? +- **RPO** (Recovery Point Objective): How much data loss is acceptable? + +**Example Targets:** + +| Workload Tier | RTO | RPO | Strategy | +|---------------|-----|-----|----------| +| Critical | <15 min | <5 min | Multi-zone HA, frequent snapshots | +| Important | <1 hour | <1 hour | Daily snapshots, documented restore | +| Standard | <4 hours | <24 hours | Weekly snapshots, manual restore | + +--- + +## Cost Optimization + +### Resource Efficiency + +**Avoid Overprovisioning:** + +From Microsoft Azure Red Hat OpenShift guidance: +> "Avoid overprovisioning by aligning resources with actual usage patterns." + +**Cost Factors:** +- Azure compute costs (worker node instances) +- OpenShift licensing +- VM operating system licensing +- Storage costs (capacity and performance tier) +- Network egress charges + +**Optimization Strategies:** + +**1. Right-size VMs based on actual usage** + +Monitor with `pods_top` and resize VMs that are consistently under-utilized. + +**2. Use appropriate storage tiers** + +Don't use Premium storage for dev/test VMs; match tier to workload requirements. + +**3. Implement auto-scaling** + +For workloads with variable demand, use horizontal scaling rather than over-provisioning. + +**4. Consolidate with overcommit** + +In dev/test environments, use higher overcommit ratios (1.5-1.8x CPU) to maximize density. + +**5. Schedule non-critical VMs** + +Stop dev/test VMs during off-hours to reduce costs. + +### Load Balancing for Efficiency + +**Rebalancing Improves Efficiency:** +- Prevents hotspots (overloaded nodes) +- Enables better resource utilization +- Reduces need for emergency node additions +- Extends hardware lifespan (even wear) + +**Regular Rebalancing Schedule:** +- **Weekly**: Review node utilization, plan migrations if imbalance detected +- **Monthly**: Comprehensive rebalancing to optimize distribution +- **Quarterly**: Capacity planning and infrastructure rightsizing + +--- + +## Security Considerations + +### Tenant Isolation + +**Namespace Separation:** + +Deploy VMs for different tenants/teams in separate namespaces. + +**Network Policies:** + +Implement NetworkPolicies to restrict inter-VM communication: + +```yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: tenant-isolation + namespace: tenant-a +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress + ingress: + - from: + - namespaceSelector: + matchLabels: + tenant: tenant-a +``` + +**RBAC:** + +Grant users permissions only for their namespace's VMs, not cluster-wide access. + +### VM Security Hardening + +**Guest OS Security:** +- Regular patching and updates +- Disable unnecessary services +- Configure firewall rules +- Enable SELinux/AppArmor + +**Secrets Management:** +- Use Kubernetes Secrets for credentials +- Inject secrets into VMs via cloud-init +- Rotate secrets regularly +- Never store secrets in VM images + +**Access Control:** +- SSH key authentication only (disable password auth) +- Implement bastion/jump hosts +- Use VPN for remote access +- Audit access logs + +--- + +## Related Documentation + +- [Live Migration Best Practices](./live-migration-best-practices.md) - Configuration parameters and requirements +- [Performance Tuning](./performance-tuning.md) - Optimization strategies +- [Anti-Patterns](./anti-patterns.md) - Common mistakes to avoid + +--- + +**Last Updated**: 2026-02-24 +**OpenShift Virtualization Versions**: 4.17, 4.18, 4.19, 4.20 +**Status**: Production-ready guidance from official Red Hat sources diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/instruction.md b/evaluation/with_skills/rh-virt__vm-rebalance/instruction.md new file mode 100644 index 00000000..b4e5c640 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/instruction.md @@ -0,0 +1,13 @@ +# VM Rebalancing Task + +You are an OpenShift Virtualization administrator. Node `hv-prod-dc1-02` is critically overloaded (88% CPU, 82% memory). Plan how to rebalance its workloads by migrating one or more VMs to less utilized nodes. + +## Requirements +- Examine current node utilization and identify which VMs on `hv-prod-dc1-02` are candidates for migration +- Evaluate migration feasibility for each candidate and determine the appropriate migration method +- Select appropriate target nodes based on available capacity and schedulability +- Identify risks and safety considerations that could affect the migration + +Use MCP tools to examine the cluster. Document your methodology, findings, and migration plan in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/solution/solve.sh b/evaluation/with_skills/rh-virt__vm-rebalance/solution/solve.sh new file mode 100644 index 00000000..1f48a04e --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/solution/solve.sh @@ -0,0 +1,41 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Rebalancing Plan + +## Current State +Node hv-prod-dc1-02 is critically overloaded: 88% CPU (14080m/16000m), 82% memory (53739Mi/65536Mi). +VMs on this node: vm-web-prod-03, vm-api-prod-01, vm-cache-prod-01, vm-etl-prod-01. + +## Migration Candidates +- vm-web-prod-03 (4 CPU, 8Gi): good candidate, RWX storage supports live migration +- vm-cache-prod-01 (2 CPU, 4Gi): good candidate, small footprint +- vm-etl-prod-01 (4 CPU, 8Gi): degraded (high I/O latency), could benefit from migration but risky during active I/O + +## Live Migration Prerequisites +1. **Storage access mode**: Must be ReadWriteMany (RWX) for live migration. ReadWriteOnce (RWO) requires cold migration (VM must be stopped first). +2. **Node schedulability**: Target node must be schedulable (not cordoned or in maintenance). +3. **CPU model compatibility**: Source and target nodes must support the same CPU model. +4. **Available capacity**: Use allocated vCPU/memory from VM spec, not runtime usage metrics. + +## Target Node Selection +- hv-prod-dc1-01: 74% CPU, 68% memory — can accept one small VM +- hv-prod-dc1-03: cordoned for maintenance — NOT schedulable +- hv-prod-dc2-01/02: different datacenter zone, only for cross-zone rebalancing + +Recommendation: Migrate vm-cache-prod-01 (2 CPU, 4Gi) to hv-prod-dc1-01. + +## Anti-Patterns to Avoid +- **No ping-pong**: Don't migrate VMs back and forth between nodes repeatedly +- **Avoid resource overcommit**: Calculate post-migration allocated resources to ensure target stays below 85% +- **Don't migrate during peak hours**: Schedule during maintenance windows +- **Cold migration caution**: Re-read VM before updating nodeAffinity to avoid resourceVersion conflict +- **Overcommit warning**: If any node exceeds 85% after rebalance, escalate + +## Migration Procedure +1. Verify vm-cache-prod-01 storage is RWX (live migration supported) +2. Verify hv-prod-dc1-01 has capacity for 2 CPU + 4Gi after migration +3. Create VirtualMachineInstanceMigration resource +4. Monitor migration progress for convergence +5. Verify VM is healthy on target node post-migration + +REPORT_EOF diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/task.toml b/evaluation/with_skills/rh-virt__vm-rebalance/task.toml new file mode 100644 index 00000000..d79dfbba --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-rebalance" +name = "rh-virt VM Rebalancing Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-rebalance", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/tests/llm_judge.py b/evaluation/with_skills/rh-virt__vm-rebalance/tests/llm_judge.py new file mode 100644 index 00000000..76052f1f --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/tests/llm_judge.py @@ -0,0 +1,92 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "cpu_compatibility_check", "file": "/root/report.md", "question": "Does the report check CPU model or feature compatibility between source and target nodes before recommending migration?", "reference": "A skilled report verifies CPU compatibility (model, features) to ensure live migration success. An unskilled report migrates VMs without CPU compatibility checks."}, + {"id": "overcommit_awareness", "file": "/root/report.md", "question": "Does the report assess overcommit risk (whether the target node will exceed capacity after receiving migrated VMs)?", "reference": "A skilled report calculates whether the target node can handle the additional load without overcommitting. An unskilled report moves VMs without capacity verification."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/tests/test.sh b/evaluation/with_skills/rh-virt__vm-rebalance/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/tests/test_outputs.py b/evaluation/with_skills/rh-virt__vm-rebalance/tests/test_outputs.py new file mode 100644 index 00000000..ea445584 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-rebalance/tests/test_outputs.py @@ -0,0 +1,57 @@ +""" +Tests for rh-virt__vm-rebalance per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_migration(self): + content = read_report().lower() + assert "migrat" in content, "report should discuss migration" + + def test_mentions_node(self): + content = read_report().lower() + assert any(t in content for t in ["node", "overload", "imbalance", "utilization"]), ( + "report should reference cluster nodes or load imbalance" + ) + + +class TestSkillDependent: + def test_cpu_compatibility(self): + """Skill: CPU model/feature compatibility between source and target nodes.""" + c = read_report().lower() + assert any(t in c for t in ["cpu model", "cpu compatible", "cpu feature", "cpu architecture", "migration compatibility"]) or ( + "cpu" in c and ("compatib" in c or "model" in c) + ), ( + "should address CPU compatibility for migration" + ) + + def test_virtualmachineinstancemigration(self): + """Skill: VirtualMachineInstanceMigration for live migration.""" + c = read_report().lower() + assert any(t in c for t in ["virtualmachineinstancemigration", "vmi migration", "migration cr", "migration resource"]), ( + "should reference VirtualMachineInstanceMigration API" + ) + + def test_overcommit_warning(self): + """Skill: Overcommit detection; warn if node exceeds 100% after rebalance.""" + c = read_report().lower() + assert any(t in c for t in ["overcommit", "over commit", "exceed 100", "capacity"]) and ( + "overcommit" in c or "100" in c or "exceed" in c + ), ( + "should address overcommit risk when rebalancing" + ) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/Dockerfile b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/Dockerfile new file mode 100644 index 00000000..ae625e01 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/.ai-index/semantic-index.json new file mode 100644 index 00000000..2512c2a3 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/.ai-index/semantic-index.json @@ -0,0 +1,148 @@ +{ + "index_version": "1.0.0", + "last_updated": "2026-02-17", + "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", + + "documents": [ + { + "file": "INDEX.md", + "title": "VM Troubleshooting Guide - Index", + "category": "navigation", + "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], + "use_cases": ["navigation", "tool-reference", "general-guidance"], + "token_estimate": 450, + "skills": ["all"] + }, + { + "file": "scheduling-errors.md", + "title": "VM Scheduling Errors", + "category": "scheduling", + "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], + "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], + "errors_covered": [ + "ErrorUnschedulable - Node Taints", + "ErrorUnschedulable - Insufficient Resources", + "ErrorUnschedulable - Node Selector Mismatch" + ], + "token_estimate": 820, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "storage-errors.md", + "title": "VM Storage Errors", + "category": "storage", + "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], + "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], + "errors_covered": [ + "ErrorDataVolumeNotReady - DataVolume Still Provisioning", + "ErrorDataVolumeNotReady - Storage Class Not Found", + "ErrorDataVolumeNotReady - Insufficient Storage Quota", + "ErrorPvcNotFound", + "Storage Deletion Failures", + "DataVolume Cloning Failures" + ], + "token_estimate": 1350, + "skills": ["vm-creator", "vm-delete", "vm-clone"] + }, + { + "file": "lifecycle-errors.md", + "title": "VM Lifecycle Errors", + "category": "lifecycle", + "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], + "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], + "errors_covered": [ + "VM Stuck in Terminating State", + "VM Won't Start (Non-Scheduling Issues)", + "VM Won't Stop" + ], + "token_estimate": 1150, + "skills": ["vm-delete", "vm-lifecycle-manager"] + }, + { + "file": "runtime-errors.md", + "title": "VM Runtime Errors", + "category": "runtime", + "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], + "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], + "errors_covered": [ + "CrashLoopBackOff" + ], + "token_estimate": 850, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "network-errors.md", + "title": "VM Network Errors", + "category": "networking", + "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], + "use_cases": ["vm-creation", "network-troubleshooting"], + "errors_covered": [ + "Network Attachment Failures" + ], + "token_estimate": 580, + "skills": ["vm-creator"] + } + ], + + "skill_to_docs_mapping": { + "vm-creator": { + "primary_docs": ["scheduling-errors.md", "storage-errors.md"], + "secondary_docs": ["runtime-errors.md", "network-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3650, + "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" + }, + "vm-delete": { + "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 2950, + "description": "VM deletion failures - stuck terminating, storage cleanup" + }, + "vm-clone": { + "primary_docs": ["storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 1800, + "description": "VM cloning failures - DataVolume cloning, storage provisioning" + }, + "vm-lifecycle-manager": { + "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], + "secondary_docs": ["runtime-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3270, + "description": "VM start/stop failures - lifecycle management, scheduling" + }, + "vm-inventory": { + "primary_docs": [], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 450, + "description": "General guidance for status interpretation" + } + }, + + "error_to_docs_mapping": { + "ErrorUnschedulable": "scheduling-errors.md", + "ErrorDataVolumeNotReady": "storage-errors.md", + "ErrorPvcNotFound": "storage-errors.md", + "Terminating": "lifecycle-errors.md", + "CrashLoopBackOff": "runtime-errors.md", + "NetworkAttachmentFailed": "network-errors.md" + }, + + "cross_references": { + "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], + "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], + "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], + "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], + "network-errors.md": ["scheduling-errors.md"] + }, + + "progressive_disclosure_strategy": { + "description": "Load only the minimum documentation needed for current task", + "pattern": "error_type → primary_doc → cross_references (if needed)", + "token_savings": "29% average reduction vs single-file (based on rh-sre data)", + "navigation_overhead_reduction": "85% vs single-file navigation" + } +} diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/INDEX.md new file mode 100644 index 00000000..a2e76963 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/INDEX.md @@ -0,0 +1,332 @@ +--- +title: VM Troubleshooting Guide - Index +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 + - title: OpenShift Virtualization - Virtual Machine Status + url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html + date_accessed: 2026-02-06 + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] +semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] +use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] +last_updated: 2026-02-17 +--- + +# VM Troubleshooting Guide - Index + +## Overview + +This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: + +- **Creation**: Scheduling failures, storage provisioning issues +- **Lifecycle**: Start/stop failures, stuck states +- **Deletion**: Resources stuck in Terminating, storage cleanup failures +- **Cloning**: DataVolume cloning errors, cross-namespace issues +- **Networking**: Secondary network attachment failures +- **Runtime**: CrashLoopBackOff, guest OS boot failures + +This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. + +--- + +## 🗂️ Troubleshooting Categories + +The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: + +### 1. [Scheduling Errors](scheduling-errors.md) +**When to use**: VM fails to schedule on any node + +**Errors covered**: +- ErrorUnschedulable - Node Taints +- ErrorUnschedulable - Insufficient Resources +- ErrorUnschedulable - Node Selector Mismatch + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 2. [Storage Errors](storage-errors.md) +**When to use**: VM has storage provisioning, deletion, or cloning issues + +**Errors covered**: +- ErrorDataVolumeNotReady (all 3 subsections) +- ErrorPvcNotFound +- Storage Deletion Failures +- DataVolume Cloning Failures + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### 3. [Lifecycle Errors](lifecycle-errors.md) +**When to use**: VM has start/stop/termination issues + +**Errors covered**: +- VM Stuck in Terminating State +- VM Won't Start (Non-Scheduling Issues) +- VM Won't Stop + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### 4. [Runtime Errors](runtime-errors.md) +**When to use**: VM repeatedly crashes or fails at runtime + +**Errors covered**: +- CrashLoopBackOff + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 5. [Network Errors](network-errors.md) +**When to use**: VM has secondary network attachment failures + +**Errors covered**: +- Network Attachment Failures (Multus, SR-IOV) + +**Skills that use this**: vm-create + +--- + +## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last + +### Critical Principle: MCP-First Approach + +**All diagnostic procedures in this guide follow the MCP-first pattern**: + +``` +1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) +2. ⚠️ IF FAILS: Ask user permission to use CLI command +3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval +``` + +### Why MCP Tools First? + +- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources +- **Consistency**: Same interface across all operations +- **Better Error Handling**: MCP tools return structured errors +- **Audit Trail**: MCP tool usage is logged and trackable + +### Command Pattern Examples + +Throughout this guide, you'll see diagnostic steps formatted like this: + +**✅ CORRECT Pattern (MCP First)**: + +```markdown +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Extract**: `.metadata.finalizers` from returned JSON + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` +``` + +### Available MCP Tools + +The `openshift-virtualization` MCP server provides these tools: + +**Resource Operations**: +- `resources_get` - Get specific resource (replaces `oc get `) +- `resources_list` - List resources (replaces `oc get `) +- `resources_delete` - Delete resource (replaces `oc delete`) +- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) + +**Pod Operations**: +- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) +- `pods_get` - Get pod details (replaces `oc get pod`) +- `pods_log` - Get pod logs (replaces `oc logs`) +- `pods_exec` - Execute in pod (replaces `oc exec`) +- `pods_delete` - Delete pod (replaces `oc delete pod`) +- `pods_top` - Pod resource usage (replaces `oc top pods`) + +**Events & Monitoring**: +- `events_list` - List events (replaces `oc get events`) +- `nodes_top` - Node resource usage (replaces `oc top nodes`) +- `nodes_log` - Node logs (replaces `oc adm node-logs`) +- `nodes_stats_summary` - Detailed node stats + +**VM Operations** (KubeVirt toolset): +- `vm_create` - Create VMs +- `vm_lifecycle` - Start/stop/restart VMs + +**Namespaces**: +- `namespaces_list` - List namespaces (replaces `oc get namespaces`) +- `projects_list` - List OpenShift projects (replaces `oc get projects`) + +### When CLI Commands Are Required + +Some operations have **NO MCP equivalent** and require CLI: + +- `virtctl` commands (console, VNC access) +- `oc debug node` (node debugging) +- `oc auth can-i` (permission checks) +- `oc adm taint` (node taint management) +- Complex JSON patch operations + +For these, the guide will note: **"CLI Only - No MCP equivalent"** + +### Quick Reference: CLI → MCP Mapping + +| CLI Command | MCP Tool Equivalent | +|-------------|---------------------| +| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | +| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | +| `oc logs -n ` | `pods_log` with name="", namespace="" | +| `oc get events -n ` | `events_list` with namespace="" | +| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | + +**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. + +### How to Read Diagnostic Sections + +Each error section includes: +1. **Symptom** - What you observe +2. **Description** - What's happening +3. **Possible Causes** - Why it's happening +4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback +5. **Solutions** - **MCP tools first**, then CLI fallback +6. **Verification** - **MCP tools first**, then CLI fallback + +**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. + +--- + +## 🔍 Quick Navigation by Skill + +**vm-create**: +- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics +- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning +- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff +- [Network Errors](network-errors.md) - Network attachment failures + +**vm-delete**: +- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state +- [Storage Errors](storage-errors.md) - Storage deletion failures + +**vm-clone**: +- [Storage Errors](storage-errors.md) - DataVolume cloning failures + +**vm-lifecycle-manager**: +- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop +- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling + +**vm-inventory**: +- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed + +--- + +## 📊 Documentation Coverage & Maintenance + +### Current Coverage + +This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: + +- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) +- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures +- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop +- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) +- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems + +**Total errors documented**: 12 error types across 6 categories + +--- + +### Encountering Undocumented Errors + +#### For AI Agents (Claude Code) + +If you encounter an error **not documented** in the categories above: + +1. **Report to user** with all available details (error message, affected resources, namespace) +2. **Provide best-effort diagnostics** using MCP tools: + - `resources_get` to inspect resource status + - `pods_log` to check virt-launcher or compute container logs + - `events_list` to view Kubernetes events +3. **Suggest documentation update**: + ``` + ⚠️ This error is not yet documented in the troubleshooting guide. + + **Error**: + **Affected resource**: / + + I recommend adding this error to the troubleshooting guide: + - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file + - If it's a new error class → Create new category file in docs/troubleshooting/ + + Would you like me to help document this error for future reference? + ``` + +4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs + +--- + +#### For Human Contributors + +To document a new error: + +1. **Determine the category**: + - Scheduling issue → `scheduling-errors.md` + - Storage problem → `storage-errors.md` + - Lifecycle/start/stop → `lifecycle-errors.md` + - Runtime crash → `runtime-errors.md` + - Network issue → `network-errors.md` + - New error class → Create new category file (e.g., `security-errors.md`) + +2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): + - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback + - **Solutions**: MCP Tool approach → Implementation → CLI Fallback + - **Verification**: MCP Tool checks → Expected results → CLI Fallback + +3. **Update semantic index**: + - Edit `.ai-index/semantic-index.json` + - Add error to `error_to_docs_mapping` + - Update relevant skill's `primary_docs` or `secondary_docs` + - Increment token estimates if needed + +4. **Reference in skill files**: + - Update skill's Reference Documentation section + - Ensure skill's Document Consultation steps point to the right category + +5. **Update this INDEX.md**: + - Add error to the appropriate category section (lines 50-120) + - Update "Current Coverage" list above + +--- + +## 📚 Additional Resources + +- [rh-virt README](../../README.md) - MCP server setup and skill documentation +- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources +- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/lifecycle-errors.md new file mode 100644 index 00000000..6167fcc8 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/lifecycle-errors.md @@ -0,0 +1,869 @@ +--- +title: VM Lifecycle Errors +category: kubevirt +sources: + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 +tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] +semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] +use_cases: [vm-deletion, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Lifecycle Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM lifecycle issues including start/stop failures and stuck termination states. + +**When to use this document**: +- VM stuck in `Terminating` state +- VM won't start (runStrategy is Always but VM never reaches Running) +- VM won't stop (runStrategy is Halted but VM never reaches Stopped) + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### VM Stuck in Terminating State + +**Symptom**: VM shows status `Terminating` but deletion never completes + +**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). + +**Possible Causes**: +- Finalizers blocking deletion +- PVC/DataVolume still attached and preventing cleanup +- VirtualMachineInstance (VMI) not terminating properly +- Custom controllers or operators blocking deletion +- Stuck virt-launcher pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check finalizers on the VM**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` + +**2. Check if VMI still exists**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name containing "virt-launcher-". + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check events for deletion issues**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results where `involvedObject.name` == "" and sort by timestamp. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' +``` + +**5. Check VM deletion timestamp**: + +Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. + +If present, VM is in deletion process. If null, VM is not being deleted. + +**Common Finalizer Patterns**: +- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer +- `foregroundDeletion` - Waits for dependent objects to be deleted +- Custom finalizers from operators + +**Solutions** (Use MCP Tools First): + +1. **Wait for dependent resources** (safest approach): + - Use `resources_list` to check if PVCs, DataVolumes are still deleting + - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) + +2. **Force delete VMI** (if VMI is stuck): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable - requires explicit user permission): + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +3. **Force delete virt-launcher pod** (if pod is stuck): + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + # Find the virt-launcher pod name first + oc get pods -n | grep virt-launcher- + + # Then delete it + oc delete pod virt-launcher--xxx -n --force --grace-period=0 + ``` + +4. **Remove finalizers** (⚠️ dangerous - use only as last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` + 2. Remove items from `.metadata.finalizers` array + 3. Update VM using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch not easily done via MCP): + ```bash + # This operation is complex for MCP - may need CLI + oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Check for protection labels** (vm-delete skill specific): + + Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. + + If value is "true", the vm-delete skill refuses deletion (this is expected behavior). + +**Verification**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error if deletion successful. + +**CLI Fallback**: +```bash +oc get vm -n +# Should return: Error from server (NotFound) +``` + +--- + + +--- + +### VM Won't Start (Non-Scheduling Issues) + +**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state + +**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. + +**Possible Causes**: +- Guest OS kernel panic or boot failure +- Cloud-init configuration errors +- Missing or corrupted disk image +- Insufficient memory for guest OS to boot +- QEMU/KVM virtualization errors +- VirtualMachineInstance (VMI) creation failures +- virt-launcher pod crashes + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VMI (VirtualMachineInstance) status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and overall VMI state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**2. Check VMI conditions for errors**: + +Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. View virt-launcher pod logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +First, get pod name from step 3, then: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +Look for QEMU errors, memory allocation failures, device errors. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**5. Check virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. + +Alternatively, use `pods_get` to get full pod details: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**6. Access VM console to see guest OS boot messages**: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +Look for kernel panic, initramfs errors, filesystem errors. + +**7. Check VM events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep -A 20 "Events:" +``` + +**Common Error Patterns**: + +1. **Guest OS Boot Failure**: + - Console shows kernel panic + - Guest hangs at GRUB or boot loader + - Cloud-init errors during first boot + +2. **Insufficient Memory**: + - Guest OS kills processes due to OOM + - VMI logs show memory allocation errors + +3. **QEMU Crashes**: + - virt-launcher logs show QEMU segfaults + - VMI repeatedly restarts + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for kernel panic, initramfs errors, filesystem errors. + +2. **Check virt-launcher pod logs for QEMU errors**: + + Use `pods_log` from diagnostic step 4 to view logs. + + Look for: + - "qemu-system-x86_64: ..." errors + - Memory allocation failures + - Device errors + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx + ``` + +3. **Increase memory if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check VMI memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +4. **Verify disk image integrity**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check PVC is bound: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + If using container disk, verify image exists and is accessible (check VMI spec). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get pvc -n + ``` + +5. **Check cloud-init configuration** (if applicable): + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. + + Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq + ``` + +6. **Restart VMI** (soft reset): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete VMI (VM controller will recreate it): + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for new VMI to start (use `resources_get` to check status). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n -w + ``` + +7. **Check virtualization extensions** (KVM): + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + + # In debug shell: + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should eventually return `Running`). + +Check VMI is running: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should show `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should eventually return: Running + +oc get vmi -n +# Should show: Running +``` + +--- + +### VM Won't Stop + +**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state + +**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. + +**Possible Causes**: +- Guest OS not responding to ACPI shutdown signal +- virt-launcher pod stuck and not terminating +- VirtualMachineInstance (VMI) deletion blocked by finalizers +- Guest shutdown scripts hanging +- Filesystem sync issues in guest OS + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (might show `Stopping`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**2. Check VMI status and deletion timestamp**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check VMI events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vmi -n | grep -A 10 "Events:" +``` + +**5. Check VMI finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.metadata.finalizers}' +``` + +**6. Check if guest is responsive** (if VMI still exists): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Patterns**: +- VMI shows `deletionTimestamp` but never actually deletes +- virt-launcher pod in `Terminating` state +- VM runStrategy is `Halted` but printableStatus shows `Stopping` + +**Solutions** (Use MCP Tools First): + +1. **Wait for graceful shutdown** (default: 30 seconds): + + Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. + +2. **Force stop by deleting VMI**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + This is the standard way to force-stop a VM. + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n + # Should return: Error from server (NotFound) + ``` + +3. **Force delete VMI with grace period** (if VMI won't delete): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +4. **Force delete virt-launcher pod**: + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find the pod using `pods_list_in_namespace` from diagnostic step 3. + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + ⚠️ **Note**: For force deletion with grace period, use CLI fallback. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') + oc delete pod $POD_NAME -n --force --grace-period=0 + ``` + +5. **Remove VMI finalizers** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VMI using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update VMI using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Patch VM runStrategy directly** (ensure consistency): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` (diagnostic step 1) + 2. Set `.spec.runStrategy` to `"Halted"` + 3. Update VM using `resources_create_or_update` with modified JSON + + **CLI Fallback** (merge patch easier via CLI): + Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Stopped` or `Halted`). + +Verify VMI is gone: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Verify virt-launcher pod is gone: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for pods matching `virt-launcher-`. Should return no results. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Stopped or Halted + +oc get vmi -n +# Should return: Error from server (NotFound) + +oc get pods -n | grep virt-launcher- +# Should return: No resources found +``` + +**Prevention**: +- Ensure guest OS has ACPI support enabled +- Use proper shutdown commands in guest OS +- Avoid forceful stops unless necessary (can corrupt guest filesystem) + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/network-errors.md new file mode 100644 index 00000000..82fa313c --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/network-errors.md @@ -0,0 +1,429 @@ +--- +title: VM Network Errors +category: kubevirt +sources: + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] +semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] +use_cases: [vm-creation, network-troubleshooting] +related_docs: [INDEX.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Network Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. + +**When to use this document**: +- VM created successfully but secondary networks not attached +- NetworkAttachmentDefinition not found errors +- Multus CNI failures +- SR-IOV device attachment issues + +**Skills that use this**: vm-create + +--- + +### Network Attachment Failures + +**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working + +**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). + +**Possible Causes**: +- NetworkAttachmentDefinition doesn't exist in the namespace +- Multus CNI not installed or not configured on cluster +- Namespace mismatch (NAD in different namespace than VM) +- Interface name conflicts in VM spec +- Bridge/network configuration errors in NAD +- SR-IOV device not available (if using SR-IOV) + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List NetworkAttachmentDefinitions in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "" +} +``` + +Review `.items[].metadata.name` for available NADs. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -n +``` + +**2. List NetworkAttachmentDefinitions in all namespaces**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition" +} +``` + +Omit `namespace` parameter to list across all namespaces. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -A +``` + +**3. Check specific NetworkAttachmentDefinition**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "", + "name": "" +} +``` + +Review `.spec.config` for CNI configuration. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definition -n -o yaml +``` + +**4. Check VM network configuration**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.networks` to see network references. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq +``` + +**5. Check VM domain interfaces**: + +Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq +``` + +**6. Check virt-launcher pod network annotations** (shows actual attachments): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**7. Check Multus is installed**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "openshift-multus" +} +``` + +Should show Multus CNI pods running. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n openshift-multus +``` + +**8. Check for errors in virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n +``` + +**Common Error Messages**: +- `"network-attachment-definition not found"` - NAD doesn't exist in namespace +- `"multus CNI not configured"` - Multus not installed or misconfigured +- `"interface name conflict"` - Duplicate interface names in VM spec +- `"failed to add network"` - CNI plugin error (check NAD config) +- `"no available devices"` - SR-IOV device not available (if using SR-IOV) + +**Solutions** (Use MCP Tools First): + +1. **Verify NetworkAttachmentDefinition exists in correct namespace**: + + Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. + + If NAD is in different namespace, copy it to VM namespace: + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get NAD from source namespace using `resources_get` + 2. Modify `.metadata.namespace` to target namespace + 3. Create NAD in target namespace using `resources_create_or_update` + + **CLI Fallback** (stream processing easier via CLI): + Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" + ```bash + oc get network-attachment-definition -n -o yaml | \ + sed "s/namespace: /namespace: /" | \ + oc apply -f - + ``` + +2. **Create missing NetworkAttachmentDefinition**: + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + Example: Linux bridge network + + **Parameters**: + ```json + { + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": { + "name": "vlan100", + "namespace": "" + }, + "spec": { + "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" + ```bash + cat < + spec: + config: '{ + "cniVersion": "0.3.1", + "type": "bridge", + "bridge": "br1", + "vlan": 100, + "ipam": { + "type": "host-local", + "subnet": "192.168.100.0/24" + } + }' + EOF + ``` + +3. **Check Multus CNI installation**: + + Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. + + To check cluster network operator: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterOperator" + } + ``` + + Filter for `network` operator. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n openshift-multus + oc get clusteroperators network + ``` + +4. **Fix interface name conflicts** (if VM has duplicate names): + + Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. + + Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. + + **CLI Fallback** (interactive edit easier via CLI): + Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' + oc edit vm -n + ``` + +5. **Validate NAD configuration syntax**: + + Use `resources_get` from diagnostic step 3, extract `.spec.config`. + + Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get network-attachment-definition -n -o jsonpath='{.spec.config}' + ``` + +6. **Check SR-IOV device availability** (if using SR-IOV networks): + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + List SR-IOV network node policies: + + **Parameters**: + ```json + { + "apiVersion": "sriovnetwork.openshift.io/v1", + "kind": "SriovNetworkNodePolicy", + "namespace": "openshift-sriov-network-operator" + } + ``` + + Check SR-IOV device plugin pods: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "openshift-sriov-network-operator" + } + ``` + + Filter for pods with "device-plugin" in name. + + Check available SR-IOV devices on node: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "Node", + "name": "" + } + ``` + + Review `.status.allocatable` for SR-IOV resources. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get sriovnetworknodepolicy -n openshift-sriov-network-operator + oc get pods -n openshift-sriov-network-operator | grep device-plugin + oc describe node | grep -A 10 "Allocatable:" + ``` + +7. **Recreate VM with corrected network configuration** (if needed): + + If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +After remediation, check virt-launcher pod network status: + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +Should show all attached networks with status. Example output: +```json +[ + { + "name": "openshift-sdn", + "interface": "eth0", + "ips": ["10.128.2.10"], + "default": true + }, + { + "name": "vlan100", + "interface": "net1", + "ips": ["192.168.100.5"] + } +] +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**Check from inside VM** (via console): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# In guest OS: +ip addr show +# Should show all network interfaces (eth0, net1, etc.) +``` + +**Common Network Types**: +- **Linux Bridge**: Layer 2 bridge for VLAN networks +- **SR-IOV**: High-performance direct device assignment +- **macvlan**: MAC-based VLAN for container networks +- **OVN-Kubernetes**: OpenShift native overlay network + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/runtime-errors.md new file mode 100644 index 00000000..804d3cbc --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/runtime-errors.md @@ -0,0 +1,616 @@ +--- +title: VM Runtime Errors +category: kubevirt +sources: + - title: Kubernetes Pod Lifecycle + url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ + date_accessed: 2026-02-17 +tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] +semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] +use_cases: [vm-creation, vm-lifecycle, diagnostics] +related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Runtime Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. + +**When to use this document**: +- VM or virt-launcher pod shows `CrashLoopBackOff` status +- virt-launcher pod repeatedly restarting +- Guest OS kernel panics on boot + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### CrashLoopBackOff + +**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting + +**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. + +**Possible Causes**: +- Guest OS kernel panic on boot +- Insufficient resources (memory/CPU) for guest OS +- Corrupted disk image or filesystem +- QEMU/libvirt crashes due to configuration errors +- Missing or incompatible device drivers in guest +- Resource limits too low for virt-launcher pod +- Virtualization features (KVM) not available on node + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check virt-launcher pod restart count**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# Look at RESTARTS column +``` + +**2. View recent crash logs** (previous container instance): + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "previous": true, + "tail": 100 +} +``` + +Look for QEMU errors, kernel panics, or segfaults. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n virt-launcher--xxx --previous +``` + +**3. Check current virt-launcher logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. + +**CLI Fallback** (if MCP unavailable or all containers needed): +```bash +oc logs -n virt-launcher--xxx --all-containers +``` + +**4. Check VMI conditions for crash details**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Extract `.status.conditions` for crash details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**5. Check pod events for crash reasons**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" +``` + +**6. Check pod resource limits**: + +**MCP Tool**: `pods_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +Extract `.spec.containers[0].resources` for resource limits. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq +``` + +**7. Check node kubelet logs for OOM kills**: + +⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +oc adm node-logs -u kubelet | grep -i oom +``` + +**8. Access guest console** (if VM briefly starts): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Crash Patterns**: + +1. **Guest Kernel Panic**: + - Console logs show kernel panic messages + - Guest crashes immediately after boot + - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" + +2. **OOM (Out of Memory)**: + - Pod killed with reason: `OOMKilled` + - Guest runs out of memory during boot or operation + - virt-launcher logs show memory allocation failures + +3. **QEMU Crash**: + - virt-launcher logs show QEMU segmentation fault + - Symptoms: "qemu-system-x86_64: terminated by signal" + - Configuration incompatibility or QEMU bug + +4. **Disk Image Corruption**: + - Guest cannot boot from disk + - Filesystem errors in guest console + - DataVolume import failed + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for kernel panic or boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for: + - Kernel panic messages + - Initramfs errors + - Filesystem mounting failures + - Missing device errors + +2. **Review virt-launcher crash logs**: + + Use `pods_log` with `previous: true` from diagnostic step 2. + + Look for: + - QEMU command line errors + - Device initialization failures + - Memory allocation errors + - Signal termination (SIGSEGV, SIGABRT) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx --previous + ``` + +3. **Check for OOM (Out of Memory) kills**: + + **MCP Tool**: `pods_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + Extract `.status.containerStatuses[0].lastState.terminated.reason`. + + If returns `"OOMKilled"`: + - Option 1: Increase virt-launcher memory limits + - Option 2: Decrease guest memory allocation + - Option 3: Use smaller instance type + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' + ``` + +4. **Increase resources if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check current memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +5. **Verify disk image integrity**: + + **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters for list**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" + } + ``` + + **Parameters for specific DV**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check `.status.phase` (should be `Succeeded`). + + If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get datavolume -n -o jsonpath='{.status.phase}' + ``` + +6. **Check virtualization (KVM) availability**: + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +7. **Simplify VM configuration** (eliminate variables): + + Try creating minimal VM using vm-create skill with: + - Small instance type + - No secondary networks + - Simple container disk (e.g., Fedora) + - No cloud-init + + If minimal VM works, add features back one by one. + +8. **Recreate VM with different workload** (test disk image): + + If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) + +After remediation, check pod restart count stops increasing: + +**Parameters for pods**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). + +Check VM reaches Running state: + +**Parameters for VM**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# RESTARTS should stabilize + +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Running +``` + +Verify guest is responsive: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# Should show login prompt or OS console +``` + +**Advanced Debugging**: + +**MCP Tool**: `pods_exec` (from openshift-virtualization) + +Check libvirt domain XML: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["virsh", "dumpxml", "1"] +} +``` + +Check QEMU process: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["ps", "aux"] +} +``` + +Filter output for "qemu" process. + +**CLI Fallback** (if MCP unavailable): +```bash +oc exec -n virt-launcher--xxx -- virsh dumpxml 1 +oc exec -n virt-launcher--xxx -- ps aux | grep qemu +``` + +**Prevention**: +- Start with minimal VM configuration and add complexity gradually +- Use recommended instance types for your workload +- Test disk images before deploying to production +- Ensure nodes have adequate resources and KVM support +- Monitor resource usage and set appropriate limits + +--- + +## Workaround Patterns for MCP Tool Limitations + +### General Pattern: Diagnose → Propose → Confirm → Execute + +When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): + +**1. Diagnose** the root cause using Kubernetes commands +```bash +oc describe vm -n +oc get events -n --field-selector involvedObject.name= +``` + +**2. Propose** a manual workaround to the user with clear explanation +- Explain why the issue occurred +- Show what will be changed +- List alternative options + +**3. Confirm** with user before executing (Human-in-the-Loop) +- Wait for explicit "yes" or "apply workaround" +- Never auto-execute modifications + +**4. Execute** the workaround using oc/kubectl +```bash +oc patch vm -n --type=merge -p '...' +``` + +**5. Verify** the fix was successful +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**6. Document** the limitation and suggest filing enhancement request +- Note this is temporary until MCP tool is enhanced +- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues + +### Example: Adding Tolerations Workaround + +**Diagnostic Output**: +```markdown +## ⚠️ VM Scheduling Issue Detected + +**Root Cause**: Node taints prevent VM scheduling + +**Details**: +- Found 3 nodes with taint: `virtualization=true:NoSchedule` +- VM spec does not include matching tolerations +- This prevents VM scheduling on virtualization-dedicated nodes +``` + +**Proposed Workaround**: +```bash +oc patch vm web-server -n vms --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' +``` + +**User Confirmation Required**: +``` +How would you like to proceed? +- "apply workaround" - I'll patch the VM with tolerations +- "manual" - I'll provide instructions for you to apply manually +- "cancel" - Delete the VM and abort creation +``` + +**After Execution**: +```markdown +## ✓ Workaround Applied Successfully + +**Action**: Added tolerations for taint `virtualization=true:NoSchedule` +**New Status**: Stopped (VM can now be scheduled) + +**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. +Future VMs in this cluster will need the same fix until the tool is enhanced. +``` + +--- + +## VM Status Reference + +### Status Values + +| Status | Meaning | Action Required | +|--------|---------|-----------------| +| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | +| `Running` | VM is running | Normal | +| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | +| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | +| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | +| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | +| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | +| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | +| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | +| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | + +### Checking VM Status + +```bash +# Get printable status +oc get vm -n -o jsonpath='{.status.printableStatus}' + +# Get detailed status and conditions +oc get vm -n -o jsonpath='{.status}' | jq + +# Watch status changes in real-time +oc get vm -n -w +``` + +--- + +## Best Practices for Agents + +When implementing diagnostic workflows: + +1. **Always verify VM status** after creation (wait 5-10 seconds first) +2. **Consult this document** when encountering error status values +3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) +4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) +5. **Respect human-in-the-loop** for all VM modifications +6. **Document temporary workarounds** and their limitations clearly +7. **Suggest filing issues** for missing MCP tool features + +### Document Consultation Pattern + +```markdown +**Document Consultation** (REQUIRED): +1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes +2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." +``` + +--- + +## Known MCP Tool Limitations + +### vm_create tool + +**Currently Supported**: +- ✓ Namespace, name (required) +- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) +- ✓ Size hints (small, medium, large) +- ✓ Storage size +- ✓ Autostart flag +- ✓ Networks (Multus NetworkAttachmentDefinitions) +- ✓ Performance family (u1, o1, c1, m1) +- ✓ Instance type, preference + +**Not Currently Supported** (requires workarounds): +- ✗ Tolerations (for node taints) +- ✗ Node selectors +- ✗ Affinity/anti-affinity rules +- ✗ Resource requests/limits (beyond instance type) +- ✗ Custom labels/annotations +- ✗ SSH keys injection +- ✗ Cloud-init user data + +**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. + +**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues + +--- + +## Additional Resources + +- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) +- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) +- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) +- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) + +--- + +[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/scheduling-errors.md new file mode 100644 index 00000000..e47c8c4f --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/scheduling-errors.md @@ -0,0 +1,417 @@ +--- +title: VM Scheduling Errors +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 +tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] +semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] +use_cases: [vm-creation, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Scheduling Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**When to use this document**: +- VM shows status `ErrorUnschedulable` after creation or start attempt +- VM events mention scheduling failures, taints, resources, or node selectors + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +## ErrorUnschedulable + +**Symptom**: VM shows status `ErrorUnschedulable` after creation + +**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**Possible Causes**: + +### 1. Node Taints (Most Common) + +Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM events for scheduling failures**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `involvedObject.name` == "" and look for messages like: +- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= +``` + +**2. Check node taints in the cluster**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' +``` + +**Common Taint Patterns**: +- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule +- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes +- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads + +**Solution - Add Tolerations to VM**: + +The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. + +**Workaround (post-creation using MCP Tools)**: + +**Step 1**: Get current VM spec + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Step 2**: Modify the returned JSON to add tolerations + +Add to `.spec.template.spec.tolerations`: +```json +{ + "tolerations": [ + { + "key": "virtualization", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule" + } + ] +} +``` + +**Step 3**: Update VM with modified spec + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +```json +{ + "resource": "" +} +``` + +Pass the complete modified VM resource as YAML or JSON string. + +**Step 4**: Verify tolerations were added + +Use `resources_get` again and check `.spec.template.spec.tolerations` in response. + +**Step 5**: Check if VM status improved + +Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. + +**CLI Fallback** (if MCP patch is too complex): +```bash +# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' + +# Verify tolerations +oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq + +# Check status +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**Example - Multiple Tolerations**: +```bash +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: "dedicated" + operator: "Equal" + value: "virt-workloads" + effect: "NoSchedule" +' +``` + +**Toleration Operators**: +- `Equal` - Key and value must match exactly +- `Exists` - Only key must exist (ignores value) + +**Toleration Effects**: +- `NoSchedule` - Don't schedule new pods (existing pods continue) +- `PreferNoSchedule` - Avoid scheduling if possible +- `NoExecute` - Don't schedule AND evict existing pods + +**Alternative Solutions**: +1. **Remove node taints** (if you have cluster-admin access): + ```bash + oc adm taint nodes virtualization=true:NoSchedule- + ``` + +2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits + +3. **File enhancement request** - Request tolerations support in openshift-mcp-server: + https://github.com/openshift/openshift-mcp-server/issues + +--- + +### 2. Insufficient Resources + +Not enough CPU, memory, or storage available on any node. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM resource requests**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' +``` + +**2. Check node resource availability**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. + +Alternatively, use `nodes_top` MCP tool for current resource usage. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe nodes | grep -A 5 "Allocated resources" +``` + +**3. Look for VM events mentioning "Insufficient"**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep "Insufficient" +``` + +**Example Event**: +``` +0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. +``` + +**Solutions** (Use MCP Tools First): + +1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) +2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill +3. **Delete unused VMs** - Use vm-delete skill to free up resources +4. **Check resource quotas**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters for quota**: + ```json + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" + } + ``` + + **Parameters for limit range**: + ```json + { + "apiVersion": "v1", + "kind": "LimitRange", + "namespace": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + oc describe limitrange -n + ``` + +--- + +### 3. Node Selector Mismatch + +VM requires specific node labels that don't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM node selector requirements**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.nodeSelector` to see required node labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' +``` + +**2. List available node labels**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.metadata.labels` for available labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes --show-labels +``` + +**3. Check if any nodes match the selector**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node", + "labelSelector": "=" +} +``` + +Should return at least one node with matching labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -l = +``` + +**Solutions** (Use MCP Tools First): + +**Option 1: Remove node selector from VM** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get current VM using `resources_get` (diagnostic step 1) +2. Remove `.spec.template.spec.nodeSelector` field +3. Update VM using `resources_create_or_update` with modified JSON + +**CLI Fallback** (JSON patch easier via CLI): +Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" +```bash +oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' +``` + +**Option 2: Add label to nodes** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get node using `resources_get` +2. Add label to `.metadata.labels` +3. Update node using `resources_create_or_update` + +⚠️ **Note**: Node labeling typically requires cluster admin privileges. + +**CLI Fallback** (simpler via CLI): +Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" +```bash +oc label node = +``` + +--- + +[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/storage-errors.md new file mode 100644 index 00000000..16ba584d --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/storage-errors.md @@ -0,0 +1,1011 @@ +--- +title: VM Storage Errors +category: kubevirt +sources: + - title: KubeVirt CDI - DataVolumes + url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ + date_accessed: 2026-02-17 + - title: Kubernetes Persistent Volumes + url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ + date_accessed: 2026-02-17 +tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] +semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] +use_cases: [vm-creation, vm-deletion, vm-cloning] +related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] +last_updated: 2026-02-17 +--- + +# VM Storage Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. + +**When to use this document**: +- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` +- Storage deletion fails after VM deletion +- DataVolume cloning operations fail +- PVC provisioning issues + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### ErrorDataVolumeNotReady + +**Symptom**: VM shows status `ErrorDataVolumeNotReady` + +**Description**: The DataVolume (persistent storage) backing the VM is not ready. + +**Possible Causes**: + +#### 1. DataVolume Still Provisioning + +Storage provisioning takes time, especially for large disks or when importing images. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Get detailed DataVolume information**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` and `.status.conditions` for provisioning details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o json +``` + +**3. Check PVC (PersistentVolumeClaim) bound status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Check `.status.phase` for each PVC (should be `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. + +#### 2. Storage Class Not Found + +The requested storage class doesn't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List available storage classes**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" +} +``` + +Review the list of available storage classes (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass +``` + +**2. Check DataVolume's requested storage class**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.spec.pvc.storageClassName` in the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' +``` + +**Solution**: +1. Use a valid storage class from the cluster +2. Recreate VM with correct storage class parameter + +#### 3. Insufficient Storage Quota + +Namespace has insufficient storage quota to provision the PVC. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check resource quotas**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**2. Check storage usage**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase +``` + +**Solution**: +1. Request quota increase from cluster admin +2. Delete unused PVCs to free quota +3. Reduce VM storage size + +--- + +### ErrorPvcNotFound + +**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List PVCs in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review the list of available PVCs (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check VM's PVC references**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' +``` + +**Solution**: +- Wait for DataVolume to create the PVC +- Manually create missing PVC +- Fix VM spec to reference correct PVC name + +--- + + +--- + +### Storage Deletion Failures + +**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace + +**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. + +**Possible Causes**: +- PVC still bound to active PersistentVolume with `Retain` policy +- DataVolume still being referenced by another resource +- CDI (Containerized Data Importer) controller issues +- Storage class retention policy preventing deletion +- Finalizers on PVC/DataVolume blocking cleanup +- PVC still mounted by a pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check PVC status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check specific PVC phase**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Released` or `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o jsonpath='{.status.phase}' +``` + +**3. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**4. Check what's using the PVC**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' +``` + +**5. Check PVC finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. + +**6. Check DataVolume finalizers**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.metadata.finalizers}' +``` + +**7. Check PV reclaim policy**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume" +} +``` + +Filter results for PV where `.spec.claimRef.name` matches ``. + +To get specific PV policy: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume", + "name": "" +} +``` + +Check `.spec.persistentVolumeReclaimPolicy`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pv | grep +oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' +``` + +**Common Finalizer Patterns**: +- `kubernetes.io/pvc-protection` - Protects PVC while in use +- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer + +**Solutions** (Use MCP Tools First): + +1. **Delete DataVolume first, then PVC**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete DataVolume first (often blocks PVC deletion): + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Wait a few seconds, then delete PVC: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete datavolume -n + oc delete pvc -n + ``` + +2. **Check for pods still using PVC**: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "" + } + ``` + + Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. + + Then delete the pods using `pods_delete`: + + **Parameters**: + ```json + { + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' + oc delete pod -n + ``` + +3. **Force delete PVC** (if safe to do so): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete pvc -n --grace-period=0 --force + ``` + +4. **Remove finalizers from PVC** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PVC using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update PVC using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Remove finalizers from DataVolume** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current DataVolume using `resources_get` (diagnostic step 6) + 2. Remove items from `.metadata.finalizers` array + 3. Update DataVolume using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Change PV reclaim policy** (if PV has Retain policy): + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PV policy using `resources_get` (diagnostic step 7) + 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` + 3. Update PV using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" + ```bash + oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' + oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' + ``` + +**Storage Quota Check** (Use MCP Tools First): + +After deletion, verify storage quota is freed: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters for quota check**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.used` to verify storage quota is freed. + +**Parameters for PVC verification**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +oc get pvc -n +``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Confirm PVC is deleted: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Confirm DataVolume is deleted: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +# Should return: Error from server (NotFound) + +oc get datavolume -n +# Should return: Error from server (NotFound) +``` + +--- + +### DataVolume Cloning Failures + +**Symptom**: VM clone created successfully but DataVolume clone operation fails + +**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. + +**Possible Causes**: +- CSI driver doesn't support volume cloning +- Source PVC storage class incompatible with cloning +- Cross-namespace cloning not permitted by storage backend +- Insufficient storage quota in target namespace +- Source PVC not in `Bound` state +- Storage class doesn't have volume cloning enabled +- CDI (Containerized Data Importer) controller issues + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume clone status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and `.metadata.name`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Check DataVolume events for errors**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe datavolume -n +``` + +**3. Check DataVolume phase**: + +Use `resources_get` from step 1, extract `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +``` + +**4. Check if storage class supports cloning**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "name": "" +} +``` + +Review the full YAML output for cloning-related configurations. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass -o yaml | grep -A 5 -i clone +``` + +**5. Check CSI driver capabilities**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "CSIDriver" +} +``` + +Review `.items[].metadata.name` for available CSI drivers. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get csidriver +``` + +**6. Check source PVC status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Bound` for cloning to work). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**7. Check target namespace storage quota**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**8. Check CDI controller logs**: + +**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) + +First, list pods in openshift-cnv namespace: + +**Parameters for pods_list_in_namespace**: +```json +{ + "namespace": "openshift-cnv", + "labelSelector": "app.kubernetes.io/component=cdi-deployment" +} +``` + +Then get logs using `pods_log`: + +**Parameters**: +```json +{ + "namespace": "openshift-cnv", + "name": "", + "tail": 100 +} +``` + +**CLI Fallback** (if MCP unavailable or easier via CLI): +```bash +oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') +``` + +**Common Error Messages**: +- `"volume cloning is not supported"` - CSI driver lacks clone capability +- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage +- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace +- `"insufficient quota"` - Target namespace lacks storage quota +- `"source PVC not bound"` - Source PVC must be in Bound state for cloning +- `"StorageClass does not support cloning"` - Storage class configuration issue + +**Solutions** (Use MCP Tools First): + +1. **Check storage class clone support**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" + } + ``` + + For each storage class in `.items[]`, check: + - `.metadata.name` (storage class name) + - `.provisioner` (CSI driver) + + Storage classes using CSI drivers typically support cloning. Look for provisioners like: + - `csi.ovirt.org` (oVirt CSI) + - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) + - `ebs.csi.aws.com` (AWS EBS CSI) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner + ``` + +2. **Verify source PVC is bound**: + + Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pvc -n -o jsonpath='{.status.phase}' + ``` + +3. **Check target namespace quota**: + + Use `resources_list` from diagnostic step 7 to check quota. + + If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + ``` + +4. **Use snapshot-based cloning** (alternative method): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Step 1**: Create VolumeSnapshot of source PVC + + **Parameters**: + ```json + { + "apiVersion": "snapshot.storage.k8s.io/v1", + "kind": "VolumeSnapshot", + "metadata": { + "name": "-snapshot", + "namespace": "" + }, + "spec": { + "source": { + "persistentVolumeClaimName": "" + } + } + } + ``` + + **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) + + **Step 3**: Create new DataVolume from snapshot + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": "-rootdisk", + "namespace": "" + }, + "spec": { + "source": { + "snapshot": { + "name": "-snapshot", + "namespace": "" + } + }, + "storage": { + "resources": { + "requests": { + "storage": "50Gi" + } + }, + "storageClassName": "" + } + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" + ```bash + cat <-snapshot + namespace: + spec: + source: + persistentVolumeClaimName: + EOF + + oc get volumesnapshot -snapshot -n + + cat <-rootdisk + namespace: + spec: + source: + snapshot: + name: -snapshot + namespace: + storage: + resources: + requests: + storage: 50Gi + storageClassName: + EOF + ``` + +5. **Use "new empty storage" option** (vm-clone skill): + - If cloning isn't supported, create VM with empty storage + - Manually copy data if needed + +6. **Cross-namespace cloning workaround**: + - Some storage backends require snapshot for cross-namespace cloning + - Create snapshot in source namespace, restore in target namespace (see solution 4 above) + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Check DataVolume reached Succeeded phase: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should return `Succeeded`). + +Check PVC was created and bound: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "-rootdisk" +} +``` + +Check `.status.phase` (should return `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +# Should return: Succeeded + +oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' +# Should return: Bound +``` + +**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/mcp-servers/mock-virt-mcp.py b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..912fb2d6 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1539 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), + ("prod-vms", {"env": "production"}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── prod-vms (instruction-specific) ────────────────────────────────── + _vm("production-db", "prod-vms", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true"}, + 8, 16, "Running", True, 1), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + +STORAGE_CLASSES = [ + { + "name": "ocs-storagecluster-ceph-rbd", + "provisioner": "openshift-storage.rbd.csi.ceph.com", + "reclaimPolicy": "Delete", + "volumeBindingMode": "Immediate", + "allowVolumeExpansion": True, + }, + { + "name": "ocs-storagecluster-cephfs", + "provisioner": "openshift-storage.cephfs.csi.ceph.com", + "reclaimPolicy": "Delete", + "volumeBindingMode": "Immediate", + "allowVolumeExpansion": False, + }, +] + +VOLUME_SNAPSHOT_CLASSES = [ + { + "name": "ocs-storagecluster-rbdplugin-snapclass", + "driver": "openshift-storage.rbd.csi.ceph.com", + "deletionPolicy": "Delete", + }, +] + + +def _build_storage_class(sc): + """Build a storage.k8s.io/v1 StorageClass resource.""" + res = { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "metadata": { + "name": sc["name"], + "uid": _uid(sc["name"]), + "creationTimestamp": CREATED, + }, + "provisioner": sc["provisioner"], + "reclaimPolicy": sc["reclaimPolicy"], + "volumeBindingMode": sc["volumeBindingMode"], + } + if sc.get("allowVolumeExpansion"): + res["allowVolumeExpansion"] = True + return res + + +def _build_volume_snapshot_class(vsc): + """Build a snapshot.storage.k8s.io/v1 VolumeSnapshotClass resource.""" + return { + "apiVersion": "snapshot.storage.k8s.io/v1", + "kind": "VolumeSnapshotClass", + "metadata": { + "name": vsc["name"], + "uid": _uid(vsc["name"]), + "creationTimestamp": CREATED, + }, + "driver": vsc["driver"], + "deletionPolicy": vsc["deletionPolicy"], + } + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "storage.k8s.io/v1" and kind == "StorageClass": + resources = [_build_storage_class(sc) for sc in STORAGE_CLASSES] + headers = ["NAME", "PROVISIONER", "RECLAIMPOLICY", "VOLUMEBINDINGMODE", "ALLOWVOLUMEEXPANSION", "AGE"] + def row(r): + return [r["metadata"]["name"], r["provisioner"], + r["reclaimPolicy"], r["volumeBindingMode"], + str(r.get("allowVolumeExpansion", False)), "90d"] + return resources, headers, row, False + + if api_version == "snapshot.storage.k8s.io/v1" and kind == "VolumeSnapshotClass": + resources = [_build_volume_snapshot_class(vsc) for vsc in VOLUME_SNAPSHOT_CLASSES] + headers = ["NAME", "DRIVER", "DELETIONPOLICY", "AGE"] + def row(r): + return [r["metadata"]["name"], r["driver"], r["deletionPolicy"], "90d"] + return resources, headers, row, False + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/skills/vm-snapshot-create/SKILL.md b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/skills/vm-snapshot-create/SKILL.md new file mode 100644 index 00000000..e651c6ef --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/skills/vm-snapshot-create/SKILL.md @@ -0,0 +1,423 @@ +--- +name: vm-snapshot-create +description: | + Create virtual machine snapshots for backup and recovery. + + Use when: + - "Create a snapshot of VM [name]" + - "Backup VM [name] before upgrade" + - "Take a snapshot of [vm]" + + Validates storage class snapshot support, CSI driver capabilities, and guest agent status before snapshot creation. + + NOT for VM cloning (use vm-clone to create independent copies). + +model: inherit +color: green +--- + +# /vm-snapshot-create Skill + +Create virtual machine snapshots in OpenShift Virtualization. Snapshots capture the state and data of a VM at a specific point in time, enabling backup, recovery, and testing workflows. + +**Implementation Note**: This skill uses generic Kubernetes resource tools (`resources_create_or_update`) to manage VirtualMachineSnapshot resources. Dedicated snapshot tools do not currently exist in the openshift-virtualization MCP server. + +## Prerequisites + +**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools**: +- `resources_create_or_update` (from openshift-virtualization) - Create VirtualMachineSnapshot +- `resources_get` (from openshift-virtualization) - Verify VM exists and get status +- `resources_list` (from openshift-virtualization) - List StorageClass, VolumeSnapshotClass + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster (>= 4.19) +- OpenShift Virtualization operator installed +- ServiceAccount with RBAC permissions to create VirtualMachineSnapshot resources +- Storage backend with snapshot support (CSI driver with snapshot capabilities) + +## When to Use This Skill + +**Trigger this skill when:** +- User wants to create a backup of a VM before changes +- User wants to create a recovery point +- User explicitly requests snapshot creation + +**User phrases that trigger this skill:** +- "Create a snapshot of VM database-01" +- "Backup VM web-server before upgrade" +- "Take a snapshot of production-app" + +**Do NOT use this skill when:** +- User wants to clone a VM → Use `vm-clone` skill (creates independent copy) +- User wants to list snapshots → Use `vm-snapshot-list` skill +- User wants to restore from snapshot → Use `vm-snapshot-restore` skill + +## Workflow + +### Step 1: Gather Snapshot Information + +**Required Information from User:** +1. **VM Name** - Name of the VM to snapshot +2. **Namespace** - Namespace where VM exists +3. **Snapshot Name** (Optional) - Name for the snapshot (auto-generated if not provided) + +If namespace not provided, ask for it explicitly. + +### Step 2: Verify VM Exists and Get Status + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Expected Output**: VirtualMachine resource with status + +**Error Handling**: +- If VM not found → Report error, suggest using vm-inventory skill +- If permission denied → Report RBAC error + +**Extract VM Details:** +- Current status (Running, Stopped) +- Storage configuration (DataVolumes, PVCs) +- **IMPORTANT**: Save `status.volumeSnapshotStatuses` for storage analysis + +### Step 3: Verify Storage Snapshot Capabilities + +**CRITICAL: This comprehensive storage analysis MUST execute BEFORE asking user about VM running state.** + +This step analyzes storage backend capabilities to determine snapshot behavior and requirements. The analysis includes 9 substeps. + +[Continue with all 9 substeps from the original file: 1c.1 through 1c.9, checking volume snapshot status, hot-plugged volumes, storage class, VolumeSnapshotClass, CSI driver capabilities, guest agent status, Windows VSS, and storing analysis results] + +### Step 4: Check VM Running State (Enhanced with Storage Analysis) + +**From the VM resource in Step 2**, check `status.printableStatus`. + +**Use storage analysis results from Step 3** to provide accurate guidance. + +[Include the three scenarios: VM must be stopped, VM can run (online supported), VM is stopped - with all the guest agent and Windows VSS warnings] + +### Step 5: Stop Running VM (if user chose "stop-and-snapshot") + +**ONLY execute if user chose "stop-and-snapshot" in Step 4.** + +Use `vm_lifecycle` MCP tool or vm-lifecycle-manager skill to stop the VM. + +### Step 6: Estimate Storage Consumption + +**From the VM resource**, estimate snapshot storage: +- Initial snapshot may be same size as VM disk +- Subsequent snapshots smaller (only deltas) + +### Step 7: Present Snapshot Configuration for Confirmation + +**Include storage analysis results from Step 3 in the configuration presentation.** + +[Include the full confirmation template with storage backend analysis, guest agent status, volumes to snapshot, etc.] + +**Wait for user confirmation.** + +**Handle response:** +- If "yes" → Proceed to Step 8 (execute snapshot) +- If "no", "cancel", or anything else → Cancel operation + +### Step 8: Create the Snapshot + +**ONLY PROCEED AFTER user confirmation in Step 7.** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Construct VirtualMachineSnapshot YAML:** + +```yaml +apiVersion: snapshot.kubevirt.io/v1beta1 +kind: VirtualMachineSnapshot +metadata: + name: + namespace: +spec: + source: + apiGroup: kubevirt.io + kind: VirtualMachine + name: +``` + +**If snapshot name not provided by user**, generate one: +- Format: `-snapshot-` +- Example: `database-01-snapshot-20260218-143022` + +**Parameters**: +```json +{ + "resource": "apiVersion: snapshot.kubevirt.io/v1beta1\nkind: VirtualMachineSnapshot\nmetadata:\n name: \n namespace: \nspec:\n source:\n apiGroup: kubevirt.io\n kind: VirtualMachine\n name: " +} +``` + +**Report progress:** +```markdown +📸 Creating VM snapshot... +✓ Snapshot `` created for VM `` +``` + +### Step 9: Monitor Snapshot Status + +**After creation, monitor snapshot readiness using `resources_get`.** + +Check `status.phase`: +- `InProgress` → Still creating +- `Succeeded` → Snapshot ready +- `Failed` → Snapshot failed + +**Wait up to 5 minutes for snapshot to complete.** + +### Step 10: Report Snapshot Creation Results + +**Extract snapshot indications** from `status.indications`: +- `GuestAgent` - Guest agent coordinated the snapshot +- `Online` - Snapshot taken while VM was running + +**On success:** + +```markdown +## ✓ VM Snapshot Created Successfully + +**VM**: `` (namespace: ``) +**Snapshot**: `` + +### Snapshot Details +- **Name**: `` +- **Status**: Ready +- **Created**: +- **VM Status at Snapshot**: + +### Snapshot Coordination (from status.indications) + +- ✅ **Guest Agent Coordination**: Active +- ✅ **Filesystem Freeze/Thaw**: Performed during snapshot +- ✅ **Actual Consistency**: Application-consistent + + + +- ⚠️ **Guest Agent Coordination**: Not active +- ⚠️ **Actual Consistency**: Crash-consistent (best-effort) + + +### Next Steps + +**To list all snapshots:** +"List snapshots for VM " + +**To restore from this snapshot:** +"Restore VM from snapshot " + +**To delete this snapshot:** +"Delete snapshot " +``` + +## Dependencies + +### Required MCP Servers +- `openshift-virtualization` - OpenShift MCP server with kubevirt toolset + +### Required MCP Tools +- `resources_create_or_update` (from openshift-virtualization) - Create VirtualMachineSnapshot +- `resources_get` (from openshift-virtualization) - Verify VM and snapshot status +- `resources_list` (from openshift-virtualization) - List StorageClass, VolumeSnapshotClass + +### Related Skills +- `vm-snapshot-list` - List snapshots after creation +- `vm-snapshot-restore` - Restore VMs from snapshots +- `vm-snapshot-delete` - Delete old snapshots +- `vm-lifecycle-manager` - Stop VMs before snapshot +- `vm-inventory` - List VMs before creating snapshots + +### Reference Documentation + +**Official Red Hat Documentation:** +- [OpenShift Virtualization Snapshots - OpenShift 4.20](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt-managing-vm-snapshots) +- [Live Snapshots in OpenShift Virtualization](https://www.redhat.com/en/blog/live-snapshots-in-openshift-virtualization) + +**Upstream Documentation:** +- [KubeVirt VM Snapshots](https://kubevirt.io/user-guide/operations/snapshot_restore_api/) +- [CSI Volume Snapshots](https://kubernetes.io/docs/concepts/storage/volume-snapshots/) + +## Critical: Human-in-the-Loop Requirements + +**IMPORTANT:** This skill performs operations that affect VM data and storage. You MUST: + +1. **Before Creating Snapshots** + - Analyze storage backend capabilities + - Verify VM exists and get current state + - Check for hot-plugged volumes (blocks snapshots) + - Show storage consumption estimate + - Present snapshot configuration + - Ask: "Proceed with snapshot creation? (yes/no)" + - Wait for explicit "yes" + +2. **Never Auto-Execute** + - **NEVER create without user confirmation** + - **NEVER skip storage analysis** + - **NEVER skip hot-plugged volume check** + +**Why This Matters:** +- **Storage Consumption**: Snapshots consume storage proportional to VM disk size +- **Hot-Plugged Volumes**: Cannot snapshot VMs with hot-plugged volumes +- **Consistency**: Online vs offline snapshots have different consistency guarantees +- **Guest Agent**: Required for application-consistent snapshots + +## Common Issues + +### Issue 1: Snapshot Creation Fails - VolumeSnapshotClass Not Found + +**Error**: "VolumeSnapshotClass not found" or "CSI driver doesn't support snapshots" + +**Cause**: The storage backend doesn't have a VolumeSnapshotClass configured for the CSI driver, or the CSI driver doesn't support snapshots at all. + +**Solution:** +1. **Check if VolumeSnapshotClass exists**: Use `resources_list` with apiVersion="snapshot.storage.k8s.io/v1", kind="VolumeSnapshotClass" +2. **Verify CSI driver supports snapshots**: Check StorageClass provisioner field +3. **Contact cluster admin**: Request VolumeSnapshotClass configuration for your storage backend +4. **Alternative**: Use `vm-clone` skill for VM backup instead of snapshots + +### Issue 2: Snapshot Creation Blocked - Hot-Plugged Volumes Detected + +**Error**: "Cannot create snapshot - VM has hot-plugged volumes" + +**Cause**: The VM has volumes that were attached after VM creation without restarting the VM. Hot-plugged volumes block snapshot creation in OpenShift Virtualization. + +**Solution:** +1. **Stop the VM**: Use vm-lifecycle-manager skill to stop the VM +2. **Remove hot-plugged volumes**: Detach volumes that aren't needed +3. **Persist volumes to VM spec**: Add hot-plugged volumes to `spec.template.spec.volumes` to make them permanent +4. **Restart the VM**: Start the VM to apply the changes +5. **Retry snapshot**: Once hot-plugged volumes are resolved, create the snapshot + +**Related**: See [OpenShift Virtualization documentation](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt-hot-plugging-virtual-disks) for hot-plugging details + +### Issue 3: Snapshot Created but Consistency Warning + +**Error**: Snapshot created successfully but shows "crash-consistent" without guest agent + +**Cause**: The VM doesn't have QEMU guest agent installed or running, so the snapshot couldn't coordinate filesystem freeze/thaw during creation. + +**Solution:** +1. **For Linux VMs**: Install qemu-guest-agent package + ```bash + # RHEL/CentOS/Fedora + sudo dnf install qemu-guest-agent + sudo systemctl enable --now qemu-guest-agent + ``` +2. **For Windows VMs**: Install VirtIO drivers which include the guest agent +3. **Verify agent status**: Check VM status for `AgentConnected: True` condition +4. **Future snapshots**: Once guest agent is running, subsequent snapshots will be application-consistent +5. **Current snapshot**: The crash-consistent snapshot is still usable, but may have minor inconsistencies + +## Security Considerations + +- **RBAC Enforcement**: Requires permissions for VirtualMachineSnapshot resources +- **Storage Quotas**: Respects namespace storage quotas +- **Hot-Plugged Volume Detection**: Prevents snapshots when hot-plugged volumes present +- **KUBECONFIG Security**: Credentials never exposed in output +- **Namespace Isolation**: Snapshots scoped to namespace boundaries +- **Audit Trail**: All snapshot operations logged in Kubernetes API audit logs + +## Example Usage + +### Example 1: Create Snapshot Before Upgrade + +``` +User: "Create a snapshot of database-01 before I upgrade it" + +Agent: "Which namespace is database-01 in?" + +User: "production" + +Agent: [Invokes vm-snapshot-create skill] + [Step 2: Verifies VM exists] + [Step 3: Analyzes storage backend - AWS EBS, online snapshots supported] + [Step 4: Checks VM status - Running, guest agent active] + [Step 6: Estimates storage - 50Gi] + [Step 7: Presents configuration] + +## VM Snapshot - Review Configuration + +**VM to Snapshot:** +- **Name**: `database-01` +- **Namespace**: `production` +- **Status**: Running +- **Storage**: 50Gi + +**Storage Backend Analysis:** +- **Storage Class**: `gp3-csi` +- **CSI Driver**: `ebs.csi.aws.com` +- **Online Snapshots**: ✅ Supported +- **Consistency Level**: `crash-consistent` +- **VolumeSnapshotClass**: `ebs-snapclass` ✓ + +**Guest Agent Status:** +- ✅ **QEMU Guest Agent**: Connected and running +- ✅ **Filesystem Freeze/Thaw**: Will be performed during snapshot +- ✅ **Expected Consistency**: Application-consistent (with freeze) + +**Snapshot Details:** +- **Snapshot Name**: `database-01-snapshot-20260218-143022` (auto-generated) +- **Type**: Online snapshot +- **Estimated Storage**: ~50Gi (initial snapshot) + +**Proceed with snapshot creation? (yes/no)** + +User: "yes" + +Agent: [Step 8: Creates snapshot] + +📸 Creating VM snapshot... +✓ Snapshot `database-01-snapshot-20260218-143022` created for VM `database-01` + + [Step 9: Monitors status] + +⏳ Snapshot creation in progress... + Status: InProgress + +⏳ Snapshot creation in progress... + Status: Succeeded + + [Step 10: Reports results] + +## ✓ VM Snapshot Created Successfully + +**VM**: `database-01` (namespace: `production`) +**Snapshot**: `database-01-snapshot-20260218-143022` + +### Snapshot Details +- **Name**: `database-01-snapshot-20260218-143022` +- **Status**: Ready +- **Created**: 2026-02-18 14:30:22 UTC +- **VM Status at Snapshot**: Running + +### Snapshot Coordination (from status.indications) +- ✅ **Guest Agent Coordination**: Active +- ✅ **Filesystem Freeze/Thaw**: Performed during snapshot +- ✅ **Actual Consistency**: Application-consistent + +### Next Steps + +**To list all snapshots:** +"List snapshots for VM database-01" + +**To restore from this snapshot:** +"Restore VM database-01 from snapshot database-01-snapshot-20260218-143022" + +You can now safely upgrade the database. If the upgrade fails, restore using the command above. +``` diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/instruction.md b/evaluation/with_skills/rh-virt__vm-snapshot-create/instruction.md new file mode 100644 index 00000000..34f38f23 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-create/instruction.md @@ -0,0 +1,12 @@ +# VM Snapshot Creation Task + +You are an OpenShift Virtualization administrator. Create a snapshot of VM `production-db` in namespace `prod-vms`. + +## Requirements +- Verify snapshot prerequisites (storage support, guest agent) +- Define the snapshot specification +- Address snapshot consistency levels and monitoring + +Use MCP tools to examine the cluster. Work autonomously — do not wait for user confirmation at any step. Document your methodology, findings, and snapshot plan in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/solution/solve.sh b/evaluation/with_skills/rh-virt__vm-snapshot-create/solution/solve.sh new file mode 100644 index 00000000..22659dde --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-create/solution/solve.sh @@ -0,0 +1,39 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Snapshot Plan + +## Target: production-db in prod-vms + +### Storage Snapshot Support Checks +1. Check VM `status.volumeSnapshotStatuses` for snapshot support +2. Verify no hot-plugged volumes (block snapshots - must stop VM and persist or remove) +3. Check StorageClass has a VolumeSnapshotClass +4. Verify CSI driver supports snapshots +5. Check for guest agent (determines consistency level) +6. Create via resources_create_or_update; poll status.phase (InProgress/Succeeded/Failed) and status.readyToUse + +### Snapshot Type +- **With guest agent**: Application-consistent (freeze/thaw of filesystem) + - `status.indications` will show `GuestAgent` +- **Without guest agent**: Crash-consistent (point-in-time disk state) + - `status.indications` will show `Online` only + +### VirtualMachineSnapshot YAML +```yaml +apiVersion: snapshot.kubevirt.io/v1beta1 +kind: VirtualMachineSnapshot +metadata: + name: production-db-backup-20240301 + namespace: prod-vms +spec: + source: + apiGroup: kubevirt.io + kind: VirtualMachine + name: production-db +``` + +### Monitoring +- Poll `status.phase`: InProgress → Succeeded or Failed +- Check `status.readyToUse: true` before relying on snapshot + +REPORT_EOF diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/task.toml b/evaluation/with_skills/rh-virt__vm-snapshot-create/task.toml new file mode 100644 index 00000000..c563a3ed --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-create/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-snapshot-create" +name = "rh-virt VM Snapshot Creation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-snapshot-create", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/tests/llm_judge.py b/evaluation/with_skills/rh-virt__vm-snapshot-create/tests/llm_judge.py new file mode 100644 index 00000000..cf067a9c --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-create/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "volume_snapshot_class", "file": "/root/report.md", "question": "Does the report check for VolumeSnapshotClass as a prerequisite for CSI snapshot support?", "reference": "A skilled report verifies VolumeSnapshotClass exists. An unskilled report attempts snapshots without checking prerequisites."}, + {"id": "hot_plugged_blocker", "file": "/root/report.md", "question": "Does the report note that hot-plugged volumes block snapshot creation entirely?", "reference": "A skilled report checks for hot-plugged volumes. An unskilled report doesn't know about this blocker."}, + {"id": "consistency_levels", "file": "/root/report.md", "question": "Does the report distinguish application-consistent (GuestAgent) from crash-consistent (Online only) snapshots?", "reference": "A skilled report checks status.indications for GuestAgent presence. An unskilled report doesn't distinguish consistency levels."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/tests/test.sh b/evaluation/with_skills/rh-virt__vm-snapshot-create/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-create/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/tests/test_outputs.py b/evaluation/with_skills/rh-virt__vm-snapshot-create/tests/test_outputs.py new file mode 100644 index 00000000..c4189fb6 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-create/tests/test_outputs.py @@ -0,0 +1,77 @@ +""" +Tests for rh-virt__vm-snapshot-create per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_snapshot(self): + content = read_report().lower() + assert "snapshot" in content, "report should mention snapshots" + + def test_mentions_vm(self): + content = read_report().lower() + assert any(t in content for t in ["vm", "virtual machine", "virtualmachine"]), ( + "report should reference the target VM" + ) + + +class TestSkillDependent: + def test_volume_snapshot_class(self): + """Skill: VolumeSnapshotClass prerequisite for CSI snapshot support.""" + c = read_report().lower() + assert any(t in c for t in ["volumesnapshotclass", "volume snapshot class", "snapshot class", "csi driver"]), ( + "should mention VolumeSnapshotClass for snapshot prerequisites" + ) + + def test_quiesce_consistency(self): + """Skill: Quiesce/freeze for application-consistent snapshots; guest agent.""" + c = read_report().lower() + assert any(t in c for t in ["quiesce", "freeze", "thaw", "guest agent", "application-consistent", "qemu-guest-agent"]), ( + "should discuss quiesce/freeze for consistency" + ) + + def test_snapshot_cr_structure(self): + """Skill: VirtualMachineSnapshot CR with spec.source.""" + c = read_report().lower() + assert "virtualmachinesnapshot" in c and any(t in c for t in ["spec", "source", "snapshot.kubevirt", "apiversion"]), ( + "should define VirtualMachineSnapshot resource structure" + ) + + def test_hot_plugged_blocker(self): + """Skill: Hot-plugged volumes block snapshot creation.""" + c = read_report().lower() + assert any(t in c for t in ["hot-plug", "hotplug", "hot plug", "block snapshot", "cannot snapshot"]), ( + "should address hot-plugged volumes blocking snapshots" + ) + + def test_status_indications(self): + """Skill: status.indications (GuestAgent, Online) for consistency level.""" + c = read_report().lower() + assert any(t in c for t in ["indications", "guestagent", "online", "status.phase", "inprogress", "succeeded"]), ( + "should reference snapshot status/indications" + ) + + def test_guest_agent_connected_check(self): + """Docs teach checking AgentConnected condition to determine if + application-consistent (vs crash-consistent) snapshots are possible. + Without docs, agents don't check guest agent status before snapshot.""" + c = read_report().lower() + assert any(t in c for t in [ + "agentconnected", "agent connected", "guest agent", + "application-consistent", "crash-consistent", + ]), "should check AgentConnected for snapshot consistency level" diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/Dockerfile b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/Dockerfile new file mode 100644 index 00000000..ae625e01 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/.ai-index/semantic-index.json new file mode 100644 index 00000000..2512c2a3 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/.ai-index/semantic-index.json @@ -0,0 +1,148 @@ +{ + "index_version": "1.0.0", + "last_updated": "2026-02-17", + "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", + + "documents": [ + { + "file": "INDEX.md", + "title": "VM Troubleshooting Guide - Index", + "category": "navigation", + "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], + "use_cases": ["navigation", "tool-reference", "general-guidance"], + "token_estimate": 450, + "skills": ["all"] + }, + { + "file": "scheduling-errors.md", + "title": "VM Scheduling Errors", + "category": "scheduling", + "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], + "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], + "errors_covered": [ + "ErrorUnschedulable - Node Taints", + "ErrorUnschedulable - Insufficient Resources", + "ErrorUnschedulable - Node Selector Mismatch" + ], + "token_estimate": 820, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "storage-errors.md", + "title": "VM Storage Errors", + "category": "storage", + "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], + "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], + "errors_covered": [ + "ErrorDataVolumeNotReady - DataVolume Still Provisioning", + "ErrorDataVolumeNotReady - Storage Class Not Found", + "ErrorDataVolumeNotReady - Insufficient Storage Quota", + "ErrorPvcNotFound", + "Storage Deletion Failures", + "DataVolume Cloning Failures" + ], + "token_estimate": 1350, + "skills": ["vm-creator", "vm-delete", "vm-clone"] + }, + { + "file": "lifecycle-errors.md", + "title": "VM Lifecycle Errors", + "category": "lifecycle", + "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], + "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], + "errors_covered": [ + "VM Stuck in Terminating State", + "VM Won't Start (Non-Scheduling Issues)", + "VM Won't Stop" + ], + "token_estimate": 1150, + "skills": ["vm-delete", "vm-lifecycle-manager"] + }, + { + "file": "runtime-errors.md", + "title": "VM Runtime Errors", + "category": "runtime", + "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], + "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], + "errors_covered": [ + "CrashLoopBackOff" + ], + "token_estimate": 850, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "network-errors.md", + "title": "VM Network Errors", + "category": "networking", + "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], + "use_cases": ["vm-creation", "network-troubleshooting"], + "errors_covered": [ + "Network Attachment Failures" + ], + "token_estimate": 580, + "skills": ["vm-creator"] + } + ], + + "skill_to_docs_mapping": { + "vm-creator": { + "primary_docs": ["scheduling-errors.md", "storage-errors.md"], + "secondary_docs": ["runtime-errors.md", "network-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3650, + "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" + }, + "vm-delete": { + "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 2950, + "description": "VM deletion failures - stuck terminating, storage cleanup" + }, + "vm-clone": { + "primary_docs": ["storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 1800, + "description": "VM cloning failures - DataVolume cloning, storage provisioning" + }, + "vm-lifecycle-manager": { + "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], + "secondary_docs": ["runtime-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3270, + "description": "VM start/stop failures - lifecycle management, scheduling" + }, + "vm-inventory": { + "primary_docs": [], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 450, + "description": "General guidance for status interpretation" + } + }, + + "error_to_docs_mapping": { + "ErrorUnschedulable": "scheduling-errors.md", + "ErrorDataVolumeNotReady": "storage-errors.md", + "ErrorPvcNotFound": "storage-errors.md", + "Terminating": "lifecycle-errors.md", + "CrashLoopBackOff": "runtime-errors.md", + "NetworkAttachmentFailed": "network-errors.md" + }, + + "cross_references": { + "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], + "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], + "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], + "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], + "network-errors.md": ["scheduling-errors.md"] + }, + + "progressive_disclosure_strategy": { + "description": "Load only the minimum documentation needed for current task", + "pattern": "error_type → primary_doc → cross_references (if needed)", + "token_savings": "29% average reduction vs single-file (based on rh-sre data)", + "navigation_overhead_reduction": "85% vs single-file navigation" + } +} diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/INDEX.md new file mode 100644 index 00000000..a2e76963 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/INDEX.md @@ -0,0 +1,332 @@ +--- +title: VM Troubleshooting Guide - Index +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 + - title: OpenShift Virtualization - Virtual Machine Status + url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html + date_accessed: 2026-02-06 + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] +semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] +use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] +last_updated: 2026-02-17 +--- + +# VM Troubleshooting Guide - Index + +## Overview + +This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: + +- **Creation**: Scheduling failures, storage provisioning issues +- **Lifecycle**: Start/stop failures, stuck states +- **Deletion**: Resources stuck in Terminating, storage cleanup failures +- **Cloning**: DataVolume cloning errors, cross-namespace issues +- **Networking**: Secondary network attachment failures +- **Runtime**: CrashLoopBackOff, guest OS boot failures + +This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. + +--- + +## 🗂️ Troubleshooting Categories + +The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: + +### 1. [Scheduling Errors](scheduling-errors.md) +**When to use**: VM fails to schedule on any node + +**Errors covered**: +- ErrorUnschedulable - Node Taints +- ErrorUnschedulable - Insufficient Resources +- ErrorUnschedulable - Node Selector Mismatch + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 2. [Storage Errors](storage-errors.md) +**When to use**: VM has storage provisioning, deletion, or cloning issues + +**Errors covered**: +- ErrorDataVolumeNotReady (all 3 subsections) +- ErrorPvcNotFound +- Storage Deletion Failures +- DataVolume Cloning Failures + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### 3. [Lifecycle Errors](lifecycle-errors.md) +**When to use**: VM has start/stop/termination issues + +**Errors covered**: +- VM Stuck in Terminating State +- VM Won't Start (Non-Scheduling Issues) +- VM Won't Stop + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### 4. [Runtime Errors](runtime-errors.md) +**When to use**: VM repeatedly crashes or fails at runtime + +**Errors covered**: +- CrashLoopBackOff + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 5. [Network Errors](network-errors.md) +**When to use**: VM has secondary network attachment failures + +**Errors covered**: +- Network Attachment Failures (Multus, SR-IOV) + +**Skills that use this**: vm-create + +--- + +## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last + +### Critical Principle: MCP-First Approach + +**All diagnostic procedures in this guide follow the MCP-first pattern**: + +``` +1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) +2. ⚠️ IF FAILS: Ask user permission to use CLI command +3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval +``` + +### Why MCP Tools First? + +- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources +- **Consistency**: Same interface across all operations +- **Better Error Handling**: MCP tools return structured errors +- **Audit Trail**: MCP tool usage is logged and trackable + +### Command Pattern Examples + +Throughout this guide, you'll see diagnostic steps formatted like this: + +**✅ CORRECT Pattern (MCP First)**: + +```markdown +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Extract**: `.metadata.finalizers` from returned JSON + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` +``` + +### Available MCP Tools + +The `openshift-virtualization` MCP server provides these tools: + +**Resource Operations**: +- `resources_get` - Get specific resource (replaces `oc get `) +- `resources_list` - List resources (replaces `oc get `) +- `resources_delete` - Delete resource (replaces `oc delete`) +- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) + +**Pod Operations**: +- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) +- `pods_get` - Get pod details (replaces `oc get pod`) +- `pods_log` - Get pod logs (replaces `oc logs`) +- `pods_exec` - Execute in pod (replaces `oc exec`) +- `pods_delete` - Delete pod (replaces `oc delete pod`) +- `pods_top` - Pod resource usage (replaces `oc top pods`) + +**Events & Monitoring**: +- `events_list` - List events (replaces `oc get events`) +- `nodes_top` - Node resource usage (replaces `oc top nodes`) +- `nodes_log` - Node logs (replaces `oc adm node-logs`) +- `nodes_stats_summary` - Detailed node stats + +**VM Operations** (KubeVirt toolset): +- `vm_create` - Create VMs +- `vm_lifecycle` - Start/stop/restart VMs + +**Namespaces**: +- `namespaces_list` - List namespaces (replaces `oc get namespaces`) +- `projects_list` - List OpenShift projects (replaces `oc get projects`) + +### When CLI Commands Are Required + +Some operations have **NO MCP equivalent** and require CLI: + +- `virtctl` commands (console, VNC access) +- `oc debug node` (node debugging) +- `oc auth can-i` (permission checks) +- `oc adm taint` (node taint management) +- Complex JSON patch operations + +For these, the guide will note: **"CLI Only - No MCP equivalent"** + +### Quick Reference: CLI → MCP Mapping + +| CLI Command | MCP Tool Equivalent | +|-------------|---------------------| +| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | +| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | +| `oc logs -n ` | `pods_log` with name="", namespace="" | +| `oc get events -n ` | `events_list` with namespace="" | +| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | + +**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. + +### How to Read Diagnostic Sections + +Each error section includes: +1. **Symptom** - What you observe +2. **Description** - What's happening +3. **Possible Causes** - Why it's happening +4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback +5. **Solutions** - **MCP tools first**, then CLI fallback +6. **Verification** - **MCP tools first**, then CLI fallback + +**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. + +--- + +## 🔍 Quick Navigation by Skill + +**vm-create**: +- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics +- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning +- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff +- [Network Errors](network-errors.md) - Network attachment failures + +**vm-delete**: +- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state +- [Storage Errors](storage-errors.md) - Storage deletion failures + +**vm-clone**: +- [Storage Errors](storage-errors.md) - DataVolume cloning failures + +**vm-lifecycle-manager**: +- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop +- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling + +**vm-inventory**: +- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed + +--- + +## 📊 Documentation Coverage & Maintenance + +### Current Coverage + +This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: + +- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) +- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures +- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop +- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) +- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems + +**Total errors documented**: 12 error types across 6 categories + +--- + +### Encountering Undocumented Errors + +#### For AI Agents (Claude Code) + +If you encounter an error **not documented** in the categories above: + +1. **Report to user** with all available details (error message, affected resources, namespace) +2. **Provide best-effort diagnostics** using MCP tools: + - `resources_get` to inspect resource status + - `pods_log` to check virt-launcher or compute container logs + - `events_list` to view Kubernetes events +3. **Suggest documentation update**: + ``` + ⚠️ This error is not yet documented in the troubleshooting guide. + + **Error**: + **Affected resource**: / + + I recommend adding this error to the troubleshooting guide: + - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file + - If it's a new error class → Create new category file in docs/troubleshooting/ + + Would you like me to help document this error for future reference? + ``` + +4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs + +--- + +#### For Human Contributors + +To document a new error: + +1. **Determine the category**: + - Scheduling issue → `scheduling-errors.md` + - Storage problem → `storage-errors.md` + - Lifecycle/start/stop → `lifecycle-errors.md` + - Runtime crash → `runtime-errors.md` + - Network issue → `network-errors.md` + - New error class → Create new category file (e.g., `security-errors.md`) + +2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): + - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback + - **Solutions**: MCP Tool approach → Implementation → CLI Fallback + - **Verification**: MCP Tool checks → Expected results → CLI Fallback + +3. **Update semantic index**: + - Edit `.ai-index/semantic-index.json` + - Add error to `error_to_docs_mapping` + - Update relevant skill's `primary_docs` or `secondary_docs` + - Increment token estimates if needed + +4. **Reference in skill files**: + - Update skill's Reference Documentation section + - Ensure skill's Document Consultation steps point to the right category + +5. **Update this INDEX.md**: + - Add error to the appropriate category section (lines 50-120) + - Update "Current Coverage" list above + +--- + +## 📚 Additional Resources + +- [rh-virt README](../../README.md) - MCP server setup and skill documentation +- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources +- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/lifecycle-errors.md new file mode 100644 index 00000000..6167fcc8 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/lifecycle-errors.md @@ -0,0 +1,869 @@ +--- +title: VM Lifecycle Errors +category: kubevirt +sources: + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 +tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] +semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] +use_cases: [vm-deletion, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Lifecycle Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM lifecycle issues including start/stop failures and stuck termination states. + +**When to use this document**: +- VM stuck in `Terminating` state +- VM won't start (runStrategy is Always but VM never reaches Running) +- VM won't stop (runStrategy is Halted but VM never reaches Stopped) + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### VM Stuck in Terminating State + +**Symptom**: VM shows status `Terminating` but deletion never completes + +**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). + +**Possible Causes**: +- Finalizers blocking deletion +- PVC/DataVolume still attached and preventing cleanup +- VirtualMachineInstance (VMI) not terminating properly +- Custom controllers or operators blocking deletion +- Stuck virt-launcher pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check finalizers on the VM**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` + +**2. Check if VMI still exists**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name containing "virt-launcher-". + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check events for deletion issues**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results where `involvedObject.name` == "" and sort by timestamp. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' +``` + +**5. Check VM deletion timestamp**: + +Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. + +If present, VM is in deletion process. If null, VM is not being deleted. + +**Common Finalizer Patterns**: +- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer +- `foregroundDeletion` - Waits for dependent objects to be deleted +- Custom finalizers from operators + +**Solutions** (Use MCP Tools First): + +1. **Wait for dependent resources** (safest approach): + - Use `resources_list` to check if PVCs, DataVolumes are still deleting + - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) + +2. **Force delete VMI** (if VMI is stuck): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable - requires explicit user permission): + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +3. **Force delete virt-launcher pod** (if pod is stuck): + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + # Find the virt-launcher pod name first + oc get pods -n | grep virt-launcher- + + # Then delete it + oc delete pod virt-launcher--xxx -n --force --grace-period=0 + ``` + +4. **Remove finalizers** (⚠️ dangerous - use only as last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` + 2. Remove items from `.metadata.finalizers` array + 3. Update VM using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch not easily done via MCP): + ```bash + # This operation is complex for MCP - may need CLI + oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Check for protection labels** (vm-delete skill specific): + + Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. + + If value is "true", the vm-delete skill refuses deletion (this is expected behavior). + +**Verification**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error if deletion successful. + +**CLI Fallback**: +```bash +oc get vm -n +# Should return: Error from server (NotFound) +``` + +--- + + +--- + +### VM Won't Start (Non-Scheduling Issues) + +**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state + +**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. + +**Possible Causes**: +- Guest OS kernel panic or boot failure +- Cloud-init configuration errors +- Missing or corrupted disk image +- Insufficient memory for guest OS to boot +- QEMU/KVM virtualization errors +- VirtualMachineInstance (VMI) creation failures +- virt-launcher pod crashes + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VMI (VirtualMachineInstance) status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and overall VMI state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**2. Check VMI conditions for errors**: + +Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. View virt-launcher pod logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +First, get pod name from step 3, then: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +Look for QEMU errors, memory allocation failures, device errors. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**5. Check virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. + +Alternatively, use `pods_get` to get full pod details: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**6. Access VM console to see guest OS boot messages**: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +Look for kernel panic, initramfs errors, filesystem errors. + +**7. Check VM events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep -A 20 "Events:" +``` + +**Common Error Patterns**: + +1. **Guest OS Boot Failure**: + - Console shows kernel panic + - Guest hangs at GRUB or boot loader + - Cloud-init errors during first boot + +2. **Insufficient Memory**: + - Guest OS kills processes due to OOM + - VMI logs show memory allocation errors + +3. **QEMU Crashes**: + - virt-launcher logs show QEMU segfaults + - VMI repeatedly restarts + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for kernel panic, initramfs errors, filesystem errors. + +2. **Check virt-launcher pod logs for QEMU errors**: + + Use `pods_log` from diagnostic step 4 to view logs. + + Look for: + - "qemu-system-x86_64: ..." errors + - Memory allocation failures + - Device errors + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx + ``` + +3. **Increase memory if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check VMI memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +4. **Verify disk image integrity**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check PVC is bound: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + If using container disk, verify image exists and is accessible (check VMI spec). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get pvc -n + ``` + +5. **Check cloud-init configuration** (if applicable): + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. + + Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq + ``` + +6. **Restart VMI** (soft reset): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete VMI (VM controller will recreate it): + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for new VMI to start (use `resources_get` to check status). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n -w + ``` + +7. **Check virtualization extensions** (KVM): + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + + # In debug shell: + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should eventually return `Running`). + +Check VMI is running: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should show `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should eventually return: Running + +oc get vmi -n +# Should show: Running +``` + +--- + +### VM Won't Stop + +**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state + +**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. + +**Possible Causes**: +- Guest OS not responding to ACPI shutdown signal +- virt-launcher pod stuck and not terminating +- VirtualMachineInstance (VMI) deletion blocked by finalizers +- Guest shutdown scripts hanging +- Filesystem sync issues in guest OS + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (might show `Stopping`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**2. Check VMI status and deletion timestamp**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check VMI events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vmi -n | grep -A 10 "Events:" +``` + +**5. Check VMI finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.metadata.finalizers}' +``` + +**6. Check if guest is responsive** (if VMI still exists): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Patterns**: +- VMI shows `deletionTimestamp` but never actually deletes +- virt-launcher pod in `Terminating` state +- VM runStrategy is `Halted` but printableStatus shows `Stopping` + +**Solutions** (Use MCP Tools First): + +1. **Wait for graceful shutdown** (default: 30 seconds): + + Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. + +2. **Force stop by deleting VMI**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + This is the standard way to force-stop a VM. + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n + # Should return: Error from server (NotFound) + ``` + +3. **Force delete VMI with grace period** (if VMI won't delete): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +4. **Force delete virt-launcher pod**: + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find the pod using `pods_list_in_namespace` from diagnostic step 3. + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + ⚠️ **Note**: For force deletion with grace period, use CLI fallback. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') + oc delete pod $POD_NAME -n --force --grace-period=0 + ``` + +5. **Remove VMI finalizers** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VMI using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update VMI using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Patch VM runStrategy directly** (ensure consistency): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` (diagnostic step 1) + 2. Set `.spec.runStrategy` to `"Halted"` + 3. Update VM using `resources_create_or_update` with modified JSON + + **CLI Fallback** (merge patch easier via CLI): + Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Stopped` or `Halted`). + +Verify VMI is gone: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Verify virt-launcher pod is gone: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for pods matching `virt-launcher-`. Should return no results. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Stopped or Halted + +oc get vmi -n +# Should return: Error from server (NotFound) + +oc get pods -n | grep virt-launcher- +# Should return: No resources found +``` + +**Prevention**: +- Ensure guest OS has ACPI support enabled +- Use proper shutdown commands in guest OS +- Avoid forceful stops unless necessary (can corrupt guest filesystem) + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/network-errors.md new file mode 100644 index 00000000..82fa313c --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/network-errors.md @@ -0,0 +1,429 @@ +--- +title: VM Network Errors +category: kubevirt +sources: + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] +semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] +use_cases: [vm-creation, network-troubleshooting] +related_docs: [INDEX.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Network Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. + +**When to use this document**: +- VM created successfully but secondary networks not attached +- NetworkAttachmentDefinition not found errors +- Multus CNI failures +- SR-IOV device attachment issues + +**Skills that use this**: vm-create + +--- + +### Network Attachment Failures + +**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working + +**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). + +**Possible Causes**: +- NetworkAttachmentDefinition doesn't exist in the namespace +- Multus CNI not installed or not configured on cluster +- Namespace mismatch (NAD in different namespace than VM) +- Interface name conflicts in VM spec +- Bridge/network configuration errors in NAD +- SR-IOV device not available (if using SR-IOV) + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List NetworkAttachmentDefinitions in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "" +} +``` + +Review `.items[].metadata.name` for available NADs. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -n +``` + +**2. List NetworkAttachmentDefinitions in all namespaces**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition" +} +``` + +Omit `namespace` parameter to list across all namespaces. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -A +``` + +**3. Check specific NetworkAttachmentDefinition**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "", + "name": "" +} +``` + +Review `.spec.config` for CNI configuration. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definition -n -o yaml +``` + +**4. Check VM network configuration**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.networks` to see network references. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq +``` + +**5. Check VM domain interfaces**: + +Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq +``` + +**6. Check virt-launcher pod network annotations** (shows actual attachments): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**7. Check Multus is installed**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "openshift-multus" +} +``` + +Should show Multus CNI pods running. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n openshift-multus +``` + +**8. Check for errors in virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n +``` + +**Common Error Messages**: +- `"network-attachment-definition not found"` - NAD doesn't exist in namespace +- `"multus CNI not configured"` - Multus not installed or misconfigured +- `"interface name conflict"` - Duplicate interface names in VM spec +- `"failed to add network"` - CNI plugin error (check NAD config) +- `"no available devices"` - SR-IOV device not available (if using SR-IOV) + +**Solutions** (Use MCP Tools First): + +1. **Verify NetworkAttachmentDefinition exists in correct namespace**: + + Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. + + If NAD is in different namespace, copy it to VM namespace: + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get NAD from source namespace using `resources_get` + 2. Modify `.metadata.namespace` to target namespace + 3. Create NAD in target namespace using `resources_create_or_update` + + **CLI Fallback** (stream processing easier via CLI): + Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" + ```bash + oc get network-attachment-definition -n -o yaml | \ + sed "s/namespace: /namespace: /" | \ + oc apply -f - + ``` + +2. **Create missing NetworkAttachmentDefinition**: + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + Example: Linux bridge network + + **Parameters**: + ```json + { + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": { + "name": "vlan100", + "namespace": "" + }, + "spec": { + "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" + ```bash + cat < + spec: + config: '{ + "cniVersion": "0.3.1", + "type": "bridge", + "bridge": "br1", + "vlan": 100, + "ipam": { + "type": "host-local", + "subnet": "192.168.100.0/24" + } + }' + EOF + ``` + +3. **Check Multus CNI installation**: + + Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. + + To check cluster network operator: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterOperator" + } + ``` + + Filter for `network` operator. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n openshift-multus + oc get clusteroperators network + ``` + +4. **Fix interface name conflicts** (if VM has duplicate names): + + Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. + + Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. + + **CLI Fallback** (interactive edit easier via CLI): + Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' + oc edit vm -n + ``` + +5. **Validate NAD configuration syntax**: + + Use `resources_get` from diagnostic step 3, extract `.spec.config`. + + Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get network-attachment-definition -n -o jsonpath='{.spec.config}' + ``` + +6. **Check SR-IOV device availability** (if using SR-IOV networks): + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + List SR-IOV network node policies: + + **Parameters**: + ```json + { + "apiVersion": "sriovnetwork.openshift.io/v1", + "kind": "SriovNetworkNodePolicy", + "namespace": "openshift-sriov-network-operator" + } + ``` + + Check SR-IOV device plugin pods: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "openshift-sriov-network-operator" + } + ``` + + Filter for pods with "device-plugin" in name. + + Check available SR-IOV devices on node: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "Node", + "name": "" + } + ``` + + Review `.status.allocatable` for SR-IOV resources. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get sriovnetworknodepolicy -n openshift-sriov-network-operator + oc get pods -n openshift-sriov-network-operator | grep device-plugin + oc describe node | grep -A 10 "Allocatable:" + ``` + +7. **Recreate VM with corrected network configuration** (if needed): + + If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +After remediation, check virt-launcher pod network status: + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +Should show all attached networks with status. Example output: +```json +[ + { + "name": "openshift-sdn", + "interface": "eth0", + "ips": ["10.128.2.10"], + "default": true + }, + { + "name": "vlan100", + "interface": "net1", + "ips": ["192.168.100.5"] + } +] +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**Check from inside VM** (via console): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# In guest OS: +ip addr show +# Should show all network interfaces (eth0, net1, etc.) +``` + +**Common Network Types**: +- **Linux Bridge**: Layer 2 bridge for VLAN networks +- **SR-IOV**: High-performance direct device assignment +- **macvlan**: MAC-based VLAN for container networks +- **OVN-Kubernetes**: OpenShift native overlay network + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/runtime-errors.md new file mode 100644 index 00000000..804d3cbc --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/runtime-errors.md @@ -0,0 +1,616 @@ +--- +title: VM Runtime Errors +category: kubevirt +sources: + - title: Kubernetes Pod Lifecycle + url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ + date_accessed: 2026-02-17 +tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] +semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] +use_cases: [vm-creation, vm-lifecycle, diagnostics] +related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Runtime Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. + +**When to use this document**: +- VM or virt-launcher pod shows `CrashLoopBackOff` status +- virt-launcher pod repeatedly restarting +- Guest OS kernel panics on boot + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### CrashLoopBackOff + +**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting + +**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. + +**Possible Causes**: +- Guest OS kernel panic on boot +- Insufficient resources (memory/CPU) for guest OS +- Corrupted disk image or filesystem +- QEMU/libvirt crashes due to configuration errors +- Missing or incompatible device drivers in guest +- Resource limits too low for virt-launcher pod +- Virtualization features (KVM) not available on node + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check virt-launcher pod restart count**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# Look at RESTARTS column +``` + +**2. View recent crash logs** (previous container instance): + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "previous": true, + "tail": 100 +} +``` + +Look for QEMU errors, kernel panics, or segfaults. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n virt-launcher--xxx --previous +``` + +**3. Check current virt-launcher logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. + +**CLI Fallback** (if MCP unavailable or all containers needed): +```bash +oc logs -n virt-launcher--xxx --all-containers +``` + +**4. Check VMI conditions for crash details**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Extract `.status.conditions` for crash details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**5. Check pod events for crash reasons**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" +``` + +**6. Check pod resource limits**: + +**MCP Tool**: `pods_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +Extract `.spec.containers[0].resources` for resource limits. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq +``` + +**7. Check node kubelet logs for OOM kills**: + +⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +oc adm node-logs -u kubelet | grep -i oom +``` + +**8. Access guest console** (if VM briefly starts): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Crash Patterns**: + +1. **Guest Kernel Panic**: + - Console logs show kernel panic messages + - Guest crashes immediately after boot + - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" + +2. **OOM (Out of Memory)**: + - Pod killed with reason: `OOMKilled` + - Guest runs out of memory during boot or operation + - virt-launcher logs show memory allocation failures + +3. **QEMU Crash**: + - virt-launcher logs show QEMU segmentation fault + - Symptoms: "qemu-system-x86_64: terminated by signal" + - Configuration incompatibility or QEMU bug + +4. **Disk Image Corruption**: + - Guest cannot boot from disk + - Filesystem errors in guest console + - DataVolume import failed + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for kernel panic or boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for: + - Kernel panic messages + - Initramfs errors + - Filesystem mounting failures + - Missing device errors + +2. **Review virt-launcher crash logs**: + + Use `pods_log` with `previous: true` from diagnostic step 2. + + Look for: + - QEMU command line errors + - Device initialization failures + - Memory allocation errors + - Signal termination (SIGSEGV, SIGABRT) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx --previous + ``` + +3. **Check for OOM (Out of Memory) kills**: + + **MCP Tool**: `pods_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + Extract `.status.containerStatuses[0].lastState.terminated.reason`. + + If returns `"OOMKilled"`: + - Option 1: Increase virt-launcher memory limits + - Option 2: Decrease guest memory allocation + - Option 3: Use smaller instance type + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' + ``` + +4. **Increase resources if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check current memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +5. **Verify disk image integrity**: + + **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters for list**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" + } + ``` + + **Parameters for specific DV**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check `.status.phase` (should be `Succeeded`). + + If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get datavolume -n -o jsonpath='{.status.phase}' + ``` + +6. **Check virtualization (KVM) availability**: + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +7. **Simplify VM configuration** (eliminate variables): + + Try creating minimal VM using vm-create skill with: + - Small instance type + - No secondary networks + - Simple container disk (e.g., Fedora) + - No cloud-init + + If minimal VM works, add features back one by one. + +8. **Recreate VM with different workload** (test disk image): + + If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) + +After remediation, check pod restart count stops increasing: + +**Parameters for pods**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). + +Check VM reaches Running state: + +**Parameters for VM**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# RESTARTS should stabilize + +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Running +``` + +Verify guest is responsive: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# Should show login prompt or OS console +``` + +**Advanced Debugging**: + +**MCP Tool**: `pods_exec` (from openshift-virtualization) + +Check libvirt domain XML: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["virsh", "dumpxml", "1"] +} +``` + +Check QEMU process: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["ps", "aux"] +} +``` + +Filter output for "qemu" process. + +**CLI Fallback** (if MCP unavailable): +```bash +oc exec -n virt-launcher--xxx -- virsh dumpxml 1 +oc exec -n virt-launcher--xxx -- ps aux | grep qemu +``` + +**Prevention**: +- Start with minimal VM configuration and add complexity gradually +- Use recommended instance types for your workload +- Test disk images before deploying to production +- Ensure nodes have adequate resources and KVM support +- Monitor resource usage and set appropriate limits + +--- + +## Workaround Patterns for MCP Tool Limitations + +### General Pattern: Diagnose → Propose → Confirm → Execute + +When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): + +**1. Diagnose** the root cause using Kubernetes commands +```bash +oc describe vm -n +oc get events -n --field-selector involvedObject.name= +``` + +**2. Propose** a manual workaround to the user with clear explanation +- Explain why the issue occurred +- Show what will be changed +- List alternative options + +**3. Confirm** with user before executing (Human-in-the-Loop) +- Wait for explicit "yes" or "apply workaround" +- Never auto-execute modifications + +**4. Execute** the workaround using oc/kubectl +```bash +oc patch vm -n --type=merge -p '...' +``` + +**5. Verify** the fix was successful +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**6. Document** the limitation and suggest filing enhancement request +- Note this is temporary until MCP tool is enhanced +- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues + +### Example: Adding Tolerations Workaround + +**Diagnostic Output**: +```markdown +## ⚠️ VM Scheduling Issue Detected + +**Root Cause**: Node taints prevent VM scheduling + +**Details**: +- Found 3 nodes with taint: `virtualization=true:NoSchedule` +- VM spec does not include matching tolerations +- This prevents VM scheduling on virtualization-dedicated nodes +``` + +**Proposed Workaround**: +```bash +oc patch vm web-server -n vms --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' +``` + +**User Confirmation Required**: +``` +How would you like to proceed? +- "apply workaround" - I'll patch the VM with tolerations +- "manual" - I'll provide instructions for you to apply manually +- "cancel" - Delete the VM and abort creation +``` + +**After Execution**: +```markdown +## ✓ Workaround Applied Successfully + +**Action**: Added tolerations for taint `virtualization=true:NoSchedule` +**New Status**: Stopped (VM can now be scheduled) + +**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. +Future VMs in this cluster will need the same fix until the tool is enhanced. +``` + +--- + +## VM Status Reference + +### Status Values + +| Status | Meaning | Action Required | +|--------|---------|-----------------| +| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | +| `Running` | VM is running | Normal | +| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | +| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | +| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | +| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | +| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | +| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | +| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | +| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | + +### Checking VM Status + +```bash +# Get printable status +oc get vm -n -o jsonpath='{.status.printableStatus}' + +# Get detailed status and conditions +oc get vm -n -o jsonpath='{.status}' | jq + +# Watch status changes in real-time +oc get vm -n -w +``` + +--- + +## Best Practices for Agents + +When implementing diagnostic workflows: + +1. **Always verify VM status** after creation (wait 5-10 seconds first) +2. **Consult this document** when encountering error status values +3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) +4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) +5. **Respect human-in-the-loop** for all VM modifications +6. **Document temporary workarounds** and their limitations clearly +7. **Suggest filing issues** for missing MCP tool features + +### Document Consultation Pattern + +```markdown +**Document Consultation** (REQUIRED): +1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes +2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." +``` + +--- + +## Known MCP Tool Limitations + +### vm_create tool + +**Currently Supported**: +- ✓ Namespace, name (required) +- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) +- ✓ Size hints (small, medium, large) +- ✓ Storage size +- ✓ Autostart flag +- ✓ Networks (Multus NetworkAttachmentDefinitions) +- ✓ Performance family (u1, o1, c1, m1) +- ✓ Instance type, preference + +**Not Currently Supported** (requires workarounds): +- ✗ Tolerations (for node taints) +- ✗ Node selectors +- ✗ Affinity/anti-affinity rules +- ✗ Resource requests/limits (beyond instance type) +- ✗ Custom labels/annotations +- ✗ SSH keys injection +- ✗ Cloud-init user data + +**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. + +**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues + +--- + +## Additional Resources + +- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) +- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) +- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) +- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) + +--- + +[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/scheduling-errors.md new file mode 100644 index 00000000..e47c8c4f --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/scheduling-errors.md @@ -0,0 +1,417 @@ +--- +title: VM Scheduling Errors +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 +tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] +semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] +use_cases: [vm-creation, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Scheduling Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**When to use this document**: +- VM shows status `ErrorUnschedulable` after creation or start attempt +- VM events mention scheduling failures, taints, resources, or node selectors + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +## ErrorUnschedulable + +**Symptom**: VM shows status `ErrorUnschedulable` after creation + +**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**Possible Causes**: + +### 1. Node Taints (Most Common) + +Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM events for scheduling failures**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `involvedObject.name` == "" and look for messages like: +- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= +``` + +**2. Check node taints in the cluster**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' +``` + +**Common Taint Patterns**: +- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule +- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes +- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads + +**Solution - Add Tolerations to VM**: + +The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. + +**Workaround (post-creation using MCP Tools)**: + +**Step 1**: Get current VM spec + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Step 2**: Modify the returned JSON to add tolerations + +Add to `.spec.template.spec.tolerations`: +```json +{ + "tolerations": [ + { + "key": "virtualization", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule" + } + ] +} +``` + +**Step 3**: Update VM with modified spec + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +```json +{ + "resource": "" +} +``` + +Pass the complete modified VM resource as YAML or JSON string. + +**Step 4**: Verify tolerations were added + +Use `resources_get` again and check `.spec.template.spec.tolerations` in response. + +**Step 5**: Check if VM status improved + +Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. + +**CLI Fallback** (if MCP patch is too complex): +```bash +# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' + +# Verify tolerations +oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq + +# Check status +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**Example - Multiple Tolerations**: +```bash +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: "dedicated" + operator: "Equal" + value: "virt-workloads" + effect: "NoSchedule" +' +``` + +**Toleration Operators**: +- `Equal` - Key and value must match exactly +- `Exists` - Only key must exist (ignores value) + +**Toleration Effects**: +- `NoSchedule` - Don't schedule new pods (existing pods continue) +- `PreferNoSchedule` - Avoid scheduling if possible +- `NoExecute` - Don't schedule AND evict existing pods + +**Alternative Solutions**: +1. **Remove node taints** (if you have cluster-admin access): + ```bash + oc adm taint nodes virtualization=true:NoSchedule- + ``` + +2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits + +3. **File enhancement request** - Request tolerations support in openshift-mcp-server: + https://github.com/openshift/openshift-mcp-server/issues + +--- + +### 2. Insufficient Resources + +Not enough CPU, memory, or storage available on any node. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM resource requests**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' +``` + +**2. Check node resource availability**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. + +Alternatively, use `nodes_top` MCP tool for current resource usage. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe nodes | grep -A 5 "Allocated resources" +``` + +**3. Look for VM events mentioning "Insufficient"**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep "Insufficient" +``` + +**Example Event**: +``` +0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. +``` + +**Solutions** (Use MCP Tools First): + +1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) +2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill +3. **Delete unused VMs** - Use vm-delete skill to free up resources +4. **Check resource quotas**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters for quota**: + ```json + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" + } + ``` + + **Parameters for limit range**: + ```json + { + "apiVersion": "v1", + "kind": "LimitRange", + "namespace": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + oc describe limitrange -n + ``` + +--- + +### 3. Node Selector Mismatch + +VM requires specific node labels that don't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM node selector requirements**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.nodeSelector` to see required node labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' +``` + +**2. List available node labels**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.metadata.labels` for available labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes --show-labels +``` + +**3. Check if any nodes match the selector**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node", + "labelSelector": "=" +} +``` + +Should return at least one node with matching labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -l = +``` + +**Solutions** (Use MCP Tools First): + +**Option 1: Remove node selector from VM** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get current VM using `resources_get` (diagnostic step 1) +2. Remove `.spec.template.spec.nodeSelector` field +3. Update VM using `resources_create_or_update` with modified JSON + +**CLI Fallback** (JSON patch easier via CLI): +Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" +```bash +oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' +``` + +**Option 2: Add label to nodes** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get node using `resources_get` +2. Add label to `.metadata.labels` +3. Update node using `resources_create_or_update` + +⚠️ **Note**: Node labeling typically requires cluster admin privileges. + +**CLI Fallback** (simpler via CLI): +Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" +```bash +oc label node = +``` + +--- + +[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/storage-errors.md new file mode 100644 index 00000000..16ba584d --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/storage-errors.md @@ -0,0 +1,1011 @@ +--- +title: VM Storage Errors +category: kubevirt +sources: + - title: KubeVirt CDI - DataVolumes + url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ + date_accessed: 2026-02-17 + - title: Kubernetes Persistent Volumes + url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ + date_accessed: 2026-02-17 +tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] +semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] +use_cases: [vm-creation, vm-deletion, vm-cloning] +related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] +last_updated: 2026-02-17 +--- + +# VM Storage Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. + +**When to use this document**: +- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` +- Storage deletion fails after VM deletion +- DataVolume cloning operations fail +- PVC provisioning issues + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### ErrorDataVolumeNotReady + +**Symptom**: VM shows status `ErrorDataVolumeNotReady` + +**Description**: The DataVolume (persistent storage) backing the VM is not ready. + +**Possible Causes**: + +#### 1. DataVolume Still Provisioning + +Storage provisioning takes time, especially for large disks or when importing images. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Get detailed DataVolume information**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` and `.status.conditions` for provisioning details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o json +``` + +**3. Check PVC (PersistentVolumeClaim) bound status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Check `.status.phase` for each PVC (should be `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. + +#### 2. Storage Class Not Found + +The requested storage class doesn't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List available storage classes**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" +} +``` + +Review the list of available storage classes (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass +``` + +**2. Check DataVolume's requested storage class**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.spec.pvc.storageClassName` in the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' +``` + +**Solution**: +1. Use a valid storage class from the cluster +2. Recreate VM with correct storage class parameter + +#### 3. Insufficient Storage Quota + +Namespace has insufficient storage quota to provision the PVC. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check resource quotas**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**2. Check storage usage**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase +``` + +**Solution**: +1. Request quota increase from cluster admin +2. Delete unused PVCs to free quota +3. Reduce VM storage size + +--- + +### ErrorPvcNotFound + +**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List PVCs in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review the list of available PVCs (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check VM's PVC references**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' +``` + +**Solution**: +- Wait for DataVolume to create the PVC +- Manually create missing PVC +- Fix VM spec to reference correct PVC name + +--- + + +--- + +### Storage Deletion Failures + +**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace + +**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. + +**Possible Causes**: +- PVC still bound to active PersistentVolume with `Retain` policy +- DataVolume still being referenced by another resource +- CDI (Containerized Data Importer) controller issues +- Storage class retention policy preventing deletion +- Finalizers on PVC/DataVolume blocking cleanup +- PVC still mounted by a pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check PVC status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check specific PVC phase**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Released` or `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o jsonpath='{.status.phase}' +``` + +**3. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**4. Check what's using the PVC**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' +``` + +**5. Check PVC finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. + +**6. Check DataVolume finalizers**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.metadata.finalizers}' +``` + +**7. Check PV reclaim policy**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume" +} +``` + +Filter results for PV where `.spec.claimRef.name` matches ``. + +To get specific PV policy: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume", + "name": "" +} +``` + +Check `.spec.persistentVolumeReclaimPolicy`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pv | grep +oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' +``` + +**Common Finalizer Patterns**: +- `kubernetes.io/pvc-protection` - Protects PVC while in use +- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer + +**Solutions** (Use MCP Tools First): + +1. **Delete DataVolume first, then PVC**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete DataVolume first (often blocks PVC deletion): + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Wait a few seconds, then delete PVC: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete datavolume -n + oc delete pvc -n + ``` + +2. **Check for pods still using PVC**: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "" + } + ``` + + Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. + + Then delete the pods using `pods_delete`: + + **Parameters**: + ```json + { + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' + oc delete pod -n + ``` + +3. **Force delete PVC** (if safe to do so): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete pvc -n --grace-period=0 --force + ``` + +4. **Remove finalizers from PVC** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PVC using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update PVC using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Remove finalizers from DataVolume** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current DataVolume using `resources_get` (diagnostic step 6) + 2. Remove items from `.metadata.finalizers` array + 3. Update DataVolume using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Change PV reclaim policy** (if PV has Retain policy): + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PV policy using `resources_get` (diagnostic step 7) + 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` + 3. Update PV using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" + ```bash + oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' + oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' + ``` + +**Storage Quota Check** (Use MCP Tools First): + +After deletion, verify storage quota is freed: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters for quota check**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.used` to verify storage quota is freed. + +**Parameters for PVC verification**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +oc get pvc -n +``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Confirm PVC is deleted: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Confirm DataVolume is deleted: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +# Should return: Error from server (NotFound) + +oc get datavolume -n +# Should return: Error from server (NotFound) +``` + +--- + +### DataVolume Cloning Failures + +**Symptom**: VM clone created successfully but DataVolume clone operation fails + +**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. + +**Possible Causes**: +- CSI driver doesn't support volume cloning +- Source PVC storage class incompatible with cloning +- Cross-namespace cloning not permitted by storage backend +- Insufficient storage quota in target namespace +- Source PVC not in `Bound` state +- Storage class doesn't have volume cloning enabled +- CDI (Containerized Data Importer) controller issues + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume clone status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and `.metadata.name`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Check DataVolume events for errors**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe datavolume -n +``` + +**3. Check DataVolume phase**: + +Use `resources_get` from step 1, extract `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +``` + +**4. Check if storage class supports cloning**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "name": "" +} +``` + +Review the full YAML output for cloning-related configurations. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass -o yaml | grep -A 5 -i clone +``` + +**5. Check CSI driver capabilities**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "CSIDriver" +} +``` + +Review `.items[].metadata.name` for available CSI drivers. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get csidriver +``` + +**6. Check source PVC status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Bound` for cloning to work). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**7. Check target namespace storage quota**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**8. Check CDI controller logs**: + +**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) + +First, list pods in openshift-cnv namespace: + +**Parameters for pods_list_in_namespace**: +```json +{ + "namespace": "openshift-cnv", + "labelSelector": "app.kubernetes.io/component=cdi-deployment" +} +``` + +Then get logs using `pods_log`: + +**Parameters**: +```json +{ + "namespace": "openshift-cnv", + "name": "", + "tail": 100 +} +``` + +**CLI Fallback** (if MCP unavailable or easier via CLI): +```bash +oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') +``` + +**Common Error Messages**: +- `"volume cloning is not supported"` - CSI driver lacks clone capability +- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage +- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace +- `"insufficient quota"` - Target namespace lacks storage quota +- `"source PVC not bound"` - Source PVC must be in Bound state for cloning +- `"StorageClass does not support cloning"` - Storage class configuration issue + +**Solutions** (Use MCP Tools First): + +1. **Check storage class clone support**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" + } + ``` + + For each storage class in `.items[]`, check: + - `.metadata.name` (storage class name) + - `.provisioner` (CSI driver) + + Storage classes using CSI drivers typically support cloning. Look for provisioners like: + - `csi.ovirt.org` (oVirt CSI) + - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) + - `ebs.csi.aws.com` (AWS EBS CSI) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner + ``` + +2. **Verify source PVC is bound**: + + Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pvc -n -o jsonpath='{.status.phase}' + ``` + +3. **Check target namespace quota**: + + Use `resources_list` from diagnostic step 7 to check quota. + + If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + ``` + +4. **Use snapshot-based cloning** (alternative method): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Step 1**: Create VolumeSnapshot of source PVC + + **Parameters**: + ```json + { + "apiVersion": "snapshot.storage.k8s.io/v1", + "kind": "VolumeSnapshot", + "metadata": { + "name": "-snapshot", + "namespace": "" + }, + "spec": { + "source": { + "persistentVolumeClaimName": "" + } + } + } + ``` + + **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) + + **Step 3**: Create new DataVolume from snapshot + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": "-rootdisk", + "namespace": "" + }, + "spec": { + "source": { + "snapshot": { + "name": "-snapshot", + "namespace": "" + } + }, + "storage": { + "resources": { + "requests": { + "storage": "50Gi" + } + }, + "storageClassName": "" + } + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" + ```bash + cat <-snapshot + namespace: + spec: + source: + persistentVolumeClaimName: + EOF + + oc get volumesnapshot -snapshot -n + + cat <-rootdisk + namespace: + spec: + source: + snapshot: + name: -snapshot + namespace: + storage: + resources: + requests: + storage: 50Gi + storageClassName: + EOF + ``` + +5. **Use "new empty storage" option** (vm-clone skill): + - If cloning isn't supported, create VM with empty storage + - Manually copy data if needed + +6. **Cross-namespace cloning workaround**: + - Some storage backends require snapshot for cross-namespace cloning + - Create snapshot in source namespace, restore in target namespace (see solution 4 above) + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Check DataVolume reached Succeeded phase: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should return `Succeeded`). + +Check PVC was created and bound: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "-rootdisk" +} +``` + +Check `.status.phase` (should return `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +# Should return: Succeeded + +oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' +# Should return: Bound +``` + +**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/mcp-servers/mock-virt-mcp.py b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..2e083d72 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1458 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/skills/vm-snapshot-delete/SKILL.md b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/skills/vm-snapshot-delete/SKILL.md new file mode 100644 index 00000000..14f2fb49 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/skills/vm-snapshot-delete/SKILL.md @@ -0,0 +1,447 @@ +--- +name: vm-snapshot-delete +description: | + Permanently delete virtual machine snapshots to free storage space. + + Use when: + - "Delete snapshot [snapshot-name]" + - "Remove old snapshots for VM [name]" + - "Free up snapshot storage" + + Requires user confirmation before deletion. + + NOT for restoring VMs (use vm-snapshot-restore instead). + +model: inherit +color: yellow +--- + +# /vm-snapshot-delete Skill + +Permanently delete virtual machine snapshots in OpenShift Virtualization. Deleting snapshots frees storage but removes recovery points. + +## Prerequisites + +**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools**: +- `resources_get` (from openshift-virtualization) - Verify snapshot exists +- `resources_list` (from openshift-virtualization) - List related snapshots +- `resources_delete` (from openshift-virtualization) - Delete snapshot + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster (>= 4.19) +- OpenShift Virtualization operator installed +- ServiceAccount with RBAC permissions to delete VirtualMachineSnapshot resources + +### Prerequisite Verification + +**Before executing, verify MCP server availability:** + +1. **Check MCP Server Configuration** + - Verify `openshift-virtualization` exists in `.mcp.json` + - If missing → Report to user with setup instructions + +2. **Check Environment Variables** + - Verify `KUBECONFIG` is set (check presence only, never expose value) + - If missing → Report to user + +## When to Use This Skill + +**Trigger this skill when:** +- User wants to free storage by removing old snapshots +- User wants to delete a specific snapshot +- User wants to implement snapshot retention policies + +**User phrases that trigger this skill:** +- "Delete snapshot pre-upgrade-backup" +- "Remove old snapshots for VM database-01" +- "Delete all snapshots older than 7 days" +- "Free up snapshot storage" + +**Do NOT use this skill when:** +- User wants to create snapshots → Use `vm-snapshot-create` skill +- User wants to restore from snapshot → Use `vm-snapshot-restore` skill +- User wants to list snapshots → Use `vm-snapshot-list` skill + +## Workflow + +### Step 1: Gather Delete Information + +**Required Information from User:** +1. **Snapshot Name** - Name of snapshot to delete +2. **Namespace** - Namespace where snapshot exists + +If namespace not provided, ask for it. + +### Step 2: Verify Snapshot Exists + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "namespace": "", + "name": "" +} +``` + +**Expected Output**: VirtualMachineSnapshot resource + +**Error Handling**: +- If snapshot not found → Report error + +**If snapshot not found:** +```markdown +❌ Snapshot Not Found + +**Snapshot**: `` not found in namespace ``. + +**To list available snapshots:** +"List snapshots in namespace " + +Delete operation cancelled. +``` + +**STOP workflow**. + +**If snapshot found**, extract snapshot details: +- `spec.source.name` - VM name +- `metadata.creationTimestamp` - Creation timestamp +- `status.phase` - Snapshot status +- Calculate age from creationTimestamp + +### Step 3: List Other Snapshots for Same VM + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "namespace": "", + "labelSelector": "vm.kubevirt.io/name=" +} +``` + +**Fallback**: If label selector doesn't work, list all snapshots and filter by `spec.source.name`. + +**Count snapshots** for the VM to determine if this is the last snapshot. + +### Step 4: Present Snapshot Details and Confirm Deletion + +```markdown +## ⚠️ Snapshot Deletion - Review + +**Snapshot to Delete**: `` + +### Snapshot Details +- **Snapshot Name**: `` +- **VM**: `` +- **Namespace**: `` +- **Created**: +- **Age**: +- **Status**: + +### Impact of Deletion +- ✗ Snapshot will be permanently deleted +- ✗ This recovery point will be lost +- ✗ Cannot restore VM to this snapshot state after deletion +- ✓ Storage will be freed + +### Recovery Impact +**Before deletion, consider:** +- Is this snapshot still needed for recovery? +- Are there other recovery points available? +- Could you need to restore to this state in the future? + +**Available snapshots for VM ``:** + + + +⚠️ **WARNING**: This is the ONLY snapshot for VM ``. After deletion, no snapshot recovery points will exist. + + +--- + +**Proceed with snapshot deletion? This action cannot be undone. (yes/no)** +``` + +**Wait for user confirmation.** + +**Handle response:** +- If "yes" → Proceed to Step 5 (execute deletion) +- If "no", "cancel", or anything else → Cancel operation + +**On cancellation:** +```markdown +Snapshot deletion cancelled by user. Snapshot `` preserved. +``` + +**STOP workflow**. + +### Step 5: Delete the Snapshot + +**ONLY PROCEED AFTER user confirmation in Step 4.** + +**MCP Tool**: `resources_delete` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "namespace": "", + "name": "" +} +``` + +**Example tool invocation:** +```json +resources_delete({ + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "namespace": "production", + "name": "old-snapshot" +}) +``` + +**Expected Output**: VirtualMachineSnapshot deleted successfully + +**Error Handling**: +- If snapshot not found → Report error (may have been deleted externally) +- If permission denied → Report RBAC error +- If snapshot in use → Report error (snapshot may be in restore process) + +**Report progress:** +```markdown +🗑️ Deleting snapshot... +✓ Snapshot `` deleted +``` + +### Step 6: Report Deletion Results + +**On success:** + +```markdown +## ✓ Snapshot Deleted Successfully + +**Snapshot**: `` (VM: ``, namespace: ``) + +### Deletion Summary +- ✓ Snapshot permanently deleted +- ✓ Storage freed +- ✓ Recovery point removed + +### Impact +- ✗ Cannot restore VM to state +- ✗ Snapshot `` no longer available + + +### Remaining Snapshots for VM `` + + + +These snapshots are still available for recovery. + + + +⚠️ **No snapshots remain** for VM ``. Consider creating new snapshots for future recovery points. + + +--- + +### Next Steps + +**To create a new snapshot:** +"Create snapshot of VM " + +**To list remaining snapshots:** +"List snapshots for VM " +``` + +**On failure:** + +```markdown +## ❌ Snapshot Deletion Failed + +**Error**: + +**Snapshot**: `` (VM: ``, namespace: ``) + +**Common Causes:** +- **Snapshot not found** - May have been deleted externally +- **Insufficient RBAC permissions** - ServiceAccount lacks delete permissions +- **Snapshot in use** - Snapshot may be in active restore process +- **Storage backend error** - CSI driver or storage backend issue + +**Troubleshooting Steps:** + +1. **Verify snapshot still exists:** + "List snapshots for VM " + +2. **Check if snapshot is being used for restore:** + Use `resources_list` to check for active VirtualMachineRestore resources + +3. **Check permissions:** + Use CLI: `oc auth can-i delete virtualmachinesnapshots -n ` + +4. **Wait and retry** if snapshot is in use by restore operation + +Would you like help troubleshooting this error? +``` + +## Common Issues + +### Issue 1: Snapshot Not Found + +**Error**: "Snapshot `` not found in namespace ``" + +**Cause**: Snapshot doesn't exist, was deleted, or wrong namespace/name. + +**Solution:** +1. List snapshots to verify name: "List snapshots in namespace " +2. Check spelling (names are case-sensitive) +3. Try listing in other namespaces if unsure + +### Issue 2: Snapshot In Use During Restore + +**Error**: "Snapshot is in use by restore operation" + +**Cause**: An active VirtualMachineRestore is using this snapshot. + +**Solution:** +1. Check for active restores: Use `resources_list` with apiVersion="snapshot.kubevirt.io/v1beta1", kind="VirtualMachineRestore" +2. Wait for restore to complete, or delete the VirtualMachineRestore resource +3. Retry snapshot deletion + +### Issue 3: Permission Denied + +**Error**: "Forbidden: User lacks permissions to delete virtualmachinesnapshots" + +**Cause**: Missing RBAC permissions for snapshot deletion. + +**Solution:** +1. Check permissions: `oc auth can-i delete virtualmachinesnapshots -n ` +2. Contact cluster admin to grant delete permissions for virtualmachinesnapshots +3. Required permissions: delete verb on snapshot.kubevirt.io/virtualmachinesnapshots + +## Dependencies + +### Required MCP Servers +- `openshift-virtualization` - OpenShift MCP server with kubevirt toolset + +### Required MCP Tools +- `resources_get` (from openshift-virtualization) - Get snapshot details + - Parameters: apiVersion, kind, namespace, name + - Source: https://github.com/openshift/openshift-mcp-server + +- `resources_list` (from openshift-virtualization) - List related snapshots + - Parameters: apiVersion, kind, namespace, labelSelector + - Source: https://github.com/openshift/openshift-mcp-server + +- `resources_delete` (from openshift-virtualization) - Delete Kubernetes resources + - Parameters: apiVersion, kind, namespace, name + - Source: https://github.com/openshift/openshift-mcp-server + +### Related Skills +- `vm-snapshot-list` - List snapshots before deletion +- `vm-snapshot-create` - Create new snapshots +- `vm-snapshot-restore` - Restore VMs from snapshots + +### Reference Documentation +- [OpenShift Virtualization Snapshots](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt-managing-vm-snapshots) +- [KubeVirt VM Snapshots](https://kubevirt.io/user-guide/operations/snapshot_restore_api/) + +## Critical: Human-in-the-Loop Requirements + +**IMPORTANT:** This skill performs destructive operations. You MUST: + +1. **Before Deleting Snapshots** + - Show snapshot details (VM, age, size) + - Confirm snapshot won't be needed for recovery + - List other available snapshots for the VM + - Ask: "Proceed with snapshot deletion? (yes/no)" + - Wait for explicit "yes" + +2. **Never Auto-Execute** + - **NEVER delete without user confirmation** + - **ALWAYS show what will be lost before deletion** + +**Why This Matters:** +- **Recovery Point Loss**: Deleting snapshots removes recovery options permanently +- **No Undo**: Snapshot deletion cannot be reversed +- **Last Snapshot Warning**: Users must know if deleting the only snapshot for a VM + +## Security Considerations + +- **RBAC Enforcement**: Requires delete permissions for VirtualMachineSnapshot resources +- **User Confirmation**: Always requires explicit "yes" before deletion +- **Last Snapshot Warning**: Warns users when deleting the only snapshot for a VM +- **Namespace Isolation**: Snapshots scoped to namespace boundaries +- **Audit Trail**: Deletions logged in Kubernetes API audit logs + +## Example Usage + +``` +User: "Delete snapshot database-01-daily-backup" + +Agent: "Which namespace is the snapshot in?" + +User: "production" + +Agent: [Invokes vm-snapshot-delete skill] + [Step 2: Verifies snapshot exists] + [Step 3: Lists other snapshots] + [Step 4: Presents deletion confirmation] + +## ⚠️ Snapshot Deletion - Review + +Snapshot to Delete: `database-01-daily-backup` + +Snapshot Details: +- Name: `database-01-daily-backup` +- VM: `database-01` +- Namespace: `production` +- Created: 2024-01-14 02:00:00 UTC +- Age: 1 day + +Impact of Deletion: +- ✗ Snapshot will be permanently deleted +- ✗ This recovery point will be lost +- ✓ Storage will be freed + +Available snapshots for VM `database-01`: +- `database-01-pre-upgrade` (created 2024-01-15 10:30) + +Proceed with snapshot deletion? This action cannot be undone. (yes/no) + +User: "yes" + +Agent: [Step 5: Deletes snapshot] + +🗑️ Deleting snapshot... +✓ Snapshot `database-01-daily-backup` deleted + + [Step 6: Reports results] + +## ✓ Snapshot Deleted Successfully + +Snapshot: `database-01-daily-backup` + +Deletion Summary: +- ✓ Snapshot permanently deleted +- ✓ Storage freed +- ✓ Recovery point removed + +Remaining Snapshots for VM `database-01`: +- `database-01-pre-upgrade` (created 2024-01-15 10:30) + +This snapshot is still available for recovery. +``` diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/instruction.md b/evaluation/with_skills/rh-virt__vm-snapshot-delete/instruction.md new file mode 100644 index 00000000..3058c144 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-delete/instruction.md @@ -0,0 +1,12 @@ +# VM Snapshot Deletion Task + +You are an OpenShift Virtualization administrator. Delete snapshot `production-db-backup-20240215` for VM `production-db` in namespace `prod-vms`. + +## Requirements +- Verify the snapshot is safe to delete (no active restores, not the last snapshot) +- Include user confirmation safeguards +- Verify deletion completed + +Use MCP tools to examine the cluster. Document your methodology, findings, and deletion plan in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/solution/solve.sh b/evaluation/with_skills/rh-virt__vm-snapshot-delete/solution/solve.sh new file mode 100644 index 00000000..11098bb3 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-delete/solution/solve.sh @@ -0,0 +1,26 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Snapshot Deletion Plan + +## Target: production-db-backup-20240215 + +### Safety Checks +1. **Restore conflict check**: Verify no active VirtualMachineRestore references this snapshot + - If snapshot is in use by a restore operation, deletion will fail +2. **Last snapshot warning**: List all snapshots for production-db + - Other snapshots exist (production-db-backup-20240301) — NOT the last snapshot + - If this were the only remaining snapshot, show explicit warning + +### Deletion Procedure +1. Verify snapshot exists (apiVersion: snapshot.kubevirt.io/v1beta1, kind: VirtualMachineSnapshot) +2. Check for active VirtualMachineRestore resources (snapshot in use blocks deletion) +3. List other snapshots for production-db via labelSelector vm.kubevirt.io/name +4. Request user confirmation (proceed yes/no) +5. Delete snapshot via resources_delete +6. Verify deletion completed +7. Impact: Storage freed, recovery point removed + +### Note +This is NOT the last snapshot — production-db-backup-20240301 remains available for restore. + +REPORT_EOF diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/task.toml b/evaluation/with_skills/rh-virt__vm-snapshot-delete/task.toml new file mode 100644 index 00000000..7d13e981 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-delete/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-snapshot-delete" +name = "rh-virt VM Snapshot Deletion Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-snapshot-delete", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/tests/llm_judge.py b/evaluation/with_skills/rh-virt__vm-snapshot-delete/tests/llm_judge.py new file mode 100644 index 00000000..92546360 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-delete/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "restore_conflict", "file": "/root/report.md", "question": "Does the report check for active VirtualMachineRestore before deleting a snapshot?", "reference": "A skilled report checks for active restores. An unskilled report deletes without checking conflicts."}, + {"id": "last_snapshot_warning", "file": "/root/report.md", "question": "Does the report warn when deleting the only remaining snapshot for a VM?", "reference": "A skilled report warns about loss of last recovery point. An unskilled report deletes without warning."}, + {"id": "label_selector_filter", "file": "/root/report.md", "question": "Does the report use spec.source.name or vm.kubevirt.io/name label to list other snapshots for the same VM?", "reference": "A skilled report uses proper filtering to find related snapshots. An unskilled report lists all snapshots without VM filtering."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/tests/test.sh b/evaluation/with_skills/rh-virt__vm-snapshot-delete/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-delete/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/tests/test_outputs.py b/evaluation/with_skills/rh-virt__vm-snapshot-delete/tests/test_outputs.py new file mode 100644 index 00000000..f7220d55 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-delete/tests/test_outputs.py @@ -0,0 +1,71 @@ +""" +Tests for rh-virt__vm-snapshot-delete per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_snapshot(self): + content = read_report().lower() + assert "snapshot" in content, "report should mention snapshots" + + def test_mentions_deletion(self): + content = read_report().lower() + assert "delet" in content, "report should discuss deletion" + + +class TestSkillDependent: + def test_restore_conflict_check(self): + """Skill: Active VirtualMachineRestore blocks snapshot deletion.""" + c = read_report().lower() + assert any(t in c for t in ["virtualmachinerestore", "restore", "in use", "active restore", "block delet"]) and ( + "restore" in c or "conflict" in c + ), ( + "should check for active restore blocking deletion" + ) + + def test_last_snapshot_warning(self): + """Skill: Warn when deleting the only snapshot for a VM.""" + c = read_report().lower() + assert any(t in c for t in ["last snapshot", "only snapshot", "no recovery", "only remaining", "no other snapshot"]) or ( + "last" in c and "snapshot" in c and ("warn" in c or "only" in c) + ), ( + "should warn when deleting the last snapshot for a VM" + ) + + def test_storage_reclaim(self): + """Skill: Storage freed by deletion; recovery point lost.""" + c = read_report().lower() + assert any(t in c for t in ["storage freed", "storage reclaim", "freed", "recovery point"]), ( + "should mention storage reclamation or recovery point loss" + ) + + def test_virtualmachinesnapshot_delete(self): + """Skill: Delete VirtualMachineSnapshot resource.""" + c = read_report().lower() + assert any(t in c for t in ["virtualmachinesnapshot", "resources_delete", "delete snapshot"]) and ( + "snapshot" in c + ), ( + "should reference VirtualMachineSnapshot deletion" + ) + + def test_list_other_snapshots(self): + """Skill: List other snapshots for same VM before delete.""" + c = read_report().lower() + assert any(t in c for t in ["spec.source.name", "label selector", "vm.kubevirt.io/name", "other snapshot", "list snapshot", "same vm"]), ( + "should list other snapshots for the VM" + ) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/Dockerfile b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/Dockerfile new file mode 100644 index 00000000..ae625e01 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/.ai-index/semantic-index.json new file mode 100644 index 00000000..2512c2a3 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/.ai-index/semantic-index.json @@ -0,0 +1,148 @@ +{ + "index_version": "1.0.0", + "last_updated": "2026-02-17", + "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", + + "documents": [ + { + "file": "INDEX.md", + "title": "VM Troubleshooting Guide - Index", + "category": "navigation", + "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], + "use_cases": ["navigation", "tool-reference", "general-guidance"], + "token_estimate": 450, + "skills": ["all"] + }, + { + "file": "scheduling-errors.md", + "title": "VM Scheduling Errors", + "category": "scheduling", + "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], + "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], + "errors_covered": [ + "ErrorUnschedulable - Node Taints", + "ErrorUnschedulable - Insufficient Resources", + "ErrorUnschedulable - Node Selector Mismatch" + ], + "token_estimate": 820, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "storage-errors.md", + "title": "VM Storage Errors", + "category": "storage", + "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], + "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], + "errors_covered": [ + "ErrorDataVolumeNotReady - DataVolume Still Provisioning", + "ErrorDataVolumeNotReady - Storage Class Not Found", + "ErrorDataVolumeNotReady - Insufficient Storage Quota", + "ErrorPvcNotFound", + "Storage Deletion Failures", + "DataVolume Cloning Failures" + ], + "token_estimate": 1350, + "skills": ["vm-creator", "vm-delete", "vm-clone"] + }, + { + "file": "lifecycle-errors.md", + "title": "VM Lifecycle Errors", + "category": "lifecycle", + "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], + "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], + "errors_covered": [ + "VM Stuck in Terminating State", + "VM Won't Start (Non-Scheduling Issues)", + "VM Won't Stop" + ], + "token_estimate": 1150, + "skills": ["vm-delete", "vm-lifecycle-manager"] + }, + { + "file": "runtime-errors.md", + "title": "VM Runtime Errors", + "category": "runtime", + "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], + "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], + "errors_covered": [ + "CrashLoopBackOff" + ], + "token_estimate": 850, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "network-errors.md", + "title": "VM Network Errors", + "category": "networking", + "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], + "use_cases": ["vm-creation", "network-troubleshooting"], + "errors_covered": [ + "Network Attachment Failures" + ], + "token_estimate": 580, + "skills": ["vm-creator"] + } + ], + + "skill_to_docs_mapping": { + "vm-creator": { + "primary_docs": ["scheduling-errors.md", "storage-errors.md"], + "secondary_docs": ["runtime-errors.md", "network-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3650, + "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" + }, + "vm-delete": { + "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 2950, + "description": "VM deletion failures - stuck terminating, storage cleanup" + }, + "vm-clone": { + "primary_docs": ["storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 1800, + "description": "VM cloning failures - DataVolume cloning, storage provisioning" + }, + "vm-lifecycle-manager": { + "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], + "secondary_docs": ["runtime-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3270, + "description": "VM start/stop failures - lifecycle management, scheduling" + }, + "vm-inventory": { + "primary_docs": [], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 450, + "description": "General guidance for status interpretation" + } + }, + + "error_to_docs_mapping": { + "ErrorUnschedulable": "scheduling-errors.md", + "ErrorDataVolumeNotReady": "storage-errors.md", + "ErrorPvcNotFound": "storage-errors.md", + "Terminating": "lifecycle-errors.md", + "CrashLoopBackOff": "runtime-errors.md", + "NetworkAttachmentFailed": "network-errors.md" + }, + + "cross_references": { + "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], + "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], + "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], + "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], + "network-errors.md": ["scheduling-errors.md"] + }, + + "progressive_disclosure_strategy": { + "description": "Load only the minimum documentation needed for current task", + "pattern": "error_type → primary_doc → cross_references (if needed)", + "token_savings": "29% average reduction vs single-file (based on rh-sre data)", + "navigation_overhead_reduction": "85% vs single-file navigation" + } +} diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/INDEX.md new file mode 100644 index 00000000..a2e76963 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/INDEX.md @@ -0,0 +1,332 @@ +--- +title: VM Troubleshooting Guide - Index +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 + - title: OpenShift Virtualization - Virtual Machine Status + url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html + date_accessed: 2026-02-06 + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] +semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] +use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] +last_updated: 2026-02-17 +--- + +# VM Troubleshooting Guide - Index + +## Overview + +This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: + +- **Creation**: Scheduling failures, storage provisioning issues +- **Lifecycle**: Start/stop failures, stuck states +- **Deletion**: Resources stuck in Terminating, storage cleanup failures +- **Cloning**: DataVolume cloning errors, cross-namespace issues +- **Networking**: Secondary network attachment failures +- **Runtime**: CrashLoopBackOff, guest OS boot failures + +This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. + +--- + +## 🗂️ Troubleshooting Categories + +The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: + +### 1. [Scheduling Errors](scheduling-errors.md) +**When to use**: VM fails to schedule on any node + +**Errors covered**: +- ErrorUnschedulable - Node Taints +- ErrorUnschedulable - Insufficient Resources +- ErrorUnschedulable - Node Selector Mismatch + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 2. [Storage Errors](storage-errors.md) +**When to use**: VM has storage provisioning, deletion, or cloning issues + +**Errors covered**: +- ErrorDataVolumeNotReady (all 3 subsections) +- ErrorPvcNotFound +- Storage Deletion Failures +- DataVolume Cloning Failures + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### 3. [Lifecycle Errors](lifecycle-errors.md) +**When to use**: VM has start/stop/termination issues + +**Errors covered**: +- VM Stuck in Terminating State +- VM Won't Start (Non-Scheduling Issues) +- VM Won't Stop + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### 4. [Runtime Errors](runtime-errors.md) +**When to use**: VM repeatedly crashes or fails at runtime + +**Errors covered**: +- CrashLoopBackOff + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 5. [Network Errors](network-errors.md) +**When to use**: VM has secondary network attachment failures + +**Errors covered**: +- Network Attachment Failures (Multus, SR-IOV) + +**Skills that use this**: vm-create + +--- + +## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last + +### Critical Principle: MCP-First Approach + +**All diagnostic procedures in this guide follow the MCP-first pattern**: + +``` +1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) +2. ⚠️ IF FAILS: Ask user permission to use CLI command +3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval +``` + +### Why MCP Tools First? + +- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources +- **Consistency**: Same interface across all operations +- **Better Error Handling**: MCP tools return structured errors +- **Audit Trail**: MCP tool usage is logged and trackable + +### Command Pattern Examples + +Throughout this guide, you'll see diagnostic steps formatted like this: + +**✅ CORRECT Pattern (MCP First)**: + +```markdown +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Extract**: `.metadata.finalizers` from returned JSON + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` +``` + +### Available MCP Tools + +The `openshift-virtualization` MCP server provides these tools: + +**Resource Operations**: +- `resources_get` - Get specific resource (replaces `oc get `) +- `resources_list` - List resources (replaces `oc get `) +- `resources_delete` - Delete resource (replaces `oc delete`) +- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) + +**Pod Operations**: +- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) +- `pods_get` - Get pod details (replaces `oc get pod`) +- `pods_log` - Get pod logs (replaces `oc logs`) +- `pods_exec` - Execute in pod (replaces `oc exec`) +- `pods_delete` - Delete pod (replaces `oc delete pod`) +- `pods_top` - Pod resource usage (replaces `oc top pods`) + +**Events & Monitoring**: +- `events_list` - List events (replaces `oc get events`) +- `nodes_top` - Node resource usage (replaces `oc top nodes`) +- `nodes_log` - Node logs (replaces `oc adm node-logs`) +- `nodes_stats_summary` - Detailed node stats + +**VM Operations** (KubeVirt toolset): +- `vm_create` - Create VMs +- `vm_lifecycle` - Start/stop/restart VMs + +**Namespaces**: +- `namespaces_list` - List namespaces (replaces `oc get namespaces`) +- `projects_list` - List OpenShift projects (replaces `oc get projects`) + +### When CLI Commands Are Required + +Some operations have **NO MCP equivalent** and require CLI: + +- `virtctl` commands (console, VNC access) +- `oc debug node` (node debugging) +- `oc auth can-i` (permission checks) +- `oc adm taint` (node taint management) +- Complex JSON patch operations + +For these, the guide will note: **"CLI Only - No MCP equivalent"** + +### Quick Reference: CLI → MCP Mapping + +| CLI Command | MCP Tool Equivalent | +|-------------|---------------------| +| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | +| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | +| `oc logs -n ` | `pods_log` with name="", namespace="" | +| `oc get events -n ` | `events_list` with namespace="" | +| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | + +**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. + +### How to Read Diagnostic Sections + +Each error section includes: +1. **Symptom** - What you observe +2. **Description** - What's happening +3. **Possible Causes** - Why it's happening +4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback +5. **Solutions** - **MCP tools first**, then CLI fallback +6. **Verification** - **MCP tools first**, then CLI fallback + +**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. + +--- + +## 🔍 Quick Navigation by Skill + +**vm-create**: +- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics +- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning +- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff +- [Network Errors](network-errors.md) - Network attachment failures + +**vm-delete**: +- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state +- [Storage Errors](storage-errors.md) - Storage deletion failures + +**vm-clone**: +- [Storage Errors](storage-errors.md) - DataVolume cloning failures + +**vm-lifecycle-manager**: +- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop +- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling + +**vm-inventory**: +- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed + +--- + +## 📊 Documentation Coverage & Maintenance + +### Current Coverage + +This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: + +- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) +- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures +- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop +- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) +- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems + +**Total errors documented**: 12 error types across 6 categories + +--- + +### Encountering Undocumented Errors + +#### For AI Agents (Claude Code) + +If you encounter an error **not documented** in the categories above: + +1. **Report to user** with all available details (error message, affected resources, namespace) +2. **Provide best-effort diagnostics** using MCP tools: + - `resources_get` to inspect resource status + - `pods_log` to check virt-launcher or compute container logs + - `events_list` to view Kubernetes events +3. **Suggest documentation update**: + ``` + ⚠️ This error is not yet documented in the troubleshooting guide. + + **Error**: + **Affected resource**: / + + I recommend adding this error to the troubleshooting guide: + - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file + - If it's a new error class → Create new category file in docs/troubleshooting/ + + Would you like me to help document this error for future reference? + ``` + +4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs + +--- + +#### For Human Contributors + +To document a new error: + +1. **Determine the category**: + - Scheduling issue → `scheduling-errors.md` + - Storage problem → `storage-errors.md` + - Lifecycle/start/stop → `lifecycle-errors.md` + - Runtime crash → `runtime-errors.md` + - Network issue → `network-errors.md` + - New error class → Create new category file (e.g., `security-errors.md`) + +2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): + - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback + - **Solutions**: MCP Tool approach → Implementation → CLI Fallback + - **Verification**: MCP Tool checks → Expected results → CLI Fallback + +3. **Update semantic index**: + - Edit `.ai-index/semantic-index.json` + - Add error to `error_to_docs_mapping` + - Update relevant skill's `primary_docs` or `secondary_docs` + - Increment token estimates if needed + +4. **Reference in skill files**: + - Update skill's Reference Documentation section + - Ensure skill's Document Consultation steps point to the right category + +5. **Update this INDEX.md**: + - Add error to the appropriate category section (lines 50-120) + - Update "Current Coverage" list above + +--- + +## 📚 Additional Resources + +- [rh-virt README](../../README.md) - MCP server setup and skill documentation +- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources +- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/lifecycle-errors.md new file mode 100644 index 00000000..6167fcc8 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/lifecycle-errors.md @@ -0,0 +1,869 @@ +--- +title: VM Lifecycle Errors +category: kubevirt +sources: + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 +tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] +semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] +use_cases: [vm-deletion, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Lifecycle Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM lifecycle issues including start/stop failures and stuck termination states. + +**When to use this document**: +- VM stuck in `Terminating` state +- VM won't start (runStrategy is Always but VM never reaches Running) +- VM won't stop (runStrategy is Halted but VM never reaches Stopped) + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### VM Stuck in Terminating State + +**Symptom**: VM shows status `Terminating` but deletion never completes + +**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). + +**Possible Causes**: +- Finalizers blocking deletion +- PVC/DataVolume still attached and preventing cleanup +- VirtualMachineInstance (VMI) not terminating properly +- Custom controllers or operators blocking deletion +- Stuck virt-launcher pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check finalizers on the VM**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` + +**2. Check if VMI still exists**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name containing "virt-launcher-". + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check events for deletion issues**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results where `involvedObject.name` == "" and sort by timestamp. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' +``` + +**5. Check VM deletion timestamp**: + +Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. + +If present, VM is in deletion process. If null, VM is not being deleted. + +**Common Finalizer Patterns**: +- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer +- `foregroundDeletion` - Waits for dependent objects to be deleted +- Custom finalizers from operators + +**Solutions** (Use MCP Tools First): + +1. **Wait for dependent resources** (safest approach): + - Use `resources_list` to check if PVCs, DataVolumes are still deleting + - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) + +2. **Force delete VMI** (if VMI is stuck): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable - requires explicit user permission): + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +3. **Force delete virt-launcher pod** (if pod is stuck): + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + # Find the virt-launcher pod name first + oc get pods -n | grep virt-launcher- + + # Then delete it + oc delete pod virt-launcher--xxx -n --force --grace-period=0 + ``` + +4. **Remove finalizers** (⚠️ dangerous - use only as last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` + 2. Remove items from `.metadata.finalizers` array + 3. Update VM using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch not easily done via MCP): + ```bash + # This operation is complex for MCP - may need CLI + oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Check for protection labels** (vm-delete skill specific): + + Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. + + If value is "true", the vm-delete skill refuses deletion (this is expected behavior). + +**Verification**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error if deletion successful. + +**CLI Fallback**: +```bash +oc get vm -n +# Should return: Error from server (NotFound) +``` + +--- + + +--- + +### VM Won't Start (Non-Scheduling Issues) + +**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state + +**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. + +**Possible Causes**: +- Guest OS kernel panic or boot failure +- Cloud-init configuration errors +- Missing or corrupted disk image +- Insufficient memory for guest OS to boot +- QEMU/KVM virtualization errors +- VirtualMachineInstance (VMI) creation failures +- virt-launcher pod crashes + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VMI (VirtualMachineInstance) status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and overall VMI state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**2. Check VMI conditions for errors**: + +Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. View virt-launcher pod logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +First, get pod name from step 3, then: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +Look for QEMU errors, memory allocation failures, device errors. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**5. Check virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. + +Alternatively, use `pods_get` to get full pod details: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**6. Access VM console to see guest OS boot messages**: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +Look for kernel panic, initramfs errors, filesystem errors. + +**7. Check VM events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep -A 20 "Events:" +``` + +**Common Error Patterns**: + +1. **Guest OS Boot Failure**: + - Console shows kernel panic + - Guest hangs at GRUB or boot loader + - Cloud-init errors during first boot + +2. **Insufficient Memory**: + - Guest OS kills processes due to OOM + - VMI logs show memory allocation errors + +3. **QEMU Crashes**: + - virt-launcher logs show QEMU segfaults + - VMI repeatedly restarts + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for kernel panic, initramfs errors, filesystem errors. + +2. **Check virt-launcher pod logs for QEMU errors**: + + Use `pods_log` from diagnostic step 4 to view logs. + + Look for: + - "qemu-system-x86_64: ..." errors + - Memory allocation failures + - Device errors + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx + ``` + +3. **Increase memory if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check VMI memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +4. **Verify disk image integrity**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check PVC is bound: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + If using container disk, verify image exists and is accessible (check VMI spec). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get pvc -n + ``` + +5. **Check cloud-init configuration** (if applicable): + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. + + Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq + ``` + +6. **Restart VMI** (soft reset): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete VMI (VM controller will recreate it): + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for new VMI to start (use `resources_get` to check status). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n -w + ``` + +7. **Check virtualization extensions** (KVM): + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + + # In debug shell: + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should eventually return `Running`). + +Check VMI is running: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should show `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should eventually return: Running + +oc get vmi -n +# Should show: Running +``` + +--- + +### VM Won't Stop + +**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state + +**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. + +**Possible Causes**: +- Guest OS not responding to ACPI shutdown signal +- virt-launcher pod stuck and not terminating +- VirtualMachineInstance (VMI) deletion blocked by finalizers +- Guest shutdown scripts hanging +- Filesystem sync issues in guest OS + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (might show `Stopping`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**2. Check VMI status and deletion timestamp**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check VMI events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vmi -n | grep -A 10 "Events:" +``` + +**5. Check VMI finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.metadata.finalizers}' +``` + +**6. Check if guest is responsive** (if VMI still exists): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Patterns**: +- VMI shows `deletionTimestamp` but never actually deletes +- virt-launcher pod in `Terminating` state +- VM runStrategy is `Halted` but printableStatus shows `Stopping` + +**Solutions** (Use MCP Tools First): + +1. **Wait for graceful shutdown** (default: 30 seconds): + + Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. + +2. **Force stop by deleting VMI**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + This is the standard way to force-stop a VM. + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n + # Should return: Error from server (NotFound) + ``` + +3. **Force delete VMI with grace period** (if VMI won't delete): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +4. **Force delete virt-launcher pod**: + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find the pod using `pods_list_in_namespace` from diagnostic step 3. + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + ⚠️ **Note**: For force deletion with grace period, use CLI fallback. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') + oc delete pod $POD_NAME -n --force --grace-period=0 + ``` + +5. **Remove VMI finalizers** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VMI using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update VMI using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Patch VM runStrategy directly** (ensure consistency): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` (diagnostic step 1) + 2. Set `.spec.runStrategy` to `"Halted"` + 3. Update VM using `resources_create_or_update` with modified JSON + + **CLI Fallback** (merge patch easier via CLI): + Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Stopped` or `Halted`). + +Verify VMI is gone: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Verify virt-launcher pod is gone: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for pods matching `virt-launcher-`. Should return no results. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Stopped or Halted + +oc get vmi -n +# Should return: Error from server (NotFound) + +oc get pods -n | grep virt-launcher- +# Should return: No resources found +``` + +**Prevention**: +- Ensure guest OS has ACPI support enabled +- Use proper shutdown commands in guest OS +- Avoid forceful stops unless necessary (can corrupt guest filesystem) + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/network-errors.md new file mode 100644 index 00000000..82fa313c --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/network-errors.md @@ -0,0 +1,429 @@ +--- +title: VM Network Errors +category: kubevirt +sources: + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] +semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] +use_cases: [vm-creation, network-troubleshooting] +related_docs: [INDEX.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Network Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. + +**When to use this document**: +- VM created successfully but secondary networks not attached +- NetworkAttachmentDefinition not found errors +- Multus CNI failures +- SR-IOV device attachment issues + +**Skills that use this**: vm-create + +--- + +### Network Attachment Failures + +**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working + +**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). + +**Possible Causes**: +- NetworkAttachmentDefinition doesn't exist in the namespace +- Multus CNI not installed or not configured on cluster +- Namespace mismatch (NAD in different namespace than VM) +- Interface name conflicts in VM spec +- Bridge/network configuration errors in NAD +- SR-IOV device not available (if using SR-IOV) + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List NetworkAttachmentDefinitions in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "" +} +``` + +Review `.items[].metadata.name` for available NADs. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -n +``` + +**2. List NetworkAttachmentDefinitions in all namespaces**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition" +} +``` + +Omit `namespace` parameter to list across all namespaces. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -A +``` + +**3. Check specific NetworkAttachmentDefinition**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "", + "name": "" +} +``` + +Review `.spec.config` for CNI configuration. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definition -n -o yaml +``` + +**4. Check VM network configuration**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.networks` to see network references. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq +``` + +**5. Check VM domain interfaces**: + +Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq +``` + +**6. Check virt-launcher pod network annotations** (shows actual attachments): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**7. Check Multus is installed**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "openshift-multus" +} +``` + +Should show Multus CNI pods running. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n openshift-multus +``` + +**8. Check for errors in virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n +``` + +**Common Error Messages**: +- `"network-attachment-definition not found"` - NAD doesn't exist in namespace +- `"multus CNI not configured"` - Multus not installed or misconfigured +- `"interface name conflict"` - Duplicate interface names in VM spec +- `"failed to add network"` - CNI plugin error (check NAD config) +- `"no available devices"` - SR-IOV device not available (if using SR-IOV) + +**Solutions** (Use MCP Tools First): + +1. **Verify NetworkAttachmentDefinition exists in correct namespace**: + + Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. + + If NAD is in different namespace, copy it to VM namespace: + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get NAD from source namespace using `resources_get` + 2. Modify `.metadata.namespace` to target namespace + 3. Create NAD in target namespace using `resources_create_or_update` + + **CLI Fallback** (stream processing easier via CLI): + Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" + ```bash + oc get network-attachment-definition -n -o yaml | \ + sed "s/namespace: /namespace: /" | \ + oc apply -f - + ``` + +2. **Create missing NetworkAttachmentDefinition**: + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + Example: Linux bridge network + + **Parameters**: + ```json + { + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": { + "name": "vlan100", + "namespace": "" + }, + "spec": { + "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" + ```bash + cat < + spec: + config: '{ + "cniVersion": "0.3.1", + "type": "bridge", + "bridge": "br1", + "vlan": 100, + "ipam": { + "type": "host-local", + "subnet": "192.168.100.0/24" + } + }' + EOF + ``` + +3. **Check Multus CNI installation**: + + Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. + + To check cluster network operator: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterOperator" + } + ``` + + Filter for `network` operator. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n openshift-multus + oc get clusteroperators network + ``` + +4. **Fix interface name conflicts** (if VM has duplicate names): + + Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. + + Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. + + **CLI Fallback** (interactive edit easier via CLI): + Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' + oc edit vm -n + ``` + +5. **Validate NAD configuration syntax**: + + Use `resources_get` from diagnostic step 3, extract `.spec.config`. + + Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get network-attachment-definition -n -o jsonpath='{.spec.config}' + ``` + +6. **Check SR-IOV device availability** (if using SR-IOV networks): + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + List SR-IOV network node policies: + + **Parameters**: + ```json + { + "apiVersion": "sriovnetwork.openshift.io/v1", + "kind": "SriovNetworkNodePolicy", + "namespace": "openshift-sriov-network-operator" + } + ``` + + Check SR-IOV device plugin pods: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "openshift-sriov-network-operator" + } + ``` + + Filter for pods with "device-plugin" in name. + + Check available SR-IOV devices on node: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "Node", + "name": "" + } + ``` + + Review `.status.allocatable` for SR-IOV resources. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get sriovnetworknodepolicy -n openshift-sriov-network-operator + oc get pods -n openshift-sriov-network-operator | grep device-plugin + oc describe node | grep -A 10 "Allocatable:" + ``` + +7. **Recreate VM with corrected network configuration** (if needed): + + If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +After remediation, check virt-launcher pod network status: + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +Should show all attached networks with status. Example output: +```json +[ + { + "name": "openshift-sdn", + "interface": "eth0", + "ips": ["10.128.2.10"], + "default": true + }, + { + "name": "vlan100", + "interface": "net1", + "ips": ["192.168.100.5"] + } +] +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**Check from inside VM** (via console): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# In guest OS: +ip addr show +# Should show all network interfaces (eth0, net1, etc.) +``` + +**Common Network Types**: +- **Linux Bridge**: Layer 2 bridge for VLAN networks +- **SR-IOV**: High-performance direct device assignment +- **macvlan**: MAC-based VLAN for container networks +- **OVN-Kubernetes**: OpenShift native overlay network + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/runtime-errors.md new file mode 100644 index 00000000..804d3cbc --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/runtime-errors.md @@ -0,0 +1,616 @@ +--- +title: VM Runtime Errors +category: kubevirt +sources: + - title: Kubernetes Pod Lifecycle + url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ + date_accessed: 2026-02-17 +tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] +semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] +use_cases: [vm-creation, vm-lifecycle, diagnostics] +related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Runtime Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. + +**When to use this document**: +- VM or virt-launcher pod shows `CrashLoopBackOff` status +- virt-launcher pod repeatedly restarting +- Guest OS kernel panics on boot + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### CrashLoopBackOff + +**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting + +**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. + +**Possible Causes**: +- Guest OS kernel panic on boot +- Insufficient resources (memory/CPU) for guest OS +- Corrupted disk image or filesystem +- QEMU/libvirt crashes due to configuration errors +- Missing or incompatible device drivers in guest +- Resource limits too low for virt-launcher pod +- Virtualization features (KVM) not available on node + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check virt-launcher pod restart count**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# Look at RESTARTS column +``` + +**2. View recent crash logs** (previous container instance): + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "previous": true, + "tail": 100 +} +``` + +Look for QEMU errors, kernel panics, or segfaults. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n virt-launcher--xxx --previous +``` + +**3. Check current virt-launcher logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. + +**CLI Fallback** (if MCP unavailable or all containers needed): +```bash +oc logs -n virt-launcher--xxx --all-containers +``` + +**4. Check VMI conditions for crash details**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Extract `.status.conditions` for crash details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**5. Check pod events for crash reasons**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" +``` + +**6. Check pod resource limits**: + +**MCP Tool**: `pods_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +Extract `.spec.containers[0].resources` for resource limits. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq +``` + +**7. Check node kubelet logs for OOM kills**: + +⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +oc adm node-logs -u kubelet | grep -i oom +``` + +**8. Access guest console** (if VM briefly starts): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Crash Patterns**: + +1. **Guest Kernel Panic**: + - Console logs show kernel panic messages + - Guest crashes immediately after boot + - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" + +2. **OOM (Out of Memory)**: + - Pod killed with reason: `OOMKilled` + - Guest runs out of memory during boot or operation + - virt-launcher logs show memory allocation failures + +3. **QEMU Crash**: + - virt-launcher logs show QEMU segmentation fault + - Symptoms: "qemu-system-x86_64: terminated by signal" + - Configuration incompatibility or QEMU bug + +4. **Disk Image Corruption**: + - Guest cannot boot from disk + - Filesystem errors in guest console + - DataVolume import failed + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for kernel panic or boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for: + - Kernel panic messages + - Initramfs errors + - Filesystem mounting failures + - Missing device errors + +2. **Review virt-launcher crash logs**: + + Use `pods_log` with `previous: true` from diagnostic step 2. + + Look for: + - QEMU command line errors + - Device initialization failures + - Memory allocation errors + - Signal termination (SIGSEGV, SIGABRT) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx --previous + ``` + +3. **Check for OOM (Out of Memory) kills**: + + **MCP Tool**: `pods_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + Extract `.status.containerStatuses[0].lastState.terminated.reason`. + + If returns `"OOMKilled"`: + - Option 1: Increase virt-launcher memory limits + - Option 2: Decrease guest memory allocation + - Option 3: Use smaller instance type + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' + ``` + +4. **Increase resources if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check current memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +5. **Verify disk image integrity**: + + **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters for list**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" + } + ``` + + **Parameters for specific DV**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check `.status.phase` (should be `Succeeded`). + + If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get datavolume -n -o jsonpath='{.status.phase}' + ``` + +6. **Check virtualization (KVM) availability**: + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +7. **Simplify VM configuration** (eliminate variables): + + Try creating minimal VM using vm-create skill with: + - Small instance type + - No secondary networks + - Simple container disk (e.g., Fedora) + - No cloud-init + + If minimal VM works, add features back one by one. + +8. **Recreate VM with different workload** (test disk image): + + If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) + +After remediation, check pod restart count stops increasing: + +**Parameters for pods**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). + +Check VM reaches Running state: + +**Parameters for VM**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# RESTARTS should stabilize + +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Running +``` + +Verify guest is responsive: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# Should show login prompt or OS console +``` + +**Advanced Debugging**: + +**MCP Tool**: `pods_exec` (from openshift-virtualization) + +Check libvirt domain XML: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["virsh", "dumpxml", "1"] +} +``` + +Check QEMU process: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["ps", "aux"] +} +``` + +Filter output for "qemu" process. + +**CLI Fallback** (if MCP unavailable): +```bash +oc exec -n virt-launcher--xxx -- virsh dumpxml 1 +oc exec -n virt-launcher--xxx -- ps aux | grep qemu +``` + +**Prevention**: +- Start with minimal VM configuration and add complexity gradually +- Use recommended instance types for your workload +- Test disk images before deploying to production +- Ensure nodes have adequate resources and KVM support +- Monitor resource usage and set appropriate limits + +--- + +## Workaround Patterns for MCP Tool Limitations + +### General Pattern: Diagnose → Propose → Confirm → Execute + +When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): + +**1. Diagnose** the root cause using Kubernetes commands +```bash +oc describe vm -n +oc get events -n --field-selector involvedObject.name= +``` + +**2. Propose** a manual workaround to the user with clear explanation +- Explain why the issue occurred +- Show what will be changed +- List alternative options + +**3. Confirm** with user before executing (Human-in-the-Loop) +- Wait for explicit "yes" or "apply workaround" +- Never auto-execute modifications + +**4. Execute** the workaround using oc/kubectl +```bash +oc patch vm -n --type=merge -p '...' +``` + +**5. Verify** the fix was successful +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**6. Document** the limitation and suggest filing enhancement request +- Note this is temporary until MCP tool is enhanced +- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues + +### Example: Adding Tolerations Workaround + +**Diagnostic Output**: +```markdown +## ⚠️ VM Scheduling Issue Detected + +**Root Cause**: Node taints prevent VM scheduling + +**Details**: +- Found 3 nodes with taint: `virtualization=true:NoSchedule` +- VM spec does not include matching tolerations +- This prevents VM scheduling on virtualization-dedicated nodes +``` + +**Proposed Workaround**: +```bash +oc patch vm web-server -n vms --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' +``` + +**User Confirmation Required**: +``` +How would you like to proceed? +- "apply workaround" - I'll patch the VM with tolerations +- "manual" - I'll provide instructions for you to apply manually +- "cancel" - Delete the VM and abort creation +``` + +**After Execution**: +```markdown +## ✓ Workaround Applied Successfully + +**Action**: Added tolerations for taint `virtualization=true:NoSchedule` +**New Status**: Stopped (VM can now be scheduled) + +**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. +Future VMs in this cluster will need the same fix until the tool is enhanced. +``` + +--- + +## VM Status Reference + +### Status Values + +| Status | Meaning | Action Required | +|--------|---------|-----------------| +| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | +| `Running` | VM is running | Normal | +| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | +| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | +| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | +| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | +| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | +| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | +| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | +| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | + +### Checking VM Status + +```bash +# Get printable status +oc get vm -n -o jsonpath='{.status.printableStatus}' + +# Get detailed status and conditions +oc get vm -n -o jsonpath='{.status}' | jq + +# Watch status changes in real-time +oc get vm -n -w +``` + +--- + +## Best Practices for Agents + +When implementing diagnostic workflows: + +1. **Always verify VM status** after creation (wait 5-10 seconds first) +2. **Consult this document** when encountering error status values +3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) +4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) +5. **Respect human-in-the-loop** for all VM modifications +6. **Document temporary workarounds** and their limitations clearly +7. **Suggest filing issues** for missing MCP tool features + +### Document Consultation Pattern + +```markdown +**Document Consultation** (REQUIRED): +1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes +2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." +``` + +--- + +## Known MCP Tool Limitations + +### vm_create tool + +**Currently Supported**: +- ✓ Namespace, name (required) +- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) +- ✓ Size hints (small, medium, large) +- ✓ Storage size +- ✓ Autostart flag +- ✓ Networks (Multus NetworkAttachmentDefinitions) +- ✓ Performance family (u1, o1, c1, m1) +- ✓ Instance type, preference + +**Not Currently Supported** (requires workarounds): +- ✗ Tolerations (for node taints) +- ✗ Node selectors +- ✗ Affinity/anti-affinity rules +- ✗ Resource requests/limits (beyond instance type) +- ✗ Custom labels/annotations +- ✗ SSH keys injection +- ✗ Cloud-init user data + +**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. + +**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues + +--- + +## Additional Resources + +- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) +- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) +- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) +- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) + +--- + +[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/scheduling-errors.md new file mode 100644 index 00000000..e47c8c4f --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/scheduling-errors.md @@ -0,0 +1,417 @@ +--- +title: VM Scheduling Errors +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 +tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] +semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] +use_cases: [vm-creation, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Scheduling Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**When to use this document**: +- VM shows status `ErrorUnschedulable` after creation or start attempt +- VM events mention scheduling failures, taints, resources, or node selectors + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +## ErrorUnschedulable + +**Symptom**: VM shows status `ErrorUnschedulable` after creation + +**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**Possible Causes**: + +### 1. Node Taints (Most Common) + +Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM events for scheduling failures**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `involvedObject.name` == "" and look for messages like: +- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= +``` + +**2. Check node taints in the cluster**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' +``` + +**Common Taint Patterns**: +- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule +- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes +- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads + +**Solution - Add Tolerations to VM**: + +The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. + +**Workaround (post-creation using MCP Tools)**: + +**Step 1**: Get current VM spec + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Step 2**: Modify the returned JSON to add tolerations + +Add to `.spec.template.spec.tolerations`: +```json +{ + "tolerations": [ + { + "key": "virtualization", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule" + } + ] +} +``` + +**Step 3**: Update VM with modified spec + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +```json +{ + "resource": "" +} +``` + +Pass the complete modified VM resource as YAML or JSON string. + +**Step 4**: Verify tolerations were added + +Use `resources_get` again and check `.spec.template.spec.tolerations` in response. + +**Step 5**: Check if VM status improved + +Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. + +**CLI Fallback** (if MCP patch is too complex): +```bash +# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' + +# Verify tolerations +oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq + +# Check status +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**Example - Multiple Tolerations**: +```bash +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: "dedicated" + operator: "Equal" + value: "virt-workloads" + effect: "NoSchedule" +' +``` + +**Toleration Operators**: +- `Equal` - Key and value must match exactly +- `Exists` - Only key must exist (ignores value) + +**Toleration Effects**: +- `NoSchedule` - Don't schedule new pods (existing pods continue) +- `PreferNoSchedule` - Avoid scheduling if possible +- `NoExecute` - Don't schedule AND evict existing pods + +**Alternative Solutions**: +1. **Remove node taints** (if you have cluster-admin access): + ```bash + oc adm taint nodes virtualization=true:NoSchedule- + ``` + +2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits + +3. **File enhancement request** - Request tolerations support in openshift-mcp-server: + https://github.com/openshift/openshift-mcp-server/issues + +--- + +### 2. Insufficient Resources + +Not enough CPU, memory, or storage available on any node. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM resource requests**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' +``` + +**2. Check node resource availability**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. + +Alternatively, use `nodes_top` MCP tool for current resource usage. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe nodes | grep -A 5 "Allocated resources" +``` + +**3. Look for VM events mentioning "Insufficient"**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep "Insufficient" +``` + +**Example Event**: +``` +0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. +``` + +**Solutions** (Use MCP Tools First): + +1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) +2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill +3. **Delete unused VMs** - Use vm-delete skill to free up resources +4. **Check resource quotas**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters for quota**: + ```json + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" + } + ``` + + **Parameters for limit range**: + ```json + { + "apiVersion": "v1", + "kind": "LimitRange", + "namespace": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + oc describe limitrange -n + ``` + +--- + +### 3. Node Selector Mismatch + +VM requires specific node labels that don't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM node selector requirements**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.nodeSelector` to see required node labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' +``` + +**2. List available node labels**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.metadata.labels` for available labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes --show-labels +``` + +**3. Check if any nodes match the selector**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node", + "labelSelector": "=" +} +``` + +Should return at least one node with matching labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -l = +``` + +**Solutions** (Use MCP Tools First): + +**Option 1: Remove node selector from VM** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get current VM using `resources_get` (diagnostic step 1) +2. Remove `.spec.template.spec.nodeSelector` field +3. Update VM using `resources_create_or_update` with modified JSON + +**CLI Fallback** (JSON patch easier via CLI): +Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" +```bash +oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' +``` + +**Option 2: Add label to nodes** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get node using `resources_get` +2. Add label to `.metadata.labels` +3. Update node using `resources_create_or_update` + +⚠️ **Note**: Node labeling typically requires cluster admin privileges. + +**CLI Fallback** (simpler via CLI): +Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" +```bash +oc label node = +``` + +--- + +[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/storage-errors.md new file mode 100644 index 00000000..16ba584d --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/storage-errors.md @@ -0,0 +1,1011 @@ +--- +title: VM Storage Errors +category: kubevirt +sources: + - title: KubeVirt CDI - DataVolumes + url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ + date_accessed: 2026-02-17 + - title: Kubernetes Persistent Volumes + url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ + date_accessed: 2026-02-17 +tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] +semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] +use_cases: [vm-creation, vm-deletion, vm-cloning] +related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] +last_updated: 2026-02-17 +--- + +# VM Storage Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. + +**When to use this document**: +- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` +- Storage deletion fails after VM deletion +- DataVolume cloning operations fail +- PVC provisioning issues + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### ErrorDataVolumeNotReady + +**Symptom**: VM shows status `ErrorDataVolumeNotReady` + +**Description**: The DataVolume (persistent storage) backing the VM is not ready. + +**Possible Causes**: + +#### 1. DataVolume Still Provisioning + +Storage provisioning takes time, especially for large disks or when importing images. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Get detailed DataVolume information**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` and `.status.conditions` for provisioning details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o json +``` + +**3. Check PVC (PersistentVolumeClaim) bound status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Check `.status.phase` for each PVC (should be `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. + +#### 2. Storage Class Not Found + +The requested storage class doesn't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List available storage classes**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" +} +``` + +Review the list of available storage classes (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass +``` + +**2. Check DataVolume's requested storage class**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.spec.pvc.storageClassName` in the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' +``` + +**Solution**: +1. Use a valid storage class from the cluster +2. Recreate VM with correct storage class parameter + +#### 3. Insufficient Storage Quota + +Namespace has insufficient storage quota to provision the PVC. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check resource quotas**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**2. Check storage usage**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase +``` + +**Solution**: +1. Request quota increase from cluster admin +2. Delete unused PVCs to free quota +3. Reduce VM storage size + +--- + +### ErrorPvcNotFound + +**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List PVCs in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review the list of available PVCs (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check VM's PVC references**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' +``` + +**Solution**: +- Wait for DataVolume to create the PVC +- Manually create missing PVC +- Fix VM spec to reference correct PVC name + +--- + + +--- + +### Storage Deletion Failures + +**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace + +**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. + +**Possible Causes**: +- PVC still bound to active PersistentVolume with `Retain` policy +- DataVolume still being referenced by another resource +- CDI (Containerized Data Importer) controller issues +- Storage class retention policy preventing deletion +- Finalizers on PVC/DataVolume blocking cleanup +- PVC still mounted by a pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check PVC status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check specific PVC phase**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Released` or `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o jsonpath='{.status.phase}' +``` + +**3. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**4. Check what's using the PVC**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' +``` + +**5. Check PVC finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. + +**6. Check DataVolume finalizers**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.metadata.finalizers}' +``` + +**7. Check PV reclaim policy**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume" +} +``` + +Filter results for PV where `.spec.claimRef.name` matches ``. + +To get specific PV policy: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume", + "name": "" +} +``` + +Check `.spec.persistentVolumeReclaimPolicy`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pv | grep +oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' +``` + +**Common Finalizer Patterns**: +- `kubernetes.io/pvc-protection` - Protects PVC while in use +- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer + +**Solutions** (Use MCP Tools First): + +1. **Delete DataVolume first, then PVC**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete DataVolume first (often blocks PVC deletion): + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Wait a few seconds, then delete PVC: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete datavolume -n + oc delete pvc -n + ``` + +2. **Check for pods still using PVC**: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "" + } + ``` + + Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. + + Then delete the pods using `pods_delete`: + + **Parameters**: + ```json + { + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' + oc delete pod -n + ``` + +3. **Force delete PVC** (if safe to do so): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete pvc -n --grace-period=0 --force + ``` + +4. **Remove finalizers from PVC** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PVC using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update PVC using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Remove finalizers from DataVolume** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current DataVolume using `resources_get` (diagnostic step 6) + 2. Remove items from `.metadata.finalizers` array + 3. Update DataVolume using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Change PV reclaim policy** (if PV has Retain policy): + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PV policy using `resources_get` (diagnostic step 7) + 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` + 3. Update PV using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" + ```bash + oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' + oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' + ``` + +**Storage Quota Check** (Use MCP Tools First): + +After deletion, verify storage quota is freed: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters for quota check**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.used` to verify storage quota is freed. + +**Parameters for PVC verification**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +oc get pvc -n +``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Confirm PVC is deleted: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Confirm DataVolume is deleted: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +# Should return: Error from server (NotFound) + +oc get datavolume -n +# Should return: Error from server (NotFound) +``` + +--- + +### DataVolume Cloning Failures + +**Symptom**: VM clone created successfully but DataVolume clone operation fails + +**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. + +**Possible Causes**: +- CSI driver doesn't support volume cloning +- Source PVC storage class incompatible with cloning +- Cross-namespace cloning not permitted by storage backend +- Insufficient storage quota in target namespace +- Source PVC not in `Bound` state +- Storage class doesn't have volume cloning enabled +- CDI (Containerized Data Importer) controller issues + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume clone status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and `.metadata.name`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Check DataVolume events for errors**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe datavolume -n +``` + +**3. Check DataVolume phase**: + +Use `resources_get` from step 1, extract `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +``` + +**4. Check if storage class supports cloning**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "name": "" +} +``` + +Review the full YAML output for cloning-related configurations. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass -o yaml | grep -A 5 -i clone +``` + +**5. Check CSI driver capabilities**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "CSIDriver" +} +``` + +Review `.items[].metadata.name` for available CSI drivers. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get csidriver +``` + +**6. Check source PVC status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Bound` for cloning to work). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**7. Check target namespace storage quota**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**8. Check CDI controller logs**: + +**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) + +First, list pods in openshift-cnv namespace: + +**Parameters for pods_list_in_namespace**: +```json +{ + "namespace": "openshift-cnv", + "labelSelector": "app.kubernetes.io/component=cdi-deployment" +} +``` + +Then get logs using `pods_log`: + +**Parameters**: +```json +{ + "namespace": "openshift-cnv", + "name": "", + "tail": 100 +} +``` + +**CLI Fallback** (if MCP unavailable or easier via CLI): +```bash +oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') +``` + +**Common Error Messages**: +- `"volume cloning is not supported"` - CSI driver lacks clone capability +- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage +- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace +- `"insufficient quota"` - Target namespace lacks storage quota +- `"source PVC not bound"` - Source PVC must be in Bound state for cloning +- `"StorageClass does not support cloning"` - Storage class configuration issue + +**Solutions** (Use MCP Tools First): + +1. **Check storage class clone support**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" + } + ``` + + For each storage class in `.items[]`, check: + - `.metadata.name` (storage class name) + - `.provisioner` (CSI driver) + + Storage classes using CSI drivers typically support cloning. Look for provisioners like: + - `csi.ovirt.org` (oVirt CSI) + - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) + - `ebs.csi.aws.com` (AWS EBS CSI) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner + ``` + +2. **Verify source PVC is bound**: + + Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pvc -n -o jsonpath='{.status.phase}' + ``` + +3. **Check target namespace quota**: + + Use `resources_list` from diagnostic step 7 to check quota. + + If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + ``` + +4. **Use snapshot-based cloning** (alternative method): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Step 1**: Create VolumeSnapshot of source PVC + + **Parameters**: + ```json + { + "apiVersion": "snapshot.storage.k8s.io/v1", + "kind": "VolumeSnapshot", + "metadata": { + "name": "-snapshot", + "namespace": "" + }, + "spec": { + "source": { + "persistentVolumeClaimName": "" + } + } + } + ``` + + **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) + + **Step 3**: Create new DataVolume from snapshot + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": "-rootdisk", + "namespace": "" + }, + "spec": { + "source": { + "snapshot": { + "name": "-snapshot", + "namespace": "" + } + }, + "storage": { + "resources": { + "requests": { + "storage": "50Gi" + } + }, + "storageClassName": "" + } + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" + ```bash + cat <-snapshot + namespace: + spec: + source: + persistentVolumeClaimName: + EOF + + oc get volumesnapshot -snapshot -n + + cat <-rootdisk + namespace: + spec: + source: + snapshot: + name: -snapshot + namespace: + storage: + resources: + requests: + storage: 50Gi + storageClassName: + EOF + ``` + +5. **Use "new empty storage" option** (vm-clone skill): + - If cloning isn't supported, create VM with empty storage + - Manually copy data if needed + +6. **Cross-namespace cloning workaround**: + - Some storage backends require snapshot for cross-namespace cloning + - Create snapshot in source namespace, restore in target namespace (see solution 4 above) + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Check DataVolume reached Succeeded phase: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should return `Succeeded`). + +Check PVC was created and bound: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "-rootdisk" +} +``` + +Check `.status.phase` (should return `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +# Should return: Succeeded + +oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' +# Should return: Bound +``` + +**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/mcp-servers/mock-virt-mcp.py b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..1d1132df --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1500 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), + ("prod-vms", {"env": "production"}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── prod-vms (instruction-specific) ────────────────────────────────── + _vm("production-db", "prod-vms", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true"}, + 8, 16, "Running", True, 1), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, + # ── prod-vms / production-db (instruction-specific) ─────────────────── + { + "name": "production-db-backup-20260210", + "namespace": "prod-vms", + "vm_name": "production-db", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-10T08:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-proddb-root-20260210"}, + ], + }, + { + "name": "production-db-snap-20260218", + "namespace": "prod-vms", + "vm_name": "production-db", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-18T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-proddb-root-20260218"}, + ], + }, + { + "name": "production-db-snap-failed", + "namespace": "prod-vms", + "vm_name": "production-db", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-22T11:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/skills/vm-snapshot-list/SKILL.md b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/skills/vm-snapshot-list/SKILL.md new file mode 100644 index 00000000..3f199f8f --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/skills/vm-snapshot-list/SKILL.md @@ -0,0 +1,402 @@ +--- +name: vm-snapshot-list +description: | + List virtual machine snapshots across namespaces with status, age, and recovery information. + + Use when: + - "List snapshots for VM [name]" + - "Show snapshots in namespace [name]" + - "What snapshots exist for [vm]?" + + Read-only operation - no user confirmation required. + + NOT for creating/deleting snapshots (use vm-snapshot-create/delete instead). + +model: inherit +color: cyan +--- + +# /vm-snapshot-list Skill + +List virtual machine snapshots in OpenShift Virtualization. This read-only skill displays snapshot information including status, age, size, and recovery options. + +## Prerequisites + +**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools**: +- `resources_list` (from openshift-virtualization) - List VirtualMachineSnapshot resources +- `resources_get` (from openshift-virtualization) - Get snapshot details + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster (>= 4.19) +- OpenShift Virtualization operator installed +- ServiceAccount with RBAC permissions to list VirtualMachineSnapshot resources + +### Prerequisite Verification + +**Before executing, verify MCP server availability:** + +1. **Check MCP Server Configuration** + - Verify `openshift-virtualization` exists in `.mcp.json` + - If missing → Report to user with setup instructions + +2. **Check Environment Variables** + - Verify `KUBECONFIG` is set (check presence only, never expose value) + - If missing → Report to user + +**Human Notification Protocol:** + +When prerequisites fail: + +``` +❌ Cannot execute vm-snapshot-list: MCP server 'openshift-virtualization' is not available + +📋 Setup Instructions: +1. Add openshift-virtualization to .mcp.json +2. Set KUBECONFIG environment variable +3. Restart Claude Code to reload MCP servers + +🔗 Documentation: https://github.com/openshift/openshift-mcp-server +``` + +## When to Use This Skill + +**Trigger this skill when:** +- User wants to list available snapshots for recovery +- User wants to see snapshot status and age +- User wants to verify snapshot existence before restore +- User wants to identify old snapshots for deletion + +**User phrases that trigger this skill:** +- "List all snapshots for web-server VM" +- "Show snapshots in namespace production" +- "What snapshots exist?" +- "Display VM snapshots" + +**Do NOT use this skill when:** +- User wants to create a snapshot → Use `vm-snapshot-create` skill +- User wants to restore from snapshot → Use `vm-snapshot-restore` skill +- User wants to delete snapshots → Use `vm-snapshot-delete` skill + +## Workflow + +### Step 1: Gather Information + +**Required Information from User:** +1. **Namespace** - Namespace to list snapshots from +2. **VM Name** (Optional) - Filter snapshots by specific VM + +If user doesn't provide namespace, ask for it. + +### Step 2: List Snapshots + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters** (with VM filter using label selector): +```json +{ + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "namespace": "", + "labelSelector": "vm.kubevirt.io/name=" +} +``` + +**Parameters** (all snapshots in namespace): +```json +{ + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "namespace": "" +} +``` + +**Note**: The label selector `vm.kubevirt.io/name=` may not always exist. If no results are returned, fall back to listing all snapshots and filtering by checking `spec.source.name` field in the results. + +**Expected Output**: List of VirtualMachineSnapshot resources + +**Parse each snapshot to extract**: +- `metadata.name` - Snapshot name +- `metadata.namespace` - Namespace +- `metadata.creationTimestamp` - Creation time +- `spec.source.name` - VM name +- `status.phase` - Status (InProgress, Succeeded, Failed) +- `status.readyToUse` - Ready for restore (true/false) + +**Error Handling**: +- If namespace not found → Report error +- If permission denied → Report RBAC error +- If no snapshots found → Report "No snapshots found" + +### Step 3: Report Snapshot List + +**If snapshots found:** + +```markdown +## VM Snapshots + +**Namespace**: `` + +**VM**: `` + + +### Available Snapshots + +| Snapshot Name | VM Name | Status | Created | Age | ReadyToUse | +|---------------|---------|--------|---------|-----|------------| +| `pre-upgrade-snapshot` | `database-01` | Succeeded ✓ | 2024-01-15 10:30 | 2 days | true | +| `backup-snapshot` | `database-01` | Succeeded ✓ | 2024-01-10 08:00 | 7 days | true | +| `test-snapshot` | `web-server` | Succeeded ✓ | 2024-01-14 14:20 | 3 days | true | + +**Total Snapshots**: 3 + +--- + +### Snapshot Details + +**Snapshot: `pre-upgrade-snapshot`** +- **VM**: `database-01` +- **Status**: Succeeded ✓ +- **Created**: 2024-01-15 10:30:00 UTC +- **Age**: 2 days +- **Ready to Use**: true + +**Snapshot: `backup-snapshot`** +- **VM**: `database-01` +- **Status**: Succeeded ✓ +- **Created**: 2024-01-10 08:00:00 UTC +- **Age**: 7 days +- **Ready to Use**: true + +**Snapshot: `test-snapshot`** +- **VM**: `web-server` +- **Status**: Succeeded ✓ +- **Created**: 2024-01-14 14:20:00 UTC +- **Age**: 3 days +- **Ready to Use**: true + +--- + +### Actions + +**To restore from a snapshot:** +``` +"Restore VM from snapshot " +``` + +**To delete a snapshot:** +``` +"Delete snapshot " +``` + +**To create a new snapshot:** +``` +"Create snapshot of VM " +``` +``` + +**If no snapshots found:** + +```markdown +## VM Snapshots + +**Namespace**: `` + +**VM**: `` + + +**No snapshots found.** + + +No snapshots exist for VM `` in namespace ``. + + +No snapshots exist in namespace ``. + + +**To create a snapshot:** +``` +"Create snapshot of VM " +``` +``` + +## Common Issues + +### Issue 1: Permission Denied + +**Error**: "Forbidden: User lacks permissions to list virtualmachinesnapshots" + +**Cause**: Missing RBAC permissions for listing snapshots. + +**Solution:** +1. Check permissions: `oc auth can-i list virtualmachinesnapshots -n ` +2. Contact cluster admin to grant list/get permissions for virtualmachinesnapshots +3. Try listing in a different namespace where you have permissions + +### Issue 2: No Snapshots Found + +**Error**: "No snapshots exist in namespace ``" + +**Cause**: Namespace has no snapshots, or wrong namespace. + +**Solution:** +1. Verify correct namespace name +2. List snapshots without VM filter to see all snapshots +3. Check other namespaces: Use `namespaces_list` to see available namespaces +4. Check if snapshots were recently deleted: Use `events_list` in namespace + +### Issue 3: Snapshot Shows Failed Status + +**Error**: Snapshot listed but `status.phase: Failed` or `readyToUse: false` + +**Cause**: Snapshot creation failed due to storage issues, hot-plugged volumes, or missing VolumeSnapshotClass. + +**Solution:** +1. Get snapshot details: Use `resources_get` to check `status.conditions` for error messages +2. Check cluster events: Use `events_list` for snapshot-related errors +3. Common fixes: + - VolumeSnapshotClass missing: Contact cluster admin + - Hot-plugged volumes: Stop VM, persist volumes, recreate snapshot + - Failed permanently: Delete and recreate snapshot + +## Dependencies + +### Required MCP Servers +- `openshift-virtualization` - OpenShift MCP server with kubevirt toolset + +### Required MCP Tools +- `resources_list` (from openshift-virtualization) - List Kubernetes resources by type + - Used for: Listing VirtualMachineSnapshot resources + - Parameters: apiVersion, kind, namespace, labelSelector (optional) + - Source: https://github.com/openshift/openshift-mcp-server + +### Related Skills +- `vm-snapshot-create` - Create VM snapshots +- `vm-snapshot-restore` - Restore VMs from snapshots +- `vm-snapshot-delete` - Delete VM snapshots +- `vm-inventory` - List VMs before creating snapshots + +### Reference Documentation +- [OpenShift Virtualization Snapshots](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt-managing-vm-snapshots) +- [KubeVirt VM Snapshots](https://kubevirt.io/user-guide/operations/snapshot_restore_api/) + +## Security Considerations + +- **RBAC Enforcement**: Requires list permissions for VirtualMachineSnapshot resources +- **Namespace Isolation**: Only lists snapshots within specified namespace +- **Read-Only Operation**: No modifications or destructive actions +- **KUBECONFIG Security**: Credentials never exposed in output + +## Example Usage + +### Example 1: List Snapshots for Specific VM + +``` +User: "List all snapshots for database-01" + +Agent: "Which namespace is database-01 in?" + +User: "production" + +Agent: [Invokes vm-snapshot-list skill] + [Lists snapshots with VM filter] + +## VM Snapshots + +**Namespace**: `production` +**VM**: `database-01` + +### Available Snapshots + +| Snapshot Name | VM Name | Status | Created | Age | ReadyToUse | +|---------------|---------|--------|---------|-----|------------| +| `database-01-pre-upgrade` | `database-01` | Succeeded ✓ | 2024-01-15 10:30 | 2 hours | true | +| `database-01-daily-backup` | `database-01` | Succeeded ✓ | 2024-01-14 02:00 | 1 day | true | + +**Total Snapshots**: 2 + +--- + +### Snapshot Details + +**Snapshot: `database-01-pre-upgrade`** +- **VM**: `database-01` +- **Status**: Succeeded ✓ +- **Created**: 2024-01-15 10:30:00 UTC +- **Age**: 2 hours +- **Ready to Use**: true + +**Snapshot: `database-01-daily-backup`** +- **VM**: `database-01` +- **Status**: Succeeded ✓ +- **Created**: 2024-01-14 02:00:00 UTC +- **Age**: 1 day +- **Ready to Use**: true + +--- + +### Actions + +**To restore from a snapshot:** +"Restore VM database-01 from snapshot " + +**To delete a snapshot:** +"Delete snapshot " + +**To create a new snapshot:** +"Create snapshot of VM database-01" +``` + +### Example 2: List All Snapshots in Namespace + +``` +User: "Show me all snapshots in namespace production" + +Agent: [Invokes vm-snapshot-list skill] + [Lists all snapshots without VM filter] + +## VM Snapshots + +**Namespace**: `production` + +### Available Snapshots + +| Snapshot Name | VM Name | Status | Created | Age | ReadyToUse | +|---------------|---------|--------|---------|-----|------------| +| `database-01-pre-upgrade` | `database-01` | Succeeded ✓ | 2024-01-15 10:30 | 2 hours | true | +| `database-01-daily-backup` | `database-01` | Succeeded ✓ | 2024-01-14 02:00 | 1 day | true | +| `web-server-backup` | `web-server` | Succeeded ✓ | 2024-01-13 03:00 | 2 days | true | + +**Total Snapshots**: 3 + +VMs with snapshots: `database-01` (2), `web-server` (1) +``` + +### Example 3: No Snapshots Found + +``` +User: "List snapshots for test-vm" + +Agent: "Which namespace is test-vm in?" + +User: "dev" + +Agent: [Invokes vm-snapshot-list skill] + [Lists snapshots - none found] + +## VM Snapshots + +**Namespace**: `dev` +**VM**: `test-vm` + +**No snapshots found.** + +No snapshots exist for VM `test-vm` in namespace `dev`. + +**To create a snapshot:** +"Create snapshot of VM test-vm" +``` diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/instruction.md b/evaluation/with_skills/rh-virt__vm-snapshot-list/instruction.md new file mode 100644 index 00000000..2c6ed187 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-list/instruction.md @@ -0,0 +1,12 @@ +# VM Snapshot Listing Task + +You are an OpenShift Virtualization administrator. List and inspect all snapshots for VM `production-db` in namespace `prod-vms`. + +## Requirements +- List all snapshots with their status and readiness +- Show creation timestamps +- Identify any failed or incomplete snapshots + +Use MCP tools to query snapshot data. Document your methodology and write the snapshot inventory in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/solution/solve.sh b/evaluation/with_skills/rh-virt__vm-snapshot-list/solution/solve.sh new file mode 100644 index 00000000..2e33f350 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-list/solution/solve.sh @@ -0,0 +1,30 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Snapshot Inventory + +## Snapshots for production-db in prod-vms + +### Query Method +- API: `resources_list(apiVersion="snapshot.kubevirt.io/v1beta1", kind="VirtualMachineSnapshot", namespace="prod-vms")` +- Filter: `labelSelector: vm.kubevirt.io/name=production-db` +- Fallback: If label missing, filter by `spec.source.name == "production-db"` + +### Snapshot List +| Name | Status | Ready | Created | +|------|--------|-------|---------| +| production-db-backup-20240301 | Succeeded | true | 2024-03-01T10:00:00Z | +| production-db-backup-20240215 | Succeeded | true | 2024-02-15T08:30:00Z | + +### Status Fields +- `status.phase`: InProgress, Succeeded, Failed +- `status.readyToUse`: true/false — snapshot can be used for restore +- `spec.source.name`: Source VM name +- `metadata.creationTimestamp`: Creation time + +### Actions +- Restore: "Restore VM production-db from snapshot " +- Delete: "Delete snapshot " + +### No failed or incomplete snapshots found. + +REPORT_EOF diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/task.toml b/evaluation/with_skills/rh-virt__vm-snapshot-list/task.toml new file mode 100644 index 00000000..3e9cc1cd --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-list/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-snapshot-list" +name = "rh-virt VM Snapshot Listing Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-snapshot-list", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/tests/llm_judge.py b/evaluation/with_skills/rh-virt__vm-snapshot-list/tests/llm_judge.py new file mode 100644 index 00000000..aa42d89d --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-list/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "ready_to_use_status", "file": "/root/report.md", "question": "Does the report show readyToUse status indicating which snapshots are safe to restore?", "reference": "A skilled report includes readyToUse for each snapshot. An unskilled report only shows names and dates."}, + {"id": "phase_and_creation", "file": "/root/report.md", "question": "Does the report show status.phase (Succeeded/Failed/InProgress) and creation timestamp for each snapshot?", "reference": "A skilled report includes phase and timestamp. An unskilled report shows minimal snapshot metadata."}, + {"id": "label_selector_filtering", "file": "/root/report.md", "question": "Does the report mention using the vm.kubevirt.io/name label or label selector to filter or identify snapshots belonging to a specific VM?", "reference": "A skilled report references the vm.kubevirt.io/name label for filtering snapshots by source VM, or shows label selector parameters in API calls. An unskilled report lists snapshots without mentioning the KubeVirt label-based filtering mechanism."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/tests/test.sh b/evaluation/with_skills/rh-virt__vm-snapshot-list/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-list/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/tests/test_outputs.py b/evaluation/with_skills/rh-virt__vm-snapshot-list/tests/test_outputs.py new file mode 100644 index 00000000..06ac48d3 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-list/tests/test_outputs.py @@ -0,0 +1,62 @@ +""" +Tests for rh-virt__vm-snapshot-list per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_snapshots(self): + content = read_report().lower() + assert "snapshot" in content, "report should mention snapshots" + + def test_has_structured_output(self): + content = read_report() + assert "|" in content or "- " in content, "report should have structured output (table or list)" + + +class TestSkillDependent: + def test_ready_to_use_status(self): + """Skill: readyToUse status for restore readiness.""" + c = read_report().lower() + assert any(t in c for t in ["readytouse", "ready to use", "ready for restore"]), ( + "should reference readyToUse status for snapshot readiness" + ) + + def test_creation_timestamp(self): + """Skill: metadata.creationTimestamp or creation time.""" + c = read_report().lower() + assert any(t in c for t in ["creationtimestamp", "creation timestamp", "created", "when"]), ( + "should show creation timestamp for each snapshot" + ) + + def test_phase_status(self): + """Skill: status.phase (Succeeded, Failed, InProgress).""" + c = read_report().lower() + assert any(t in c for t in ["succeeded", "failed", "inprogress", "status.phase", "phase"]) and ( + "succeeded" in c or "failed" in c or "phase" in c + ), ( + "should show phase (Succeeded/Failed/InProgress)" + ) + + def test_label_selector_for_vm_filtering(self): + """Skill teaches using vm.kubevirt.io/name label selector to + filter snapshots by source VM. Without skill, agents list all + snapshots without label-based filtering.""" + c = read_report() + assert "vm.kubevirt.io" in c or "labelSelector" in c or "label selector" in c.lower(), ( + "should reference vm.kubevirt.io/name label for snapshot filtering" + ) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/Dockerfile b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/Dockerfile new file mode 100644 index 00000000..ae625e01 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/Dockerfile @@ -0,0 +1,70 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY skills /root/.claude/skills +COPY docs /root/.claude/docs +COPY skills /root/.codex/skills +COPY docs /root/.codex/docs +COPY skills /root/.opencode/skill +COPY docs /root/.opencode/docs +COPY skills /root/.goose/skills +COPY docs /root/.goose/docs +COPY skills /root/.factory/skills +COPY docs /root/.factory/docs +COPY skills /root/.agents/skills +COPY docs /root/.agents/docs +COPY skills /root/.gemini/skills +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/.ai-index/semantic-index.json new file mode 100644 index 00000000..2512c2a3 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/.ai-index/semantic-index.json @@ -0,0 +1,148 @@ +{ + "index_version": "1.0.0", + "last_updated": "2026-02-17", + "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", + + "documents": [ + { + "file": "INDEX.md", + "title": "VM Troubleshooting Guide - Index", + "category": "navigation", + "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], + "use_cases": ["navigation", "tool-reference", "general-guidance"], + "token_estimate": 450, + "skills": ["all"] + }, + { + "file": "scheduling-errors.md", + "title": "VM Scheduling Errors", + "category": "scheduling", + "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], + "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], + "errors_covered": [ + "ErrorUnschedulable - Node Taints", + "ErrorUnschedulable - Insufficient Resources", + "ErrorUnschedulable - Node Selector Mismatch" + ], + "token_estimate": 820, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "storage-errors.md", + "title": "VM Storage Errors", + "category": "storage", + "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], + "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], + "errors_covered": [ + "ErrorDataVolumeNotReady - DataVolume Still Provisioning", + "ErrorDataVolumeNotReady - Storage Class Not Found", + "ErrorDataVolumeNotReady - Insufficient Storage Quota", + "ErrorPvcNotFound", + "Storage Deletion Failures", + "DataVolume Cloning Failures" + ], + "token_estimate": 1350, + "skills": ["vm-creator", "vm-delete", "vm-clone"] + }, + { + "file": "lifecycle-errors.md", + "title": "VM Lifecycle Errors", + "category": "lifecycle", + "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], + "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], + "errors_covered": [ + "VM Stuck in Terminating State", + "VM Won't Start (Non-Scheduling Issues)", + "VM Won't Stop" + ], + "token_estimate": 1150, + "skills": ["vm-delete", "vm-lifecycle-manager"] + }, + { + "file": "runtime-errors.md", + "title": "VM Runtime Errors", + "category": "runtime", + "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], + "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], + "errors_covered": [ + "CrashLoopBackOff" + ], + "token_estimate": 850, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "network-errors.md", + "title": "VM Network Errors", + "category": "networking", + "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], + "use_cases": ["vm-creation", "network-troubleshooting"], + "errors_covered": [ + "Network Attachment Failures" + ], + "token_estimate": 580, + "skills": ["vm-creator"] + } + ], + + "skill_to_docs_mapping": { + "vm-creator": { + "primary_docs": ["scheduling-errors.md", "storage-errors.md"], + "secondary_docs": ["runtime-errors.md", "network-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3650, + "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" + }, + "vm-delete": { + "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 2950, + "description": "VM deletion failures - stuck terminating, storage cleanup" + }, + "vm-clone": { + "primary_docs": ["storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 1800, + "description": "VM cloning failures - DataVolume cloning, storage provisioning" + }, + "vm-lifecycle-manager": { + "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], + "secondary_docs": ["runtime-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3270, + "description": "VM start/stop failures - lifecycle management, scheduling" + }, + "vm-inventory": { + "primary_docs": [], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 450, + "description": "General guidance for status interpretation" + } + }, + + "error_to_docs_mapping": { + "ErrorUnschedulable": "scheduling-errors.md", + "ErrorDataVolumeNotReady": "storage-errors.md", + "ErrorPvcNotFound": "storage-errors.md", + "Terminating": "lifecycle-errors.md", + "CrashLoopBackOff": "runtime-errors.md", + "NetworkAttachmentFailed": "network-errors.md" + }, + + "cross_references": { + "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], + "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], + "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], + "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], + "network-errors.md": ["scheduling-errors.md"] + }, + + "progressive_disclosure_strategy": { + "description": "Load only the minimum documentation needed for current task", + "pattern": "error_type → primary_doc → cross_references (if needed)", + "token_savings": "29% average reduction vs single-file (based on rh-sre data)", + "navigation_overhead_reduction": "85% vs single-file navigation" + } +} diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/INDEX.md new file mode 100644 index 00000000..a2e76963 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/INDEX.md @@ -0,0 +1,332 @@ +--- +title: VM Troubleshooting Guide - Index +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 + - title: OpenShift Virtualization - Virtual Machine Status + url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html + date_accessed: 2026-02-06 + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] +semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] +use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] +last_updated: 2026-02-17 +--- + +# VM Troubleshooting Guide - Index + +## Overview + +This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: + +- **Creation**: Scheduling failures, storage provisioning issues +- **Lifecycle**: Start/stop failures, stuck states +- **Deletion**: Resources stuck in Terminating, storage cleanup failures +- **Cloning**: DataVolume cloning errors, cross-namespace issues +- **Networking**: Secondary network attachment failures +- **Runtime**: CrashLoopBackOff, guest OS boot failures + +This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. + +--- + +## 🗂️ Troubleshooting Categories + +The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: + +### 1. [Scheduling Errors](scheduling-errors.md) +**When to use**: VM fails to schedule on any node + +**Errors covered**: +- ErrorUnschedulable - Node Taints +- ErrorUnschedulable - Insufficient Resources +- ErrorUnschedulable - Node Selector Mismatch + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 2. [Storage Errors](storage-errors.md) +**When to use**: VM has storage provisioning, deletion, or cloning issues + +**Errors covered**: +- ErrorDataVolumeNotReady (all 3 subsections) +- ErrorPvcNotFound +- Storage Deletion Failures +- DataVolume Cloning Failures + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### 3. [Lifecycle Errors](lifecycle-errors.md) +**When to use**: VM has start/stop/termination issues + +**Errors covered**: +- VM Stuck in Terminating State +- VM Won't Start (Non-Scheduling Issues) +- VM Won't Stop + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### 4. [Runtime Errors](runtime-errors.md) +**When to use**: VM repeatedly crashes or fails at runtime + +**Errors covered**: +- CrashLoopBackOff + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 5. [Network Errors](network-errors.md) +**When to use**: VM has secondary network attachment failures + +**Errors covered**: +- Network Attachment Failures (Multus, SR-IOV) + +**Skills that use this**: vm-create + +--- + +## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last + +### Critical Principle: MCP-First Approach + +**All diagnostic procedures in this guide follow the MCP-first pattern**: + +``` +1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) +2. ⚠️ IF FAILS: Ask user permission to use CLI command +3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval +``` + +### Why MCP Tools First? + +- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources +- **Consistency**: Same interface across all operations +- **Better Error Handling**: MCP tools return structured errors +- **Audit Trail**: MCP tool usage is logged and trackable + +### Command Pattern Examples + +Throughout this guide, you'll see diagnostic steps formatted like this: + +**✅ CORRECT Pattern (MCP First)**: + +```markdown +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Extract**: `.metadata.finalizers` from returned JSON + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` +``` + +### Available MCP Tools + +The `openshift-virtualization` MCP server provides these tools: + +**Resource Operations**: +- `resources_get` - Get specific resource (replaces `oc get `) +- `resources_list` - List resources (replaces `oc get `) +- `resources_delete` - Delete resource (replaces `oc delete`) +- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) + +**Pod Operations**: +- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) +- `pods_get` - Get pod details (replaces `oc get pod`) +- `pods_log` - Get pod logs (replaces `oc logs`) +- `pods_exec` - Execute in pod (replaces `oc exec`) +- `pods_delete` - Delete pod (replaces `oc delete pod`) +- `pods_top` - Pod resource usage (replaces `oc top pods`) + +**Events & Monitoring**: +- `events_list` - List events (replaces `oc get events`) +- `nodes_top` - Node resource usage (replaces `oc top nodes`) +- `nodes_log` - Node logs (replaces `oc adm node-logs`) +- `nodes_stats_summary` - Detailed node stats + +**VM Operations** (KubeVirt toolset): +- `vm_create` - Create VMs +- `vm_lifecycle` - Start/stop/restart VMs + +**Namespaces**: +- `namespaces_list` - List namespaces (replaces `oc get namespaces`) +- `projects_list` - List OpenShift projects (replaces `oc get projects`) + +### When CLI Commands Are Required + +Some operations have **NO MCP equivalent** and require CLI: + +- `virtctl` commands (console, VNC access) +- `oc debug node` (node debugging) +- `oc auth can-i` (permission checks) +- `oc adm taint` (node taint management) +- Complex JSON patch operations + +For these, the guide will note: **"CLI Only - No MCP equivalent"** + +### Quick Reference: CLI → MCP Mapping + +| CLI Command | MCP Tool Equivalent | +|-------------|---------------------| +| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | +| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | +| `oc logs -n ` | `pods_log` with name="", namespace="" | +| `oc get events -n ` | `events_list` with namespace="" | +| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | + +**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. + +### How to Read Diagnostic Sections + +Each error section includes: +1. **Symptom** - What you observe +2. **Description** - What's happening +3. **Possible Causes** - Why it's happening +4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback +5. **Solutions** - **MCP tools first**, then CLI fallback +6. **Verification** - **MCP tools first**, then CLI fallback + +**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. + +--- + +## 🔍 Quick Navigation by Skill + +**vm-create**: +- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics +- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning +- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff +- [Network Errors](network-errors.md) - Network attachment failures + +**vm-delete**: +- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state +- [Storage Errors](storage-errors.md) - Storage deletion failures + +**vm-clone**: +- [Storage Errors](storage-errors.md) - DataVolume cloning failures + +**vm-lifecycle-manager**: +- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop +- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling + +**vm-inventory**: +- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed + +--- + +## 📊 Documentation Coverage & Maintenance + +### Current Coverage + +This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: + +- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) +- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures +- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop +- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) +- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems + +**Total errors documented**: 12 error types across 6 categories + +--- + +### Encountering Undocumented Errors + +#### For AI Agents (Claude Code) + +If you encounter an error **not documented** in the categories above: + +1. **Report to user** with all available details (error message, affected resources, namespace) +2. **Provide best-effort diagnostics** using MCP tools: + - `resources_get` to inspect resource status + - `pods_log` to check virt-launcher or compute container logs + - `events_list` to view Kubernetes events +3. **Suggest documentation update**: + ``` + ⚠️ This error is not yet documented in the troubleshooting guide. + + **Error**: + **Affected resource**: / + + I recommend adding this error to the troubleshooting guide: + - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file + - If it's a new error class → Create new category file in docs/troubleshooting/ + + Would you like me to help document this error for future reference? + ``` + +4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs + +--- + +#### For Human Contributors + +To document a new error: + +1. **Determine the category**: + - Scheduling issue → `scheduling-errors.md` + - Storage problem → `storage-errors.md` + - Lifecycle/start/stop → `lifecycle-errors.md` + - Runtime crash → `runtime-errors.md` + - Network issue → `network-errors.md` + - New error class → Create new category file (e.g., `security-errors.md`) + +2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): + - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback + - **Solutions**: MCP Tool approach → Implementation → CLI Fallback + - **Verification**: MCP Tool checks → Expected results → CLI Fallback + +3. **Update semantic index**: + - Edit `.ai-index/semantic-index.json` + - Add error to `error_to_docs_mapping` + - Update relevant skill's `primary_docs` or `secondary_docs` + - Increment token estimates if needed + +4. **Reference in skill files**: + - Update skill's Reference Documentation section + - Ensure skill's Document Consultation steps point to the right category + +5. **Update this INDEX.md**: + - Add error to the appropriate category section (lines 50-120) + - Update "Current Coverage" list above + +--- + +## 📚 Additional Resources + +- [rh-virt README](../../README.md) - MCP server setup and skill documentation +- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources +- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/lifecycle-errors.md new file mode 100644 index 00000000..6167fcc8 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/lifecycle-errors.md @@ -0,0 +1,869 @@ +--- +title: VM Lifecycle Errors +category: kubevirt +sources: + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 +tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] +semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] +use_cases: [vm-deletion, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Lifecycle Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM lifecycle issues including start/stop failures and stuck termination states. + +**When to use this document**: +- VM stuck in `Terminating` state +- VM won't start (runStrategy is Always but VM never reaches Running) +- VM won't stop (runStrategy is Halted but VM never reaches Stopped) + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### VM Stuck in Terminating State + +**Symptom**: VM shows status `Terminating` but deletion never completes + +**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). + +**Possible Causes**: +- Finalizers blocking deletion +- PVC/DataVolume still attached and preventing cleanup +- VirtualMachineInstance (VMI) not terminating properly +- Custom controllers or operators blocking deletion +- Stuck virt-launcher pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check finalizers on the VM**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` + +**2. Check if VMI still exists**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name containing "virt-launcher-". + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check events for deletion issues**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results where `involvedObject.name` == "" and sort by timestamp. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' +``` + +**5. Check VM deletion timestamp**: + +Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. + +If present, VM is in deletion process. If null, VM is not being deleted. + +**Common Finalizer Patterns**: +- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer +- `foregroundDeletion` - Waits for dependent objects to be deleted +- Custom finalizers from operators + +**Solutions** (Use MCP Tools First): + +1. **Wait for dependent resources** (safest approach): + - Use `resources_list` to check if PVCs, DataVolumes are still deleting + - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) + +2. **Force delete VMI** (if VMI is stuck): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable - requires explicit user permission): + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +3. **Force delete virt-launcher pod** (if pod is stuck): + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + # Find the virt-launcher pod name first + oc get pods -n | grep virt-launcher- + + # Then delete it + oc delete pod virt-launcher--xxx -n --force --grace-period=0 + ``` + +4. **Remove finalizers** (⚠️ dangerous - use only as last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` + 2. Remove items from `.metadata.finalizers` array + 3. Update VM using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch not easily done via MCP): + ```bash + # This operation is complex for MCP - may need CLI + oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Check for protection labels** (vm-delete skill specific): + + Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. + + If value is "true", the vm-delete skill refuses deletion (this is expected behavior). + +**Verification**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error if deletion successful. + +**CLI Fallback**: +```bash +oc get vm -n +# Should return: Error from server (NotFound) +``` + +--- + + +--- + +### VM Won't Start (Non-Scheduling Issues) + +**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state + +**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. + +**Possible Causes**: +- Guest OS kernel panic or boot failure +- Cloud-init configuration errors +- Missing or corrupted disk image +- Insufficient memory for guest OS to boot +- QEMU/KVM virtualization errors +- VirtualMachineInstance (VMI) creation failures +- virt-launcher pod crashes + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VMI (VirtualMachineInstance) status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and overall VMI state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**2. Check VMI conditions for errors**: + +Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. View virt-launcher pod logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +First, get pod name from step 3, then: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +Look for QEMU errors, memory allocation failures, device errors. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**5. Check virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. + +Alternatively, use `pods_get` to get full pod details: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**6. Access VM console to see guest OS boot messages**: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +Look for kernel panic, initramfs errors, filesystem errors. + +**7. Check VM events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep -A 20 "Events:" +``` + +**Common Error Patterns**: + +1. **Guest OS Boot Failure**: + - Console shows kernel panic + - Guest hangs at GRUB or boot loader + - Cloud-init errors during first boot + +2. **Insufficient Memory**: + - Guest OS kills processes due to OOM + - VMI logs show memory allocation errors + +3. **QEMU Crashes**: + - virt-launcher logs show QEMU segfaults + - VMI repeatedly restarts + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for kernel panic, initramfs errors, filesystem errors. + +2. **Check virt-launcher pod logs for QEMU errors**: + + Use `pods_log` from diagnostic step 4 to view logs. + + Look for: + - "qemu-system-x86_64: ..." errors + - Memory allocation failures + - Device errors + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx + ``` + +3. **Increase memory if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check VMI memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +4. **Verify disk image integrity**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check PVC is bound: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + If using container disk, verify image exists and is accessible (check VMI spec). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get pvc -n + ``` + +5. **Check cloud-init configuration** (if applicable): + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. + + Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq + ``` + +6. **Restart VMI** (soft reset): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete VMI (VM controller will recreate it): + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for new VMI to start (use `resources_get` to check status). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n -w + ``` + +7. **Check virtualization extensions** (KVM): + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + + # In debug shell: + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should eventually return `Running`). + +Check VMI is running: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should show `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should eventually return: Running + +oc get vmi -n +# Should show: Running +``` + +--- + +### VM Won't Stop + +**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state + +**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. + +**Possible Causes**: +- Guest OS not responding to ACPI shutdown signal +- virt-launcher pod stuck and not terminating +- VirtualMachineInstance (VMI) deletion blocked by finalizers +- Guest shutdown scripts hanging +- Filesystem sync issues in guest OS + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (might show `Stopping`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**2. Check VMI status and deletion timestamp**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check VMI events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vmi -n | grep -A 10 "Events:" +``` + +**5. Check VMI finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.metadata.finalizers}' +``` + +**6. Check if guest is responsive** (if VMI still exists): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Patterns**: +- VMI shows `deletionTimestamp` but never actually deletes +- virt-launcher pod in `Terminating` state +- VM runStrategy is `Halted` but printableStatus shows `Stopping` + +**Solutions** (Use MCP Tools First): + +1. **Wait for graceful shutdown** (default: 30 seconds): + + Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. + +2. **Force stop by deleting VMI**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + This is the standard way to force-stop a VM. + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n + # Should return: Error from server (NotFound) + ``` + +3. **Force delete VMI with grace period** (if VMI won't delete): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +4. **Force delete virt-launcher pod**: + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find the pod using `pods_list_in_namespace` from diagnostic step 3. + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + ⚠️ **Note**: For force deletion with grace period, use CLI fallback. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') + oc delete pod $POD_NAME -n --force --grace-period=0 + ``` + +5. **Remove VMI finalizers** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VMI using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update VMI using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Patch VM runStrategy directly** (ensure consistency): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` (diagnostic step 1) + 2. Set `.spec.runStrategy` to `"Halted"` + 3. Update VM using `resources_create_or_update` with modified JSON + + **CLI Fallback** (merge patch easier via CLI): + Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Stopped` or `Halted`). + +Verify VMI is gone: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Verify virt-launcher pod is gone: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for pods matching `virt-launcher-`. Should return no results. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Stopped or Halted + +oc get vmi -n +# Should return: Error from server (NotFound) + +oc get pods -n | grep virt-launcher- +# Should return: No resources found +``` + +**Prevention**: +- Ensure guest OS has ACPI support enabled +- Use proper shutdown commands in guest OS +- Avoid forceful stops unless necessary (can corrupt guest filesystem) + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/network-errors.md new file mode 100644 index 00000000..82fa313c --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/network-errors.md @@ -0,0 +1,429 @@ +--- +title: VM Network Errors +category: kubevirt +sources: + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] +semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] +use_cases: [vm-creation, network-troubleshooting] +related_docs: [INDEX.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Network Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. + +**When to use this document**: +- VM created successfully but secondary networks not attached +- NetworkAttachmentDefinition not found errors +- Multus CNI failures +- SR-IOV device attachment issues + +**Skills that use this**: vm-create + +--- + +### Network Attachment Failures + +**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working + +**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). + +**Possible Causes**: +- NetworkAttachmentDefinition doesn't exist in the namespace +- Multus CNI not installed or not configured on cluster +- Namespace mismatch (NAD in different namespace than VM) +- Interface name conflicts in VM spec +- Bridge/network configuration errors in NAD +- SR-IOV device not available (if using SR-IOV) + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List NetworkAttachmentDefinitions in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "" +} +``` + +Review `.items[].metadata.name` for available NADs. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -n +``` + +**2. List NetworkAttachmentDefinitions in all namespaces**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition" +} +``` + +Omit `namespace` parameter to list across all namespaces. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -A +``` + +**3. Check specific NetworkAttachmentDefinition**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "", + "name": "" +} +``` + +Review `.spec.config` for CNI configuration. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definition -n -o yaml +``` + +**4. Check VM network configuration**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.networks` to see network references. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq +``` + +**5. Check VM domain interfaces**: + +Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq +``` + +**6. Check virt-launcher pod network annotations** (shows actual attachments): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**7. Check Multus is installed**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "openshift-multus" +} +``` + +Should show Multus CNI pods running. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n openshift-multus +``` + +**8. Check for errors in virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n +``` + +**Common Error Messages**: +- `"network-attachment-definition not found"` - NAD doesn't exist in namespace +- `"multus CNI not configured"` - Multus not installed or misconfigured +- `"interface name conflict"` - Duplicate interface names in VM spec +- `"failed to add network"` - CNI plugin error (check NAD config) +- `"no available devices"` - SR-IOV device not available (if using SR-IOV) + +**Solutions** (Use MCP Tools First): + +1. **Verify NetworkAttachmentDefinition exists in correct namespace**: + + Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. + + If NAD is in different namespace, copy it to VM namespace: + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get NAD from source namespace using `resources_get` + 2. Modify `.metadata.namespace` to target namespace + 3. Create NAD in target namespace using `resources_create_or_update` + + **CLI Fallback** (stream processing easier via CLI): + Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" + ```bash + oc get network-attachment-definition -n -o yaml | \ + sed "s/namespace: /namespace: /" | \ + oc apply -f - + ``` + +2. **Create missing NetworkAttachmentDefinition**: + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + Example: Linux bridge network + + **Parameters**: + ```json + { + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": { + "name": "vlan100", + "namespace": "" + }, + "spec": { + "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" + ```bash + cat < + spec: + config: '{ + "cniVersion": "0.3.1", + "type": "bridge", + "bridge": "br1", + "vlan": 100, + "ipam": { + "type": "host-local", + "subnet": "192.168.100.0/24" + } + }' + EOF + ``` + +3. **Check Multus CNI installation**: + + Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. + + To check cluster network operator: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterOperator" + } + ``` + + Filter for `network` operator. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n openshift-multus + oc get clusteroperators network + ``` + +4. **Fix interface name conflicts** (if VM has duplicate names): + + Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. + + Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. + + **CLI Fallback** (interactive edit easier via CLI): + Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' + oc edit vm -n + ``` + +5. **Validate NAD configuration syntax**: + + Use `resources_get` from diagnostic step 3, extract `.spec.config`. + + Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get network-attachment-definition -n -o jsonpath='{.spec.config}' + ``` + +6. **Check SR-IOV device availability** (if using SR-IOV networks): + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + List SR-IOV network node policies: + + **Parameters**: + ```json + { + "apiVersion": "sriovnetwork.openshift.io/v1", + "kind": "SriovNetworkNodePolicy", + "namespace": "openshift-sriov-network-operator" + } + ``` + + Check SR-IOV device plugin pods: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "openshift-sriov-network-operator" + } + ``` + + Filter for pods with "device-plugin" in name. + + Check available SR-IOV devices on node: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "Node", + "name": "" + } + ``` + + Review `.status.allocatable` for SR-IOV resources. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get sriovnetworknodepolicy -n openshift-sriov-network-operator + oc get pods -n openshift-sriov-network-operator | grep device-plugin + oc describe node | grep -A 10 "Allocatable:" + ``` + +7. **Recreate VM with corrected network configuration** (if needed): + + If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +After remediation, check virt-launcher pod network status: + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +Should show all attached networks with status. Example output: +```json +[ + { + "name": "openshift-sdn", + "interface": "eth0", + "ips": ["10.128.2.10"], + "default": true + }, + { + "name": "vlan100", + "interface": "net1", + "ips": ["192.168.100.5"] + } +] +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**Check from inside VM** (via console): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# In guest OS: +ip addr show +# Should show all network interfaces (eth0, net1, etc.) +``` + +**Common Network Types**: +- **Linux Bridge**: Layer 2 bridge for VLAN networks +- **SR-IOV**: High-performance direct device assignment +- **macvlan**: MAC-based VLAN for container networks +- **OVN-Kubernetes**: OpenShift native overlay network + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/runtime-errors.md new file mode 100644 index 00000000..804d3cbc --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/runtime-errors.md @@ -0,0 +1,616 @@ +--- +title: VM Runtime Errors +category: kubevirt +sources: + - title: Kubernetes Pod Lifecycle + url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ + date_accessed: 2026-02-17 +tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] +semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] +use_cases: [vm-creation, vm-lifecycle, diagnostics] +related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Runtime Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. + +**When to use this document**: +- VM or virt-launcher pod shows `CrashLoopBackOff` status +- virt-launcher pod repeatedly restarting +- Guest OS kernel panics on boot + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### CrashLoopBackOff + +**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting + +**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. + +**Possible Causes**: +- Guest OS kernel panic on boot +- Insufficient resources (memory/CPU) for guest OS +- Corrupted disk image or filesystem +- QEMU/libvirt crashes due to configuration errors +- Missing or incompatible device drivers in guest +- Resource limits too low for virt-launcher pod +- Virtualization features (KVM) not available on node + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check virt-launcher pod restart count**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# Look at RESTARTS column +``` + +**2. View recent crash logs** (previous container instance): + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "previous": true, + "tail": 100 +} +``` + +Look for QEMU errors, kernel panics, or segfaults. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n virt-launcher--xxx --previous +``` + +**3. Check current virt-launcher logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. + +**CLI Fallback** (if MCP unavailable or all containers needed): +```bash +oc logs -n virt-launcher--xxx --all-containers +``` + +**4. Check VMI conditions for crash details**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Extract `.status.conditions` for crash details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**5. Check pod events for crash reasons**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" +``` + +**6. Check pod resource limits**: + +**MCP Tool**: `pods_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +Extract `.spec.containers[0].resources` for resource limits. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq +``` + +**7. Check node kubelet logs for OOM kills**: + +⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +oc adm node-logs -u kubelet | grep -i oom +``` + +**8. Access guest console** (if VM briefly starts): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Crash Patterns**: + +1. **Guest Kernel Panic**: + - Console logs show kernel panic messages + - Guest crashes immediately after boot + - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" + +2. **OOM (Out of Memory)**: + - Pod killed with reason: `OOMKilled` + - Guest runs out of memory during boot or operation + - virt-launcher logs show memory allocation failures + +3. **QEMU Crash**: + - virt-launcher logs show QEMU segmentation fault + - Symptoms: "qemu-system-x86_64: terminated by signal" + - Configuration incompatibility or QEMU bug + +4. **Disk Image Corruption**: + - Guest cannot boot from disk + - Filesystem errors in guest console + - DataVolume import failed + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for kernel panic or boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for: + - Kernel panic messages + - Initramfs errors + - Filesystem mounting failures + - Missing device errors + +2. **Review virt-launcher crash logs**: + + Use `pods_log` with `previous: true` from diagnostic step 2. + + Look for: + - QEMU command line errors + - Device initialization failures + - Memory allocation errors + - Signal termination (SIGSEGV, SIGABRT) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx --previous + ``` + +3. **Check for OOM (Out of Memory) kills**: + + **MCP Tool**: `pods_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + Extract `.status.containerStatuses[0].lastState.terminated.reason`. + + If returns `"OOMKilled"`: + - Option 1: Increase virt-launcher memory limits + - Option 2: Decrease guest memory allocation + - Option 3: Use smaller instance type + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' + ``` + +4. **Increase resources if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check current memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +5. **Verify disk image integrity**: + + **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters for list**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" + } + ``` + + **Parameters for specific DV**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check `.status.phase` (should be `Succeeded`). + + If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get datavolume -n -o jsonpath='{.status.phase}' + ``` + +6. **Check virtualization (KVM) availability**: + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +7. **Simplify VM configuration** (eliminate variables): + + Try creating minimal VM using vm-create skill with: + - Small instance type + - No secondary networks + - Simple container disk (e.g., Fedora) + - No cloud-init + + If minimal VM works, add features back one by one. + +8. **Recreate VM with different workload** (test disk image): + + If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) + +After remediation, check pod restart count stops increasing: + +**Parameters for pods**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). + +Check VM reaches Running state: + +**Parameters for VM**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# RESTARTS should stabilize + +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Running +``` + +Verify guest is responsive: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# Should show login prompt or OS console +``` + +**Advanced Debugging**: + +**MCP Tool**: `pods_exec` (from openshift-virtualization) + +Check libvirt domain XML: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["virsh", "dumpxml", "1"] +} +``` + +Check QEMU process: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["ps", "aux"] +} +``` + +Filter output for "qemu" process. + +**CLI Fallback** (if MCP unavailable): +```bash +oc exec -n virt-launcher--xxx -- virsh dumpxml 1 +oc exec -n virt-launcher--xxx -- ps aux | grep qemu +``` + +**Prevention**: +- Start with minimal VM configuration and add complexity gradually +- Use recommended instance types for your workload +- Test disk images before deploying to production +- Ensure nodes have adequate resources and KVM support +- Monitor resource usage and set appropriate limits + +--- + +## Workaround Patterns for MCP Tool Limitations + +### General Pattern: Diagnose → Propose → Confirm → Execute + +When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): + +**1. Diagnose** the root cause using Kubernetes commands +```bash +oc describe vm -n +oc get events -n --field-selector involvedObject.name= +``` + +**2. Propose** a manual workaround to the user with clear explanation +- Explain why the issue occurred +- Show what will be changed +- List alternative options + +**3. Confirm** with user before executing (Human-in-the-Loop) +- Wait for explicit "yes" or "apply workaround" +- Never auto-execute modifications + +**4. Execute** the workaround using oc/kubectl +```bash +oc patch vm -n --type=merge -p '...' +``` + +**5. Verify** the fix was successful +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**6. Document** the limitation and suggest filing enhancement request +- Note this is temporary until MCP tool is enhanced +- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues + +### Example: Adding Tolerations Workaround + +**Diagnostic Output**: +```markdown +## ⚠️ VM Scheduling Issue Detected + +**Root Cause**: Node taints prevent VM scheduling + +**Details**: +- Found 3 nodes with taint: `virtualization=true:NoSchedule` +- VM spec does not include matching tolerations +- This prevents VM scheduling on virtualization-dedicated nodes +``` + +**Proposed Workaround**: +```bash +oc patch vm web-server -n vms --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' +``` + +**User Confirmation Required**: +``` +How would you like to proceed? +- "apply workaround" - I'll patch the VM with tolerations +- "manual" - I'll provide instructions for you to apply manually +- "cancel" - Delete the VM and abort creation +``` + +**After Execution**: +```markdown +## ✓ Workaround Applied Successfully + +**Action**: Added tolerations for taint `virtualization=true:NoSchedule` +**New Status**: Stopped (VM can now be scheduled) + +**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. +Future VMs in this cluster will need the same fix until the tool is enhanced. +``` + +--- + +## VM Status Reference + +### Status Values + +| Status | Meaning | Action Required | +|--------|---------|-----------------| +| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | +| `Running` | VM is running | Normal | +| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | +| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | +| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | +| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | +| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | +| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | +| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | +| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | + +### Checking VM Status + +```bash +# Get printable status +oc get vm -n -o jsonpath='{.status.printableStatus}' + +# Get detailed status and conditions +oc get vm -n -o jsonpath='{.status}' | jq + +# Watch status changes in real-time +oc get vm -n -w +``` + +--- + +## Best Practices for Agents + +When implementing diagnostic workflows: + +1. **Always verify VM status** after creation (wait 5-10 seconds first) +2. **Consult this document** when encountering error status values +3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) +4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) +5. **Respect human-in-the-loop** for all VM modifications +6. **Document temporary workarounds** and their limitations clearly +7. **Suggest filing issues** for missing MCP tool features + +### Document Consultation Pattern + +```markdown +**Document Consultation** (REQUIRED): +1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes +2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." +``` + +--- + +## Known MCP Tool Limitations + +### vm_create tool + +**Currently Supported**: +- ✓ Namespace, name (required) +- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) +- ✓ Size hints (small, medium, large) +- ✓ Storage size +- ✓ Autostart flag +- ✓ Networks (Multus NetworkAttachmentDefinitions) +- ✓ Performance family (u1, o1, c1, m1) +- ✓ Instance type, preference + +**Not Currently Supported** (requires workarounds): +- ✗ Tolerations (for node taints) +- ✗ Node selectors +- ✗ Affinity/anti-affinity rules +- ✗ Resource requests/limits (beyond instance type) +- ✗ Custom labels/annotations +- ✗ SSH keys injection +- ✗ Cloud-init user data + +**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. + +**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues + +--- + +## Additional Resources + +- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) +- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) +- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) +- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) + +--- + +[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/scheduling-errors.md new file mode 100644 index 00000000..e47c8c4f --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/scheduling-errors.md @@ -0,0 +1,417 @@ +--- +title: VM Scheduling Errors +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 +tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] +semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] +use_cases: [vm-creation, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Scheduling Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**When to use this document**: +- VM shows status `ErrorUnschedulable` after creation or start attempt +- VM events mention scheduling failures, taints, resources, or node selectors + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +## ErrorUnschedulable + +**Symptom**: VM shows status `ErrorUnschedulable` after creation + +**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**Possible Causes**: + +### 1. Node Taints (Most Common) + +Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM events for scheduling failures**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `involvedObject.name` == "" and look for messages like: +- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= +``` + +**2. Check node taints in the cluster**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' +``` + +**Common Taint Patterns**: +- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule +- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes +- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads + +**Solution - Add Tolerations to VM**: + +The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. + +**Workaround (post-creation using MCP Tools)**: + +**Step 1**: Get current VM spec + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Step 2**: Modify the returned JSON to add tolerations + +Add to `.spec.template.spec.tolerations`: +```json +{ + "tolerations": [ + { + "key": "virtualization", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule" + } + ] +} +``` + +**Step 3**: Update VM with modified spec + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +```json +{ + "resource": "" +} +``` + +Pass the complete modified VM resource as YAML or JSON string. + +**Step 4**: Verify tolerations were added + +Use `resources_get` again and check `.spec.template.spec.tolerations` in response. + +**Step 5**: Check if VM status improved + +Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. + +**CLI Fallback** (if MCP patch is too complex): +```bash +# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' + +# Verify tolerations +oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq + +# Check status +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**Example - Multiple Tolerations**: +```bash +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: "dedicated" + operator: "Equal" + value: "virt-workloads" + effect: "NoSchedule" +' +``` + +**Toleration Operators**: +- `Equal` - Key and value must match exactly +- `Exists` - Only key must exist (ignores value) + +**Toleration Effects**: +- `NoSchedule` - Don't schedule new pods (existing pods continue) +- `PreferNoSchedule` - Avoid scheduling if possible +- `NoExecute` - Don't schedule AND evict existing pods + +**Alternative Solutions**: +1. **Remove node taints** (if you have cluster-admin access): + ```bash + oc adm taint nodes virtualization=true:NoSchedule- + ``` + +2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits + +3. **File enhancement request** - Request tolerations support in openshift-mcp-server: + https://github.com/openshift/openshift-mcp-server/issues + +--- + +### 2. Insufficient Resources + +Not enough CPU, memory, or storage available on any node. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM resource requests**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' +``` + +**2. Check node resource availability**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. + +Alternatively, use `nodes_top` MCP tool for current resource usage. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe nodes | grep -A 5 "Allocated resources" +``` + +**3. Look for VM events mentioning "Insufficient"**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep "Insufficient" +``` + +**Example Event**: +``` +0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. +``` + +**Solutions** (Use MCP Tools First): + +1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) +2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill +3. **Delete unused VMs** - Use vm-delete skill to free up resources +4. **Check resource quotas**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters for quota**: + ```json + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" + } + ``` + + **Parameters for limit range**: + ```json + { + "apiVersion": "v1", + "kind": "LimitRange", + "namespace": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + oc describe limitrange -n + ``` + +--- + +### 3. Node Selector Mismatch + +VM requires specific node labels that don't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM node selector requirements**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.nodeSelector` to see required node labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' +``` + +**2. List available node labels**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.metadata.labels` for available labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes --show-labels +``` + +**3. Check if any nodes match the selector**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node", + "labelSelector": "=" +} +``` + +Should return at least one node with matching labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -l = +``` + +**Solutions** (Use MCP Tools First): + +**Option 1: Remove node selector from VM** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get current VM using `resources_get` (diagnostic step 1) +2. Remove `.spec.template.spec.nodeSelector` field +3. Update VM using `resources_create_or_update` with modified JSON + +**CLI Fallback** (JSON patch easier via CLI): +Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" +```bash +oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' +``` + +**Option 2: Add label to nodes** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get node using `resources_get` +2. Add label to `.metadata.labels` +3. Update node using `resources_create_or_update` + +⚠️ **Note**: Node labeling typically requires cluster admin privileges. + +**CLI Fallback** (simpler via CLI): +Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" +```bash +oc label node = +``` + +--- + +[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/storage-errors.md new file mode 100644 index 00000000..16ba584d --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/storage-errors.md @@ -0,0 +1,1011 @@ +--- +title: VM Storage Errors +category: kubevirt +sources: + - title: KubeVirt CDI - DataVolumes + url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ + date_accessed: 2026-02-17 + - title: Kubernetes Persistent Volumes + url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ + date_accessed: 2026-02-17 +tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] +semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] +use_cases: [vm-creation, vm-deletion, vm-cloning] +related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] +last_updated: 2026-02-17 +--- + +# VM Storage Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. + +**When to use this document**: +- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` +- Storage deletion fails after VM deletion +- DataVolume cloning operations fail +- PVC provisioning issues + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### ErrorDataVolumeNotReady + +**Symptom**: VM shows status `ErrorDataVolumeNotReady` + +**Description**: The DataVolume (persistent storage) backing the VM is not ready. + +**Possible Causes**: + +#### 1. DataVolume Still Provisioning + +Storage provisioning takes time, especially for large disks or when importing images. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Get detailed DataVolume information**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` and `.status.conditions` for provisioning details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o json +``` + +**3. Check PVC (PersistentVolumeClaim) bound status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Check `.status.phase` for each PVC (should be `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. + +#### 2. Storage Class Not Found + +The requested storage class doesn't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List available storage classes**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" +} +``` + +Review the list of available storage classes (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass +``` + +**2. Check DataVolume's requested storage class**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.spec.pvc.storageClassName` in the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' +``` + +**Solution**: +1. Use a valid storage class from the cluster +2. Recreate VM with correct storage class parameter + +#### 3. Insufficient Storage Quota + +Namespace has insufficient storage quota to provision the PVC. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check resource quotas**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**2. Check storage usage**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase +``` + +**Solution**: +1. Request quota increase from cluster admin +2. Delete unused PVCs to free quota +3. Reduce VM storage size + +--- + +### ErrorPvcNotFound + +**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List PVCs in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review the list of available PVCs (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check VM's PVC references**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' +``` + +**Solution**: +- Wait for DataVolume to create the PVC +- Manually create missing PVC +- Fix VM spec to reference correct PVC name + +--- + + +--- + +### Storage Deletion Failures + +**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace + +**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. + +**Possible Causes**: +- PVC still bound to active PersistentVolume with `Retain` policy +- DataVolume still being referenced by another resource +- CDI (Containerized Data Importer) controller issues +- Storage class retention policy preventing deletion +- Finalizers on PVC/DataVolume blocking cleanup +- PVC still mounted by a pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check PVC status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check specific PVC phase**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Released` or `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o jsonpath='{.status.phase}' +``` + +**3. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**4. Check what's using the PVC**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' +``` + +**5. Check PVC finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. + +**6. Check DataVolume finalizers**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.metadata.finalizers}' +``` + +**7. Check PV reclaim policy**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume" +} +``` + +Filter results for PV where `.spec.claimRef.name` matches ``. + +To get specific PV policy: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume", + "name": "" +} +``` + +Check `.spec.persistentVolumeReclaimPolicy`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pv | grep +oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' +``` + +**Common Finalizer Patterns**: +- `kubernetes.io/pvc-protection` - Protects PVC while in use +- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer + +**Solutions** (Use MCP Tools First): + +1. **Delete DataVolume first, then PVC**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete DataVolume first (often blocks PVC deletion): + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Wait a few seconds, then delete PVC: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete datavolume -n + oc delete pvc -n + ``` + +2. **Check for pods still using PVC**: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "" + } + ``` + + Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. + + Then delete the pods using `pods_delete`: + + **Parameters**: + ```json + { + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' + oc delete pod -n + ``` + +3. **Force delete PVC** (if safe to do so): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete pvc -n --grace-period=0 --force + ``` + +4. **Remove finalizers from PVC** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PVC using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update PVC using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Remove finalizers from DataVolume** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current DataVolume using `resources_get` (diagnostic step 6) + 2. Remove items from `.metadata.finalizers` array + 3. Update DataVolume using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Change PV reclaim policy** (if PV has Retain policy): + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PV policy using `resources_get` (diagnostic step 7) + 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` + 3. Update PV using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" + ```bash + oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' + oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' + ``` + +**Storage Quota Check** (Use MCP Tools First): + +After deletion, verify storage quota is freed: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters for quota check**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.used` to verify storage quota is freed. + +**Parameters for PVC verification**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +oc get pvc -n +``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Confirm PVC is deleted: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Confirm DataVolume is deleted: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +# Should return: Error from server (NotFound) + +oc get datavolume -n +# Should return: Error from server (NotFound) +``` + +--- + +### DataVolume Cloning Failures + +**Symptom**: VM clone created successfully but DataVolume clone operation fails + +**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. + +**Possible Causes**: +- CSI driver doesn't support volume cloning +- Source PVC storage class incompatible with cloning +- Cross-namespace cloning not permitted by storage backend +- Insufficient storage quota in target namespace +- Source PVC not in `Bound` state +- Storage class doesn't have volume cloning enabled +- CDI (Containerized Data Importer) controller issues + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume clone status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and `.metadata.name`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Check DataVolume events for errors**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe datavolume -n +``` + +**3. Check DataVolume phase**: + +Use `resources_get` from step 1, extract `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +``` + +**4. Check if storage class supports cloning**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "name": "" +} +``` + +Review the full YAML output for cloning-related configurations. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass -o yaml | grep -A 5 -i clone +``` + +**5. Check CSI driver capabilities**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "CSIDriver" +} +``` + +Review `.items[].metadata.name` for available CSI drivers. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get csidriver +``` + +**6. Check source PVC status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Bound` for cloning to work). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**7. Check target namespace storage quota**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**8. Check CDI controller logs**: + +**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) + +First, list pods in openshift-cnv namespace: + +**Parameters for pods_list_in_namespace**: +```json +{ + "namespace": "openshift-cnv", + "labelSelector": "app.kubernetes.io/component=cdi-deployment" +} +``` + +Then get logs using `pods_log`: + +**Parameters**: +```json +{ + "namespace": "openshift-cnv", + "name": "", + "tail": 100 +} +``` + +**CLI Fallback** (if MCP unavailable or easier via CLI): +```bash +oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') +``` + +**Common Error Messages**: +- `"volume cloning is not supported"` - CSI driver lacks clone capability +- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage +- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace +- `"insufficient quota"` - Target namespace lacks storage quota +- `"source PVC not bound"` - Source PVC must be in Bound state for cloning +- `"StorageClass does not support cloning"` - Storage class configuration issue + +**Solutions** (Use MCP Tools First): + +1. **Check storage class clone support**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" + } + ``` + + For each storage class in `.items[]`, check: + - `.metadata.name` (storage class name) + - `.provisioner` (CSI driver) + + Storage classes using CSI drivers typically support cloning. Look for provisioners like: + - `csi.ovirt.org` (oVirt CSI) + - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) + - `ebs.csi.aws.com` (AWS EBS CSI) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner + ``` + +2. **Verify source PVC is bound**: + + Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pvc -n -o jsonpath='{.status.phase}' + ``` + +3. **Check target namespace quota**: + + Use `resources_list` from diagnostic step 7 to check quota. + + If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + ``` + +4. **Use snapshot-based cloning** (alternative method): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Step 1**: Create VolumeSnapshot of source PVC + + **Parameters**: + ```json + { + "apiVersion": "snapshot.storage.k8s.io/v1", + "kind": "VolumeSnapshot", + "metadata": { + "name": "-snapshot", + "namespace": "" + }, + "spec": { + "source": { + "persistentVolumeClaimName": "" + } + } + } + ``` + + **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) + + **Step 3**: Create new DataVolume from snapshot + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": "-rootdisk", + "namespace": "" + }, + "spec": { + "source": { + "snapshot": { + "name": "-snapshot", + "namespace": "" + } + }, + "storage": { + "resources": { + "requests": { + "storage": "50Gi" + } + }, + "storageClassName": "" + } + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" + ```bash + cat <-snapshot + namespace: + spec: + source: + persistentVolumeClaimName: + EOF + + oc get volumesnapshot -snapshot -n + + cat <-rootdisk + namespace: + spec: + source: + snapshot: + name: -snapshot + namespace: + storage: + resources: + requests: + storage: 50Gi + storageClassName: + EOF + ``` + +5. **Use "new empty storage" option** (vm-clone skill): + - If cloning isn't supported, create VM with empty storage + - Manually copy data if needed + +6. **Cross-namespace cloning workaround**: + - Some storage backends require snapshot for cross-namespace cloning + - Create snapshot in source namespace, restore in target namespace (see solution 4 above) + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Check DataVolume reached Succeeded phase: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should return `Succeeded`). + +Check PVC was created and bound: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "-rootdisk" +} +``` + +Check `.status.phase` (should return `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +# Should return: Succeeded + +oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' +# Should return: Bound +``` + +**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/mcp-servers/mock-virt-mcp.py b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..2e083d72 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1458 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/skills/vm-snapshot-restore/SKILL.md b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/skills/vm-snapshot-restore/SKILL.md new file mode 100644 index 00000000..f4e2fb19 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/skills/vm-snapshot-restore/SKILL.md @@ -0,0 +1,495 @@ +--- +name: vm-snapshot-restore +description: | + Restore virtual machines from snapshots with strict safety confirmations to prevent data loss. + + Use when: + - "Restore VM [name] from snapshot [snapshot-name]" + - "Roll back VM [name] to snapshot" + - "Recover VM [name] from backup" + + CRITICAL: Requires VM to be stopped and typed snapshot name confirmation before restore. + + NOT for creating snapshots (use vm-snapshot-create instead). + +model: inherit +color: red +--- + +# /vm-snapshot-restore Skill + +Restore virtual machines from snapshots in OpenShift Virtualization. **CRITICAL**: This operation replaces current VM state with snapshot data. ALL changes since the snapshot will be LOST. + +**Implementation Note**: This skill uses generic Kubernetes resource tools (`resources_create_or_update`) to create VirtualMachineRestore resources. Dedicated restore tools do not currently exist in the openshift-virtualization MCP server. + +## Prerequisites + +**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools**: +- `resources_create_or_update` (from openshift-virtualization) - Create VirtualMachineRestore +- `resources_get` (from openshift-virtualization) - Verify VM/snapshot exists, monitor restore +- `vm_lifecycle` (from openshift-virtualization) - Stop VM if running + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster (>= 4.19) +- OpenShift Virtualization operator installed +- ServiceAccount with RBAC permissions to create VirtualMachineRestore resources + +## When to Use This Skill + +**Trigger this skill when:** +- User wants to restore a VM to a previous state +- User wants to recover from failed changes/upgrades +- User explicitly requests snapshot restore + +**User phrases that trigger this skill:** +- "Restore VM api-server from snapshot snapshot-20240115" +- "Roll back database-01 to pre-upgrade snapshot" +- "Recover VM web-server from backup" + +**Do NOT use this skill when:** +- User wants to create snapshots → Use `vm-snapshot-create` skill +- User wants to list snapshots → Use `vm-snapshot-list` skill +- User wants to clone a VM → Use `vm-clone` skill + +## Workflow + +### Step 1: Gather Restore Information + +**Required Information from User:** +1. **VM Name** - VM to restore +2. **Namespace** - Namespace where VM exists +3. **Snapshot Name** - Snapshot to restore from + +If any information missing, ask for it. + +### Step 2: Verify VM Exists + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Error Handling**: +- If VM not found → Report error +- If permission denied → Report RBAC error + +### Step 3: Check VM Running State + +**From the VM resource in Step 2**, check `status.printableStatus`. + +**If VM is Running:** +```markdown +⚠️ VM Must Be Stopped Before Restore + +**VM**: `` (namespace: ``) +**Status**: Running + +**Safety Requirement**: VMs must be stopped before restore to prevent data corruption. + +**Options:** +1. "stop-and-restore" - Stop the VM first, then restore from snapshot +2. "cancel" - Cancel restore operation + +How would you like to proceed? +``` + +**Wait for user response.** + +- If "stop-and-restore" → Stop VM using vm_lifecycle, then continue +- If "cancel" → Stop workflow + +### Step 4: Verify Snapshot Exists + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "namespace": "", + "name": "" +} +``` + +**If snapshot not found:** +```markdown +❌ Snapshot Not Found + +**Snapshot**: `` does not exist in namespace ``. + +**To list available snapshots:** +"List snapshots for VM " + +Restore operation cancelled. +``` + +**STOP workflow**. + +**Extract snapshot details:** +- `metadata.creationTimestamp` - Creation time +- `status.phase` - Must be "Succeeded" +- `status.readyToUse` - Must be `true` +- `spec.source.name` - Verify it matches the VM name + +**If snapshot status is not Ready:** +```markdown +❌ Snapshot Not Ready + +**Snapshot**: `` +**Status**: +**Ready to Use**: + +Snapshot is not ready for restore. Only snapshots with "Succeeded" phase and readyToUse=true can be used. + +Restore operation cancelled. +``` + +**STOP workflow**. + +### Step 5: Present Restore Preview and Get Typed Confirmation + +**CRITICAL: User must type the snapshot name to confirm.** + +```markdown +## 🔴 VM RESTORE - Data Loss Warning + +**⚠️ THIS WILL REPLACE CURRENT VM STATE WITH SNAPSHOT DATA ⚠️** + +### What Will Happen + +**VM to Restore**: `` (namespace: ``) +**Snapshot to Restore From**: `` + +**Current VM State** (WILL BE LOST): +- **Last Modified**: +- **Changes Since Snapshot**: ALL changes made after WILL BE PERMANENTLY LOST + +**Snapshot State** (WILL BE RESTORED): +- **Created**: +- **Age**: + +**Time Range of Data Loss**: +- **⚠️ ALL CHANGES in the last WILL BE LOST ⚠️** + +### What Will Be Restored +- ✓ VM configuration (from snapshot time) +- ✓ Disk data (from snapshot time) + +### What Will Be Lost +- ✗ **ALL disk changes** made after +- ✗ **ALL configuration changes** made after + +--- + +**⚠️ CRITICAL: This restore is permanent. Current VM state cannot be recovered unless you create a snapshot now.** + +**To proceed with restore, type the snapshot name exactly as shown:** + +Type `` to confirm: _____ +``` + +**Wait for user to type the snapshot name.** + +**Validation:** +- Compare user input with snapshot name (case-sensitive, exact match) +- **If match**: Proceed to Step 6 +- **If mismatch**: Cancel operation + +**On mismatch:** +```markdown +❌ Confirmation Failed + +**You typed**: `` +**Expected**: `` + +Names do not match. Restore cancelled for safety. + +Operation cancelled. Current VM state preserved. +``` + +**STOP workflow**. + +### Step 6: Final Confirmation Before Restore + +**After typed verification succeeds**, ask for final explicit confirmation. + +```markdown +## ✓ Typed Verification Passed + +**Confirmation received for snapshot**: `` + +### Ready to Restore + +**VM**: `` (namespace: ``) +**From Snapshot**: `` + +**Impact**: +- Current VM state will be replaced with snapshot state +- All changes in the last will be permanently lost + +--- + +**Proceed with VM restore? This action cannot be undone.** +- Type "yes" to execute restore +- Type "cancel" to abort + +Your choice: _____ +``` + +**Wait for user response.** + +**Handle response:** +- If "yes" → Proceed to Step 7 (execute restore) +- If "cancel", "no", "wait", or anything else → Cancel operation + +**On cancellation:** +```markdown +Restore operation cancelled by user. Current VM state preserved. +``` + +**STOP workflow**. + +### Step 7: Execute Restore + +**ONLY PROCEED AFTER**: +- ✓ VM verified (exists, stopped) +- ✓ Snapshot verified (exists, ready) +- ✓ User typed snapshot name correctly +- ✓ User confirmed "yes" + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Construct VirtualMachineRestore YAML:** + +```yaml +apiVersion: snapshot.kubevirt.io/v1beta1 +kind: VirtualMachineRestore +metadata: + name: + namespace: +spec: + target: + apiGroup: kubevirt.io + kind: VirtualMachine + name: + virtualMachineSnapshotName: +``` + +**Generate restore name**: +- Format: `restore--` +- Example: `restore-database-01-20260218-143500` + +**Parameters**: +```json +{ + "resource": "apiVersion: snapshot.kubevirt.io/v1beta1\nkind: VirtualMachineRestore\nmetadata:\n name: \n namespace: \nspec:\n target:\n apiGroup: kubevirt.io\n kind: VirtualMachine\n name: \n virtualMachineSnapshotName: " +} +``` + +**Report progress:** +```markdown +🔄 Restoring VM from snapshot... +⏳ This may take several minutes... +``` + +### Step 8: Monitor Restore Progress + +**Use `resources_get` to monitor VirtualMachineRestore status.** + +Check `status.complete`: +- `true` → Restore completed +- `false` → Restore in progress + +**Wait up to 10 minutes for restore to complete.** + +### Step 9: Report Restore Results + +**On success:** + +```markdown +## ✓ VM Restored Successfully + +**VM**: `` (namespace: ``) +**Restored From**: Snapshot `` + +### Restore Details +- **Snapshot Created**: +- **Restore Completed**: +- **VM Status**: Stopped (ready to start) + +### Data Loss Confirmation +- ⚠️ All changes made after have been lost + +### Next Steps + +**To start the restored VM:** +"Start VM in namespace " +``` + +**On failure:** + +```markdown +## ❌ VM Restore Failed + +**Error**: + +**VM**: `` +**Snapshot**: `` + +**Current VM State**: UNKNOWN - may be partially restored or unchanged + +**CRITICAL**: Do not start VM until restore issue is resolved + +**Recovery Options:** +1. Try restore again after resolving the error +2. Restore from a different snapshot +3. Contact cluster admin for investigation +``` + +## Dependencies + +### Required MCP Servers +- `openshift-virtualization` - OpenShift MCP server with kubevirt toolset + +### Required MCP Tools +- `resources_create_or_update` (from openshift-virtualization) - Create VirtualMachineRestore +- `resources_get` (from openshift-virtualization) - Verify and monitor +- `vm_lifecycle` (from openshift-virtualization) - Stop VM if running + +### Related Skills +- `vm-snapshot-list` - List snapshots before restore +- `vm-snapshot-create` - Create snapshots before risky operations +- `vm-snapshot-delete` - Delete old snapshots +- `vm-lifecycle-manager` - Start VM after restore + +### Reference Documentation + +**Official Red Hat Documentation:** +- [OpenShift Virtualization Snapshots - OpenShift 4.20](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt-managing-vm-snapshots) + +**Upstream Documentation:** +- [KubeVirt VM Snapshots](https://kubevirt.io/user-guide/operations/snapshot_restore_api/) + +## Critical: Human-in-the-Loop Requirements + +**IMPORTANT:** This skill performs DESTRUCTIVE operations. You MUST: + +1. **Before Restoring Snapshots** (CRITICAL - Data Loss Risk) + - **REQUIRE VM to be stopped first** if currently running + - Display what will be lost (current VM state since snapshot) + - Show snapshot details (creation time, age) + - **Require typed confirmation** - user must type snapshot name exactly + - Ask: "Proceed with restore? This will replace current VM state. (yes/cancel)" + - Wait for explicit "yes" + +2. **Never Auto-Execute** + - **NEVER restore without user confirmation** + - **NEVER restore to running VMs** without stopping first + - **NEVER skip typed verification for restore operations** + +**Why This Matters:** +- **Data Loss on Restore**: Restoring replaces current VM state - all changes since snapshot are PERMANENTLY LOST +- **No Undo**: Restore cannot be reversed - current data cannot be recovered +- **Typed Confirmation**: Prevents accidental restores to wrong snapshots + +## Common Issues + +### Issue 1: Restore Fails - Insufficient Storage Capacity + +**Error**: "Failed to restore: insufficient storage capacity" or "PVC provisioning failed" + +**Cause**: The namespace doesn't have enough storage quota or the storage backend is full. + +**Solution:** +1. Check namespace storage quota: `resources_list` with kind="ResourceQuota" +2. Check PVC status: `resources_list` for PersistentVolumeClaims +3. Delete unnecessary snapshots: Use vm-snapshot-delete skill +4. Request quota increase: Contact cluster admin +5. Retry restore once storage is available + +### Issue 2: Restore Stuck in Progress + +**Error**: VirtualMachineRestore status shows `complete: false` for extended period + +**Cause**: The storage backend is slow, the snapshot is corrupted, or there's a CSI driver issue. + +**Solution:** +1. Check VirtualMachineRestore `status.conditions` for detailed error messages +2. Verify snapshot is "Succeeded": Use vm-snapshot-list skill +3. Wait longer: Large VMs may take 10+ minutes to restore +4. Cancel and retry: Delete VirtualMachineRestore resource and try again +5. Try alternative snapshot if restore continues to fail +6. Check CSI driver logs (requires cluster admin access) + +### Issue 3: User Typed Wrong Snapshot Name + +**Error**: "Confirmation Failed - Names do not match" + +**Cause**: User made a typo when typing the snapshot name for confirmation. + +**Solution:** +1. Retry with exact name: Copy-paste the snapshot name from the previous message +2. Check capitalization: Snapshot names are case-sensitive +3. Check for spaces: Ensure no leading/trailing spaces in the typed name +4. Use vm-snapshot-list skill to see all available snapshots and verify the correct name + +**This safety mechanism is intentional** - it prevents accidental data loss from restoring to the wrong snapshot. + +## Security Considerations + +- **Data Protection**: Restore requires TWO confirmations (typed + yes) to prevent accidental data loss +- **VM Stop Requirement**: VMs must be stopped before restore +- **Typed Verification**: User must type exact snapshot name +- **RBAC Enforcement**: Requires permissions for VirtualMachineRestore resources +- **Namespace Isolation**: Restore scoped to namespace boundaries +- **Audit Trail**: All restore operations logged in Kubernetes API audit logs +- **No Rollback**: Restore is permanent - current state cannot be recovered after restore + +## Example Usage + +**User**: "The database upgrade failed. Restore database-01 from snapshot pre-upgrade-snapshot" + +**Agent**: "Which namespace is database-01 in?" + +**User**: "production" + +**Agent**: [Verifies VM - Running status detected] +"⚠️ VM Must Be Stopped Before Restore. Options: 1) stop-and-restore 2) cancel" + +**User**: "stop-and-restore" + +**Agent**: [Stops VM, verifies snapshot exists] +"🔴 VM RESTORE - Data Loss Warning +- VM: database-01 (production) +- Snapshot: pre-upgrade-snapshot (created 2026-02-18 10:00, age 6h 30m) +- ⚠️ ALL CHANGES in last 6h 30m WILL BE LOST +Type `pre-upgrade-snapshot` to confirm: _____" + +**User**: "pre-upgrade-snapshot" + +**Agent**: "✓ Typed Verification Passed. Proceed with restore? (yes/cancel)" + +**User**: "yes" + +**Agent**: "🔄 Restoring VM from snapshot... ⏳ This may take several minutes..." +[Monitors progress] +"✓ VM Restored Successfully +- VM: database-01 (production) +- Restored From: pre-upgrade-snapshot (created 2026-02-18 10:00) +- Status: Stopped (ready to start) +- ⚠️ All changes after 2026-02-18 10:00 have been lost + +To start: 'Start VM database-01 in namespace production'" diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/instruction.md b/evaluation/with_skills/rh-virt__vm-snapshot-restore/instruction.md new file mode 100644 index 00000000..d28e79fd --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-restore/instruction.md @@ -0,0 +1,12 @@ +# VM Snapshot Restore Task + +You are an OpenShift Virtualization administrator. Restore VM `production-db` from snapshot `production-db-backup-20240301` in namespace `prod-vms`. + +## Requirements +- Verify snapshot is ready and valid +- Address VM state requirements for restore +- Include safeguards (this is a destructive operation) + +Use MCP tools to examine the cluster. Document your methodology, findings, and restore plan in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/solution/solve.sh b/evaluation/with_skills/rh-virt__vm-snapshot-restore/solution/solve.sh new file mode 100644 index 00000000..d4698552 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-restore/solution/solve.sh @@ -0,0 +1,42 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Snapshot Restore Plan + +## Restore production-db from production-db-backup-20240301 + +### Prerequisites +1. Verify snapshot exists and `status.phase == "Succeeded"` and `status.readyToUse == true` +2. **VM must be stopped** before restore — use `vm_lifecycle` action=stop +3. Verify no active VirtualMachineRestore in progress + +### VirtualMachineRestore YAML +```yaml +apiVersion: snapshot.kubevirt.io/v1beta1 +kind: VirtualMachineRestore +metadata: + name: restore-production-db-20240301 + namespace: prod-vms +spec: + target: + apiGroup: kubevirt.io + kind: VirtualMachine + name: production-db + virtualMachineSnapshotName: production-db-backup-20240301 +``` + +### Procedure +1. Stop VM production-db +2. Verify snapshot is ready (readyToUse: true) +3. **Typed confirmation**: Type snapshot name for safety +4. Create VirtualMachineRestore resource +5. Monitor restore progress (poll status.phase) +6. Start VM after restore completes + +### Warning +- Restore **overwrites** current VM state with snapshot state +- All changes since snapshot will be lost +- **Typed confirmation**: User must type exact snapshot name +- Monitor VirtualMachineRestore status.complete +- Create via resources_create_or_update + +REPORT_EOF diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/task.toml b/evaluation/with_skills/rh-virt__vm-snapshot-restore/task.toml new file mode 100644 index 00000000..bf15ebed --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-restore/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-snapshot-restore" +name = "rh-virt VM Snapshot Restore Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-snapshot-restore", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/tests/llm_judge.py b/evaluation/with_skills/rh-virt__vm-snapshot-restore/tests/llm_judge.py new file mode 100644 index 00000000..0a348593 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-restore/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "vm_stopped_prerequisite", "file": "/root/report.md", "question": "Does the report require the VM to be stopped before restore and explain this is enforced by the platform?", "reference": "A skilled report enforces stop-before-restore. An unskilled report doesn't mention this prerequisite."}, + {"id": "restore_cr_definition", "file": "/root/report.md", "question": "Does the report define a VirtualMachineRestore CR with virtualMachineSnapshotName reference?", "reference": "A skilled report creates proper VirtualMachineRestore resource. An unskilled report doesn't know the restore API."}, + {"id": "destructive_warning_and_verification", "file": "/root/report.md", "question": "Does the report warn about data loss (changes since snapshot) and verify restore completion via status.complete?", "reference": "A skilled report warns about destructive nature and verifies completion. An unskilled report restores without warnings."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/tests/test.sh b/evaluation/with_skills/rh-virt__vm-snapshot-restore/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-restore/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/tests/test_outputs.py b/evaluation/with_skills/rh-virt__vm-snapshot-restore/tests/test_outputs.py new file mode 100644 index 00000000..e02b5cf9 --- /dev/null +++ b/evaluation/with_skills/rh-virt__vm-snapshot-restore/tests/test_outputs.py @@ -0,0 +1,71 @@ +""" +Tests for rh-virt__vm-snapshot-restore per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_restore(self): + content = read_report().lower() + assert "restor" in content, "report should discuss restore operation" + + def test_mentions_snapshot(self): + content = read_report().lower() + assert "snapshot" in content or "backup" in content, "report should mention the snapshot" + + +class TestSkillDependent: + def test_vm_stopped_prerequisite(self): + """Skill: VM must be stopped before restore; stop-and-restore option.""" + c = read_report().lower() + assert any(t in c for t in ["stop before restor", "must be stopped", "stop-and-restore", "vm must be stopped", "halt"]) and ( + "stop" in c and "restor" in c + ), ( + "should require VM stopped before restore" + ) + + def test_destructive_warning(self): + """Skill: Data loss warning; changes since snapshot will be lost.""" + c = read_report().lower() + assert any(t in c for t in ["data loss", "changes since", "will be lost", "overwrite", "destructive", "replace current", "cannot recover"]), ( + "should warn about data loss from restore" + ) + + def test_restore_cr(self): + """Skill: VirtualMachineRestore CR with target and snapshot reference.""" + c = read_report().lower() + assert "virtualmachinerestore" in c and any(t in c for t in ["target", "virtualmachinesnapshotname", "spec"]), ( + "should define VirtualMachineRestore resource" + ) + + def test_post_restore_verification(self): + """Skill: Verify restore complete; status.complete; start VM after.""" + c = read_report().lower() + assert any(t in c for t in ["status.complete", "restore complete", "post-restore", "after restore", "start vm", "start the vm"]) and ( + "restor" in c or "complete" in c or "start" in c + ), ( + "should include post-restore verification or start step" + ) + + def test_typed_confirmation(self): + """Skill: Typed snapshot name confirmation before restore.""" + c = read_report().lower() + assert any(t in c for t in ["type", "typed", "exact name", "to confirm", "snapshot name"]) and ( + "confirm" in c or "type" in c + ), ( + "should require typed snapshot name confirmation" + ) diff --git a/evaluation/without_skills/ocp-admin__cluster-report/environment/Dockerfile b/evaluation/without_skills/ocp-admin__cluster-report/environment/Dockerfile new file mode 100644 index 00000000..5fe00bae --- /dev/null +++ b/evaluation/without_skills/ocp-admin__cluster-report/environment/Dockerfile @@ -0,0 +1,63 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-ocp-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/ocp-admin__cluster-report/environment/docs/multi-cluster-auth.md b/evaluation/without_skills/ocp-admin__cluster-report/environment/docs/multi-cluster-auth.md new file mode 100644 index 00000000..e187471b --- /dev/null +++ b/evaluation/without_skills/ocp-admin__cluster-report/environment/docs/multi-cluster-auth.md @@ -0,0 +1,248 @@ +# Multi-Cluster Authentication with Service Account Tokens + +Set up non-interactive, long-lived authentication for running `cluster-report` across many OpenShift clusters without repeated `oc login` sessions. + +## Overview + +The `cluster-report` skill requires valid kubeconfig contexts for every cluster it reports on. Interactive `oc login --web` opens a browser for each cluster and produces tokens that expire in ~24 hours which make it difficult to do at scale. + +**Solution**: Create a read-only ServiceAccount on each cluster with a non-expiring token. A builder script assembles these tokens into a single merged kubeconfig that the skill uses unchanged. + +## Prerequisites + +- `oc` or `kubectl` CLI +- `python3` (stdlib only, no extra packages) +- `cluster-admin` access on each target cluster (one-time setup only) + +## Quick Start (Automated) + +If you're currently logged into all the clusters you would like to get a report for via `oc login`: + +```bash +# Step 1: Setup — applies RBAC to each cluster, extracts SA tokens +python3 ocp-admin/scripts/cluster-report/build-kubeconfig.py setup --all-contexts + +# Step 2: Build — assembles a merged kubeconfig from the inventory +python3 ocp-admin/scripts/cluster-report/build-kubeconfig.py \ + build --clusters ~/.ocp-clusters/clusters.json --verify + +# Step 3: Use — export and run the skill +export KUBECONFIG=/tmp/cluster-report-kubeconfig +# Then in Claude Code use the skill: /cluster-report +``` + +After the one-time setup, only Steps 2–3 are needed for future report sessions. + +## Manual Setup (Per Cluster) + +If you prefer to set up each cluster individually: + +### 1. Apply RBAC + +> **Required permissions**: The manifest creates cluster-scoped resources (ClusterRole, ClusterRoleBinding), so the user applying it needs `cluster-admin` privileges. This is a one-time setup step. + +```bash +oc login +oc apply -f ocp-admin/scripts/cluster-report/cluster-reporter-rbac.yaml +``` + +This creates: + +- Namespace `cluster-reporter-system` +- ServiceAccount `cluster-reporter` with a read-only ClusterRole +- ClusterRoleBinding `cluster-reporter-binding` (binds the SA to the ClusterRole) +- Token Secret `cluster-reporter-token` (non-expiring) + +### 2. Extract the Token + +```bash +oc get secret cluster-reporter-token -n cluster-reporter-system \ + -o jsonpath='{.data.token}' | base64 -d +``` + +Save this token securely. It grants read-only access to nodes, pods, namespaces, projects, cluster version, and metrics. + +> **AI Safety**: Never display token values in conversation output. Verify tokens are set, but never print or echo their contents. + +### 3. Add to Inventory File + +Create or edit `~/.ocp-clusters/clusters.json`: + +```json +{ + "clusters": [ + { + "name": "prod-us-east", + "api_url": "https://api.prod-us-east.example.com:6443", + "token": "sha256~your-token-here" + } + ] +} +``` + +Set permissions: `chmod 600 ~/.ocp-clusters/clusters.json` + +### 4. Build Kubeconfig + +```bash +python3 ocp-admin/scripts/cluster-report/build-kubeconfig.py \ + build --clusters ~/.ocp-clusters/clusters.json --output ~/.kube/cluster-report-kubeconfig +``` + +## RBAC Permissions + +The `cluster-reporter-readonly` ClusterRole grants the minimum permissions required by the `cluster-report` skill: + + +| Resource | API Group | Verbs | Used By | +| ----------------------- | -------------------- | --------- | ------------------------------------------------------------- | +| nodes, namespaces, pods | core | get, list | `nodes_top`, `resources_list`, `namespaces_list`, `pods_list` | +| clusterversions | config.openshift.io | get | `resources_get` (OpenShift verification) | +| projects | project.openshift.io | list | `projects_list` | +| nodes, pods (metrics) | metrics.k8s.io | get, list | `nodes_top` | + + +No create, update, delete, or watch permissions are granted. + +## Clusters Inventory Format + +The inventory file (`clusters.json`) supports two token modes: + +### Inline Tokens (Simple) + +```json +{ + "clusters": [ + { + "name": "prod-us-east", + "api_url": "https://api.prod-us-east.example.com:6443", + "token": "sha256~abc123..." + } + ] +} +``` + +The file itself contains secrets — keep it out of git and set `chmod 600`. + +### Environment Variable References (More Secure) + +```json +{ + "clusters": [ + { + "name": "prod-us-east", + "api_url": "https://api.prod-us-east.example.com:6443", + "token_env": "CLUSTER_TOKEN_PROD_US_EAST" + } + ] +} +``` + +The file contains no secrets. Load tokens into environment variables from your secrets manager before running `--build`. + +### Optional: CA Certificate + +```json +{ + "clusters": [ + { + "name": "prod-us-east", + "api_url": "https://api.prod-us-east.example.com:6443", + "token": "sha256~abc123...", + "ca_cert": "/path/to/prod-us-east-ca.crt" + } + ] +} +``` + +If `ca_cert` is omitted, TLS verification is skipped (`--insecure-skip-tls-verify`). + +## Script Reference + +### `setup` Subcommand + +```bash +python3 build-kubeconfig.py setup [OPTIONS] +``` + + +| Flag | Description | Default | +| --------------------------- | ----------------------------- | ------------------------------- | +| `--all-contexts` | Setup all kubeconfig contexts | Lists contexts and exits | +| `--contexts ctx1,ctx2` | Setup only specified contexts | — | +| `--output-inventory ` | Inventory file path | `~/.ocp-clusters/clusters.json` | + + +Behavior: + +- Applies `cluster-reporter-rbac.yaml` to each cluster +- Waits up to 15 seconds for the token Secret to populate +- Extracts and saves the token to the inventory file +- Skips unreachable clusters with an error message +- Appends to existing inventory (deduplicates by name) + +### `build` Subcommand + +```bash +python3 build-kubeconfig.py build --clusters [OPTIONS] +``` + + +| Flag | Description | Default | +| ------------------- | -------------------------------- | -------------------------------- | +| `--clusters ` | Inventory file path (required) | — | +| `--output ` | Kubeconfig output path | `/tmp/cluster-report-kubeconfig` | +| `--verify` | Test each context after building | Off | + + +Behavior: + +- Reads inventory, resolves tokens (inline or env var) +- Builds kubeconfig with `kubectl config set-cluster/set-credentials/set-context` +- Partial success: continues on individual failures +- `--verify` tests each context with `cluster-info` +- Outputs JSON summary with success/error counts + +## Token Rotation + +SA token Secrets do not expire, but you may want to rotate them periodically: + +```bash +oc delete secret cluster-reporter-token -n cluster-reporter-system +oc apply -f ocp-admin/scripts/cluster-report/cluster-reporter-rbac.yaml + +oc get secret cluster-reporter-token -n cluster-reporter-system \ + -o jsonpath='{.data.token}' | base64 -d + +python3 build-kubeconfig.py build --clusters ~/.ocp-clusters/clusters.json --verify +``` + +To detect expired or invalid tokens: + +```bash +python3 build-kubeconfig.py build --clusters ~/.ocp-clusters/clusters.json --verify +``` + +## Security Best Practices + +1. **Never commit tokens to git** — add `clusters.json` to `.gitignore` +2. **File permissions** — `chmod 600` on both `clusters.json` and the generated kubeconfig +3. **Prefer `token_env`** — store actual tokens in a secrets manager, not in files +4. **Minimum RBAC** — the ClusterRole grants read-only access only +5. **Dedicated namespace** — the SA lives in `cluster-reporter-system`, not `kube-system` +6. **Generated kubeconfig is ephemeral** — `/tmp/` is fine for session use; for persistent storage use `~/.kube/` with `chmod 600` +7. **Never display tokens in AI conversations** — verify tokens are set but never print, echo, or expose their values in output + +## Troubleshooting + + +| Problem | Cause | Fix | +| ---------------------------------------- | ----------------------------------------- | ------------------------------------------------------------- | +| `--setup` skips a cluster | Not logged in or auth expired | `oc login ` first, then re-run setup | +| `--verify` fails for a cluster | Token expired or Secret deleted | Re-run `--setup --contexts ` for that cluster | +| `cluster-report` shows 401 for a cluster | Token invalid | Same as above — re-run setup for that cluster | +| `cluster-report` shows 403 | SA missing permissions | Re-apply `cluster-reporter-rbac.yaml` on that cluster | +| Token Secret not populated | Token controller slow or SA doesn't exist | Wait and retry; verify SA exists in `cluster-reporter-system` | +| `--build` says "env var not set" | Using `token_env` but env not loaded | Export the token env vars before running `--build` | + + diff --git a/evaluation/without_skills/ocp-admin__cluster-report/environment/mcp-servers/mock-ocp-mcp.py b/evaluation/without_skills/ocp-admin__cluster-report/environment/mcp-servers/mock-ocp-mcp.py new file mode 100644 index 00000000..65e0b6b5 --- /dev/null +++ b/evaluation/without_skills/ocp-admin__cluster-report/environment/mcp-servers/mock-ocp-mcp.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 + +import json +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + +CONTEXTS = [ + ("prod-us-east", "https://api.prod-us-east.example.com:6443", "OpenShift 4.16.3", 6, "high"), + ("prod-eu-west", "https://api.prod-eu-west.example.com:6443", "OpenShift 4.15.12", 4, "moderate"), + ("staging-central", "https://api.staging-central.example.com:6443", "OpenShift 4.16.1", 3, "low"), + ("dev-k8s", "https://dev-k8s.internal.example.com:6443", "Kubernetes", 2, "low"), + ("legacy-dc", "https://legacy-dc.example.com:6443", "OpenShift 4.14", 5, "unknown"), +] + +UNREACHABLE = {"legacy-dc"} +OPENSHIFT_CONTEXTS = {"prod-us-east", "prod-eu-west", "staging-central", "legacy-dc"} +NON_OPENSHIFT = {"dev-k8s"} + + +def _check_context(context): + ctx = (context or "prod-us-east").strip() + if ctx in UNREACHABLE: + raise ConnectionError(f"Connection refused to {ctx}") + valid = {c[0] for c in CONTEXTS} + if ctx not in valid: + raise ValueError(f"Unknown context: {ctx}") + return ctx + + +def _format_tabular(headers, rows): + if not headers or not rows: + return "" + widths = [len(h) for h in headers] + for row in rows: + for i, h in enumerate(headers): + val = str(row.get(h, "")) + widths[i] = max(widths[i], len(val)) + lines = [] + header_line = "".join(h.ljust(w + 2) for h, w in zip(headers, widths)) + lines.append(header_line.rstrip()) + for row in rows: + line = "".join(str(row.get(h, "")).ljust(w + 2) for h, w in zip(headers, widths)) + lines.append(line.rstrip()) + return "\n".join(lines) + + +# Node data for resources_get (Node kind) +NODE_DATA = { + "prod-us-east": { + "node-us-master-1": { + "metadata": {"name": "node-us-master-1", "labels": {"node-role.kubernetes.io/master": ""}}, + "status": {"allocatable": {"cpu": "4", "memory": "16Gi", "pods": "250"}, "conditions": []}, + }, + "node-us-master-2": { + "metadata": {"name": "node-us-master-2", "labels": {"node-role.kubernetes.io/master": ""}}, + "status": {"allocatable": {"cpu": "4", "memory": "16Gi", "pods": "250"}, "conditions": []}, + }, + "node-us-master-3": { + "metadata": {"name": "node-us-master-3", "labels": {"node-role.kubernetes.io/master": ""}}, + "status": {"allocatable": {"cpu": "4", "memory": "16Gi", "pods": "250"}, "conditions": []}, + }, + "node-us-worker-1": { + "metadata": {"name": "node-us-worker-1", "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": { + "allocatable": {"cpu": "32", "memory": "128Gi", "pods": "250", "nvidia.com/gpu": "4"}, + "conditions": [], + }, + }, + "node-us-worker-2": { + "metadata": {"name": "node-us-worker-2", "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": {"allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, "conditions": []}, + }, + "node-us-worker-3": { + "metadata": {"name": "node-us-worker-3", "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": { + "allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250", "nvidia.com/gpu": "4"}, + "conditions": [], + }, + }, + }, + "prod-eu-west": { + "node-eu-master-1": { + "metadata": {"name": "node-eu-master-1", "labels": {"node-role.kubernetes.io/master": ""}}, + "status": {"allocatable": {"cpu": "4", "memory": "16Gi", "pods": "250"}, "conditions": []}, + }, + "node-eu-worker-1": { + "metadata": {"name": "node-eu-worker-1", "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": {"allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, "conditions": []}, + }, + "node-eu-worker-2": { + "metadata": {"name": "node-eu-worker-2", "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": {"allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, "conditions": []}, + }, + "node-eu-worker-3": { + "metadata": {"name": "node-eu-worker-3", "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": {"allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, "conditions": []}, + }, + }, + "staging-central": { + "node-staging-master-1": { + "metadata": {"name": "node-staging-master-1", "labels": {"node-role.kubernetes.io/master": ""}}, + "status": {"allocatable": {"cpu": "4", "memory": "16Gi", "pods": "250"}, "conditions": []}, + }, + "node-staging-worker-1": { + "metadata": {"name": "node-staging-worker-1", "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": {"allocatable": {"cpu": "8", "memory": "32Gi", "pods": "250"}, "conditions": []}, + }, + "node-staging-worker-2": { + "metadata": {"name": "node-staging-worker-2", "labels": {"node-role.kubernetes.io/worker": ""}}, + "status": {"allocatable": {"cpu": "8", "memory": "32Gi", "pods": "250"}, "conditions": []}, + }, + }, + "dev-k8s": { + "node-dev-1": { + "metadata": {"name": "node-dev-1", "labels": {"node-role.kubernetes.io/control-plane": ""}}, + "status": {"allocatable": {"cpu": "4", "memory": "8Gi", "pods": "110"}, "conditions": []}, + }, + "node-dev-2": { + "metadata": {"name": "node-dev-2", "labels": {}}, + "status": {"allocatable": {"cpu": "4", "memory": "8Gi", "pods": "110"}, "conditions": []}, + }, + }, +} + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all kubeconfig contexts with server URLs and cluster info.""" + headers = ["CONTEXT", "SERVER", "VERSION", "NODES", "UTILIZATION"] + rows = [{"CONTEXT": c[0], "SERVER": c[1], "VERSION": c[2], "NODES": str(c[3]), "UTILIZATION": c[4]} for c in CONTEXTS] + return _format_tabular(headers, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: str | None = None, + context: str | None = None, +) -> str: + """Get a single Kubernetes resource by apiVersion, kind, and name.""" + ctx = _check_context(context) + + if apiVersion == "config.openshift.io/v1" and kind == "ClusterVersion": + if ctx in NON_OPENSHIFT: + raise ValueError("ClusterVersion not found (non-OpenShift cluster)") + versions = { + "prod-us-east": "4.16.3", + "prod-eu-west": "4.15.12", + "staging-central": "4.16.1", + "legacy-dc": "4.14", + } + ver = versions.get(ctx, "4.16.0") + return f'{{"apiVersion":"config.openshift.io/v1","kind":"ClusterVersion","metadata":{{"name":"version"}},"status":{{"desired":{{"version":"{ver}"}}}}}}' + + if apiVersion == "v1" and kind == "Node": + nodes = NODE_DATA.get(ctx, {}) + if name not in nodes: + raise ValueError(f"Node {name} not found") + return json.dumps(nodes[name]) + + raise ValueError(f"Unsupported resource: {apiVersion}/{kind}") + + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: str | None = None, + context: str | None = None, +) -> str: + """List Kubernetes resources by apiVersion and kind.""" + ctx = _check_context(context) + + if apiVersion == "v1" and kind == "Node": + nodes = NODE_DATA.get(ctx, {}) + return json.dumps(list(nodes.values())) + + if apiVersion == "v1" and kind == "Namespace": + return namespaces_list(context=ctx) + + raise ValueError(f"Unsupported list: {apiVersion}/{kind}") + + +@mcp.tool() +def nodes_top(context: str | None = None) -> str: + """Return node CPU and memory usage from Metrics Server.""" + ctx = _check_context(context) + + # prod-us-east: node-us-worker-1 (28.4/32=89%, 112.6/128=88%), node-us-worker-3 (14.2/16=89%, 56.8/64=89%) + if ctx == "prod-us-east": + rows = [ + {"NAME": "node-us-master-1", "CPU(cores)": "1.2", "MEMORY(bytes)": "4Gi"}, + {"NAME": "node-us-master-2", "CPU(cores)": "1.1", "MEMORY(bytes)": "3.8Gi"}, + {"NAME": "node-us-master-3", "CPU(cores)": "1.0", "MEMORY(bytes)": "3.6Gi"}, + {"NAME": "node-us-worker-1", "CPU(cores)": "28.4", "MEMORY(bytes)": "112.6Gi"}, + {"NAME": "node-us-worker-2", "CPU(cores)": "8.2", "MEMORY(bytes)": "32Gi"}, + {"NAME": "node-us-worker-3", "CPU(cores)": "14.2", "MEMORY(bytes)": "56.8Gi"}, + ] + elif ctx == "prod-eu-west": + rows = [ + {"NAME": "node-eu-master-1", "CPU(cores)": "0.8", "MEMORY(bytes)": "3Gi"}, + {"NAME": "node-eu-worker-1", "CPU(cores)": "6.2", "MEMORY(bytes)": "24Gi"}, + {"NAME": "node-eu-worker-2", "CPU(cores)": "5.8", "MEMORY(bytes)": "22Gi"}, + {"NAME": "node-eu-worker-3", "CPU(cores)": "7.1", "MEMORY(bytes)": "28Gi"}, + ] + elif ctx == "staging-central": + rows = [ + {"NAME": "node-staging-master-1", "CPU(cores)": "0.5", "MEMORY(bytes)": "2Gi"}, + {"NAME": "node-staging-worker-1", "CPU(cores)": "2.1", "MEMORY(bytes)": "8Gi"}, + {"NAME": "node-staging-worker-2", "CPU(cores)": "1.8", "MEMORY(bytes)": "7Gi"}, + ] + elif ctx == "dev-k8s": + rows = [ + {"NAME": "node-dev-1", "CPU(cores)": "1.2", "MEMORY(bytes)": "3Gi"}, + {"NAME": "node-dev-2", "CPU(cores)": "2.0", "MEMORY(bytes)": "5Gi"}, + ] + else: + rows = [] + + headers = ["NAME", "CPU(cores)", "MEMORY(bytes)"] + return _format_tabular(headers, rows) + + +@mcp.tool() +def pods_list(namespace: str | None = None, context: str | None = None) -> str: + """List pods across namespaces.""" + ctx = _check_context(context) + + if ctx == "prod-us-east": + rows = [ + {"NAMESPACE": "batch-jobs", "NAME": "data-pipeline-batch-abc", "STATUS": "Failed"}, + {"NAMESPACE": "batch-jobs", "NAME": "data-pipeline-batch-def", "STATUS": "Failed"}, + {"NAMESPACE": "ci-cd", "NAME": "image-builder", "STATUS": "CrashLoopBackOff"}, + {"NAMESPACE": "app-platform", "NAME": "deploy-canary", "STATUS": "Pending"}, + {"NAMESPACE": "default", "NAME": "api-server", "STATUS": "Running"}, + {"NAMESPACE": "default", "NAME": "web-frontend", "STATUS": "Running"}, + {"NAMESPACE": "openshift-monitoring", "NAME": "prometheus-0", "STATUS": "Running"}, + ] + elif ctx == "prod-eu-west": + rows = [ + {"NAMESPACE": "security", "NAME": "compliance-scanner-failed", "STATUS": "Failed"}, + {"NAMESPACE": "default", "NAME": "api-eu", "STATUS": "Running"}, + ] + elif ctx == "staging-central": + rows = [ + {"NAMESPACE": "staging-apps", "NAME": "image-pull-broken-pod", "STATUS": "ImagePullBackOff"}, + {"NAMESPACE": "default", "NAME": "staging-api", "STATUS": "Running"}, + ] + elif ctx == "dev-k8s": + rows = [ + {"NAMESPACE": "default", "NAME": "dev-pod-1", "STATUS": "Running"}, + {"NAMESPACE": "kube-system", "NAME": "coredns-xyz", "STATUS": "Running"}, + ] + else: + rows = [] + + headers = ["NAMESPACE", "NAME", "STATUS"] + return _format_tabular(headers, rows) + + +@mcp.tool() +def projects_list(context: str | None = None) -> str: + """List OpenShift projects.""" + ctx = _check_context(context) + if ctx in NON_OPENSHIFT: + raise ValueError("projects_list is OpenShift-only; use namespaces_list for vanilla Kubernetes") + + counts = {"prod-us-east": 21, "prod-eu-west": 16, "staging-central": 12, "legacy-dc": 8} + n = counts.get(ctx, 5) + rows = [{"NAME": f"project-{i}"} for i in range(1, n + 1)] + headers = ["NAME"] + return _format_tabular(headers, rows) + + +@mcp.tool() +def namespaces_list(context: str | None = None) -> str: + """List all namespaces in a cluster.""" + ctx = _check_context(context) + + if ctx == "dev-k8s": + # 6 namespaces for vanilla Kubernetes + rows = [ + {"NAME": "default"}, + {"NAME": "kube-system"}, + {"NAME": "kube-public"}, + {"NAME": "kube-node-lease"}, + {"NAME": "app-dev"}, + {"NAME": "monitoring"}, + ] + else: + # OpenShift: projects map to namespaces + counts = {"prod-us-east": 21, "prod-eu-west": 16, "staging-central": 12} + n = counts.get(ctx, 5) + rows = [{"NAME": f"project-{i}"} for i in range(1, n + 1)] + + headers = ["NAME"] + return _format_tabular(headers, rows) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/without_skills/ocp-admin__cluster-report/instruction.md b/evaluation/without_skills/ocp-admin__cluster-report/instruction.md new file mode 100644 index 00000000..b13ffc9a --- /dev/null +++ b/evaluation/without_skills/ocp-admin__cluster-report/instruction.md @@ -0,0 +1,17 @@ +# Cluster Report Task + +You are an OpenShift cluster administrator. Your operations lead has requested a comprehensive infrastructure health snapshot for the weekly review. Your environment has multiple cluster contexts configured. + +## Requirements +- Discover all available cluster contexts in your environment +- For each accessible OpenShift cluster, report: + - Cluster version and API server URL + - All nodes with their status (Ready/NotReady), roles, and resource utilization (CPU and memory usage vs capacity) + - All projects/namespaces with their status + - Workload counts: total pods, running vs failing, and any pods in error states +- Explicitly note any contexts that are not OpenShift clusters or could not be reached, and explain why +- Highlight any issues that need attention (unhealthy nodes, resource pressure, failing workloads) + +Use MCP tools to examine the clusters. Write the complete cluster report in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/ocp-admin__cluster-report/solution/solve.sh b/evaluation/without_skills/ocp-admin__cluster-report/solution/solve.sh new file mode 100644 index 00000000..62bd7e47 --- /dev/null +++ b/evaluation/without_skills/ocp-admin__cluster-report/solution/solve.sh @@ -0,0 +1,30 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Multi-Cluster Health Report + +## Cluster Discovery +Use configuration_contexts_list for kubeconfig contexts. Verify each with resources_get(apiVersion="config.openshift.io/v1", kind="ClusterVersion", name="version"). + +## Cluster Contexts +| Context | Type | Server | +|---------|------|--------| +| ocp-prod | OpenShift (ClusterVersion detected) | https://api.ocp-prod.example.com:6443 | + +### OpenShift Detection +Check for ClusterVersion resource: config.openshift.io/v1. Non-OpenShift contexts excluded by default. + +## Node Resources +| Node | CPU | Memory | GPUs | +|------|-----|--------|------| +| worker-01 | 16 cores (45% used) | 64Gi (60% used) | 2 | +| worker-02 | 16 cores (30% used) | 64Gi (40% used) | 0 | + +## Pod Status +| Namespace | Running | Pending | Failed | +|-----------|---------|---------|--------| +| default | 5 | 0 | 0 | +| openshift-operators | 12 | 0 | 1 | + +### Generated using assemble.py and aggregate.py helper scripts +Persist MCP output to /tmp/cluster-report/. Manifest with $file refs. Projects_list (fallback namespaces_list for non-OpenShift) +REPORT_EOF diff --git a/evaluation/without_skills/ocp-admin__cluster-report/task.toml b/evaluation/without_skills/ocp-admin__cluster-report/task.toml new file mode 100644 index 00000000..51a06299 --- /dev/null +++ b/evaluation/without_skills/ocp-admin__cluster-report/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "ocp-admin__cluster-report" +name = "ocp-admin Multi-Cluster Health Report Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["ocp-admin", "cluster-report", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/ocp-admin__cluster-report/tests/llm_judge.py b/evaluation/without_skills/ocp-admin__cluster-report/tests/llm_judge.py new file mode 100644 index 00000000..6c379f29 --- /dev/null +++ b/evaluation/without_skills/ocp-admin__cluster-report/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "clusterversion_verification", + "file": "/root/report.md", + "question": "Does the report explicitly mention probing or checking the ClusterVersion resource as the method used to determine whether each context is an OpenShift cluster?", + "reference": "A skilled report should mention 'ClusterVersion' as the API resource used to verify OpenShift status. It should explain that dev-k8s was classified as non-OpenShift because no ClusterVersion resource was found. Simply saying 'vanilla Kubernetes' or 'not OpenShift' without mentioning the ClusterVersion verification mechanism is insufficient." + }, + { + "id": "exclusion_methodology", + "file": "/root/report.md", + "question": "Does the report treat non-OpenShift clusters (like dev-k8s) as EXCLUDED from the detailed report — listing them briefly in an exclusion section — rather than including them as full sections with node/pod details?", + "reference": "A skilled report should have a separate 'Excluded Clusters' or 'Non-OpenShift' section where dev-k8s is listed briefly with the reason for exclusion. A report that includes dev-k8s as a full section with node details, namespaces, and pod data is NOT demonstrating the skill's exclusion methodology." + }, + { + "id": "aggregated_totals", + "file": "/root/report.md", + "question": "Does the report include aggregated totals across all OpenShift clusters — total nodes, total CPU, total memory, total GPUs — in a comparison or summary table?", + "reference": "A skilled report should have a comparison table with a 'Total' row showing aggregate counts (e.g., 13 nodes total, 148 CPU cores, 592 GiB memory, 8 GPUs). Reports that list each cluster's data without cross-cluster aggregation are insufficient." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/ocp-admin__cluster-report/tests/test.sh b/evaluation/without_skills/ocp-admin__cluster-report/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/ocp-admin__cluster-report/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/ocp-admin__cluster-report/tests/test_outputs.py b/evaluation/without_skills/ocp-admin__cluster-report/tests/test_outputs.py new file mode 100644 index 00000000..5c65747c --- /dev/null +++ b/evaluation/without_skills/ocp-admin__cluster-report/tests/test_outputs.py @@ -0,0 +1,105 @@ +""" +Tests for ocp-admin__cluster-report per-skill evaluation. +Baseline tests: any competent agent should pass. +Skill-dependent tests: based on empirical gaps between skilled and unskilled agent outputs. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_cluster(self): + content = read_report().lower() + assert any(t in content for t in ["cluster", "openshift", "node"]), ( + "report should mention cluster" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 200, "report should have substantial content" + + +class TestSkillDependent: + def test_clusterversion_resource(self): + """Skill teaches to probe the ClusterVersion resource to verify OpenShift. + Without skill, agents say 'vanilla Kubernetes' without mentioning the mechanism.""" + c = read_report().lower() + assert "clusterversion" in c or "cluster version resource" in c, ( + "should mention ClusterVersion resource as the OpenShift verification method" + ) + + def test_aggregated_cross_cluster_totals(self): + """Skill teaches a comparison table with aggregated totals across clusters. + Without skill, agents report each cluster separately without totals.""" + c = read_report().lower() + has_total_label = "total" in c or "aggregate" in c or "combined" in c + has_aggregate_context = any(t in c for t in [ + "total node", "total cpu", "total memory", "total gpu", + "across cluster", "combined resource", "aggregate", + ]) or (has_total_label and any(t in c for t in ["node", "cpu", "core", "memory", "gi"])) + assert has_total_label and has_aggregate_context, ( + "should include aggregated cross-cluster totals (total nodes, CPU, memory)" + ) + + def test_non_openshift_exclusion(self): + """Skill teaches to EXCLUDE non-OpenShift clusters from detailed reporting. + Without skill, agents include dev-k8s as a full section with nodes/pods/namespaces.""" + c = read_report().lower() + has_exclusion = any(t in c for t in [ + "excluded", "exclude", "excluded by default", "not included", + "omitted", "non-openshift", + ]) + assert has_exclusion and "dev-k8s" in c, ( + "should explicitly exclude non-OpenShift clusters from detailed data" + ) + + def test_unreachable_reporting(self): + """Both agents should mention unreachable clusters, but skill teaches categorization.""" + c = read_report().lower() + assert "legacy-dc" in c and any(t in c for t in [ + "unreachable", "connection refused", "offline", + ]), "should report legacy-dc as unreachable" + + def test_gpu_inventory(self): + """Skill template includes GPU column — moderate discriminator.""" + c = read_report().lower() + assert "gpu" in c, "should include GPU information" + + def test_version_numbers(self): + """Both agents get versions from MCP, but skill ensures all clusters are covered.""" + c = read_report() + versions = sum(1 for v in ["4.16.3", "4.15.12", "4.16.1"] if v in c) + assert versions >= 2, "should report exact version numbers for multiple clusters" + + def test_multi_cluster_tooling(self): + """Docs teach multi-cluster tooling/automation for consistent reporting. + Without docs, agents rely on manual kubectl context switching.""" + c = read_report().lower() + assert any(t in c for t in [ + "build-kubeconfig", "kubeconfig.py", "cluster-reporter", + "multi-cluster", "multiple context", "all contexts", + "setup script", "automation", + ]), "should reference multi-cluster tooling or automation approach" + + def test_rbac_for_reporting(self): + """Docs teach read-only RBAC (ClusterRole/ServiceAccount) for cluster reporting + instead of admin credentials.""" + c = read_report().lower() + assert any(t in c for t in [ + "cluster-reporter-readonly", "cluster-reporter-system", + "readonly", "read-only", "clusterrole", + "service account", "serviceaccount", "rbac", + "least privilege", "non-admin", + ]), "should reference read-only RBAC for cluster reporting" diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/Dockerfile b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/Dockerfile new file mode 100644 index 00000000..93448fa3 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/Dockerfile @@ -0,0 +1,71 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "rhoai": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-rhoai-mcp.py"] \ + }, \ + "observability": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-observability-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/common-issues.md b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/examples/model-deploy.md b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/examples/model-deploy.md new file mode 100644 index 00000000..dcd79e44 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/examples/model-deploy.md @@ -0,0 +1,160 @@ +# /model-deploy Examples + +Example interactions demonstrating the model-deploy skill workflow. + +## Example 1: Deploy Llama 3.1 8B with vLLM + +``` +User: "Deploy Llama 3.1 8B on my cluster" + +Agent: [Invokes /model-deploy skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand runtime capabilities. + +Step 1 of 10: Gathering deployment information. + +| Setting | Value | Source | +|---------|-------|--------| +| Model | Llama 3.1 8B Instruct | user input | +| Runtime | TBD | auto-detect | +| Namespace | ? | needs input | +| Model Source | ? | needs input | +| Deployment Mode | Serverless | default | + +What namespace should I deploy to, and where are the model weights stored? +(e.g., S3 bucket, OCI registry, PVC) + +User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" + +Agent: Step 2 of 10: Determining runtime. + +I consulted supported-runtimes.md to understand runtime capabilities. + +Based on Llama 3.1 8B (open-source HuggingFace model): + +Recommended runtime: vLLM +Reason: Default runtime for open-source LLMs. Broad compatibility, + OpenAI-compatible API, no additional setup required. + +Proceed with vLLM? (yes / no / suggest alternative) + +User: "yes" + +Agent: Step 3 of 10: Looking up hardware profile. + +I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. + +| Requirement | Value | +|-------------|-------| +| GPUs | 1x A10G/L4/A100/H100 | +| VRAM | 16GB | +| Key Args | --max-model-len=4096 --tool-call-parser hermes | + +Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). +Ensure your cluster has at least 1 GPU with 16GB VRAM. + +Step 5 of 10: Skipped (NIM platform check not needed for vLLM). + +Step 6-7 of 10: Generating InferenceService YAML. + +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: llama-3-1-8b + namespace: ai-models + annotations: + serving.kserve.io/deploymentMode: Serverless +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: vllm-runtime + storageUri: s3://my-bucket/llama-3.1-8b + resources: + limits: + nvidia.com/gpu: "1" + requests: + cpu: "4" + memory: "16Gi" + args: + - --max-model-len=4096 + - --tool-call-parser=hermes + +| Setting | Value | +|---------|-------| +| Model | llama-3-1-8b | +| Runtime | vllm-runtime | +| GPUs | 1 | +| Namespace | ai-models | +| Mode | Serverless | + +Proceed with creating this InferenceService? (yes/no/modify) + +User: "yes" + +Agent: Step 8 of 10: Creating InferenceService... + [Creates via resources_create_or_update] + + InferenceService created. Monitoring rollout... + +Step 9 of 10: Monitoring rollout. + +| Stage | Status | +|-------|--------| +| Pod Scheduled | Complete | +| Image Pulled | Complete | +| Container Started | Complete | +| Model Loaded | Complete | +| Ready | Complete | + +Step 10 of 10: Deployment complete! + +Model Deployment Successful + +Model: llama-3-1-8b +Runtime: vLLM +Namespace: ai-models +GPUs: 1 + +Inference Endpoint: + URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com + API: OpenAI-compatible REST + +Quick Test: + curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' + +Next Steps: + Monitor performance: "/ai-observability llama-3-1-8b" +``` + +## Example 2: Deploy a Model with NIM (NIM Not Set Up) + +``` +User: "Deploy Llama 3.1 70B with NIM" + +Agent: [Invokes /model-deploy skill] + ... + Step 2: User requested NIM runtime. + + Step 5: Checking NIM platform... + +NIM platform is not set up in namespace ai-models. + +The NIM runtime requires a one-time platform setup. Would you like to: +1. Run /nim-setup now to configure the NIM platform +2. Switch to vLLM runtime instead (no setup required) +3. Abort deployment + +User: "1" + +Agent: [Invokes /nim-setup skill — walks through NGC credential setup] + ... + NIM Platform Setup Complete. + + Resuming model deployment... + [Continues from Step 6 with NIM runtime] +``` diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/examples/nim-setup.md b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/examples/nim-setup.md new file mode 100644 index 00000000..ccfc1a5d --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/examples/nim-setup.md @@ -0,0 +1,115 @@ +# /nim-setup Examples + +Example interactions demonstrating the nim-setup skill workflow. + +## Example 1: First-Time NIM Setup + +``` +User: "Set up NIM on my cluster" + +Agent: [Invokes /nim-setup skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand NIM platform requirements. + +Let me verify the prerequisites for NIM setup. + +Step 1 of 7: Checking GPU Operator and NFD Operator... + +GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) +NFD Operator: nfd.v4.19.0 (Succeeded) + +Both operators are installed and healthy. + +Step 2 of 7: I need your NGC credentials. + +To set up NIM, I need your NVIDIA NGC API key. +You can generate one at: https://ngc.nvidia.com/setup/api-key + +Please provide: +1. Your NGC API key +2. The target namespace for NIM resources + +User: "Here's my key: [user provides NGC API key], namespace: ai-models" + +Agent: Step 3 of 7: Creating NGC image pull secret... + +I'll create a docker-registry secret for pulling NIM images from nvcr.io: + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-image-pull-secret + namespace: ai-models + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] + +Should I create this image pull secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + Image pull secret created successfully. + +Step 4 of 7: Creating NGC API key secret... + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-api-key + namespace: ai-models + type: Opaque + stringData: + NGC_API_KEY: "[REDACTED]" + +Should I create this API key secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + API key secret created successfully. + +Step 5 of 7: Creating NIM Account CR... + + apiVersion: nim.opendatahub.io/v1 + kind: Account + metadata: + name: nim-account + namespace: ai-models + spec: + apiKeySecret: + name: ngc-api-key + imagePullSecret: + name: ngc-image-pull-secret + +Should I create this NIM Account CR in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates Account CR via resources_create_or_update] + Account CR created successfully. + +Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) + +User: "no" + +Agent: Step 7 of 7: Validating NIM platform readiness... + +NIM Platform Setup Complete + +Account CR: `nim-account` (namespace: ai-models) + Status: Ready + +NGC Credentials: + Image Pull Secret: ngc-image-pull-secret [created] + API Key Secret: ngc-api-key [created] + +NIM ServingRuntimes Available: + - nim-llama-3-1-8b-instruct + - nim-llama-3-1-70b-instruct + - nim-mistral-7b-instruct + +Next Steps: + To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" +``` diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/live-doc-lookup.md b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/references/known-model-profiles.md b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/references/known-model-profiles.md new file mode 100644 index 00000000..40148981 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/references/known-model-profiles.md @@ -0,0 +1,83 @@ +--- +title: Known Model Hardware Profiles +category: references +tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] +semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] +use_cases: [model-deploy, debug-inference, ai-observability] +last_updated: 2026-03-10 +--- + +# Known Model Hardware Profiles + +Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). + +**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. + +## Model Source Conventions + +Each model lists a recommended `storageUri` with its authentication requirements: +- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** +- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). +- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. + +When the user does not specify a model source, use the `hf://` URI listed in the profile below. + +## Llama 3.x (Meta) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | +| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | +| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | +| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | + +- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) +- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly + +## Granite 3.x (IBM/Red Hat) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | +| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | + +- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) +- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser granite --chat-template` +- Red Hat-supported model family on RHOAI + +## Mixtral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | +| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) +- Mixture-of-Experts architecture: only ~13B/45B parameters active per token + +## Mistral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | +| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) + +## When a Model Is Not Listed + +If the requested model is not in this file, the agent MUST use the live doc lookup protocol: + +1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure +2. Fetch hardware specs from the appropriate source +3. Report findings to the user before proceeding with deployment + +Common cases requiring live lookup: +- Newly released models (after this file's last update) +- Domain-specific fine-tuned models +- Models with custom quantization +- NIM-specific optimized profiles diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/references/supported-runtimes.md b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/references/supported-runtimes.md new file mode 100644 index 00000000..9d018651 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/references/supported-runtimes.md @@ -0,0 +1,104 @@ +--- +title: Supported Serving Runtimes +category: references +tags: [runtimes, vllm, nim, caikit, tgis, serving] +semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] +use_cases: [model-deploy, serving-runtime-config, nim-setup] +last_updated: 2026-02-26 +--- + +# Supported Serving Runtimes + +This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. + +## Runtime Comparison + +| Runtime | API Type | Model Format | GPU Support | License | Setup Required | +|---------|----------|-------------|-------------|---------|----------------| +| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | +| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | +| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | +| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | + +## vLLM + +**Default runtime for most open-source models.** + +- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) +- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized +- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) +- **Model source**: S3-compatible storage, OCI registry, PVC, URI +- **Key features**: + - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) + - PagedAttention for efficient memory management + - Tensor parallelism for multi-GPU inference + - Tool/function calling support (`--tool-call-parser`) + - Continuous batching for high throughput +- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name + +## NVIDIA NIM + +**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** + +- **API**: OpenAI-compatible REST +- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog +- **GPU support**: NVIDIA only (requires specific GPU models per model profile) +- **Model source**: NGC model catalog (pulled automatically via NGC credentials) +- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) +- **Key features**: + - TensorRT-LLM optimization for lower latency + - Pre-compiled model engines (no compilation on first load) + - Optimized and generic profiles per GPU type + - Automatic model download from NGC +- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. +- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM +- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. +- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state + +## Caikit+TGIS + +**Red Hat's Caikit format with Text Generation Inference Server.** + +- **API**: gRPC (not REST) +- **Model formats**: Caikit format (requires conversion from HuggingFace) +- **GPU support**: NVIDIA +- **Model source**: S3-compatible storage +- **Key features**: + - Red Hat-supported runtime + - gRPC API for streaming inference + - Integrated with RHOAI model serving platform +- **When to choose**: When the model is already in Caikit format or when gRPC API is required. +- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name + +## Custom Runtimes + +**User-provided ServingRuntime CRs for unsupported frameworks.** + +- **API**: Defined by the custom runtime +- **Model formats**: Defined by the custom runtime +- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. +- **How to create**: Use `/serving-runtime-config` skill +- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility + +## Runtime Selection Decision Tree + +``` +Is the user's preferred runtime explicitly stated? +├── Yes → Use that runtime +└── No → Continue + +Is the model available in the NGC NIM catalog? +├── Yes → Suggest NIM (with vLLM as fallback) +│ Note: Requires /nim-setup and NVIDIA GPUs +└── No → Continue + +Is the model in Caikit format? +├── Yes → Caikit+TGIS +└── No → Continue + +Is the model a standard open-source LLM (HuggingFace-compatible)? +├── Yes → vLLM (default) +└── No → Custom runtime via /serving-runtime-config +``` diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/skill-conventions.md b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-observability-mcp.py b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-observability-mcp.py new file mode 100644 index 00000000..f150dcff --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-observability-mcp.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +"""Mock Observability MCP server for SkillsBench rh-ai-engineer__ai-observability task. + +Simulates Prometheus/Grafana-style metrics for inference services: latency, +throughput, error rates, GPU utilization, resource usage, and alerts. + +Scenario (aligned with rhoai/openshift mocks): +- ml-production namespace: + - text-gen-legacy (Mistral 7B on vLLM): OOMKilled; before crash: 22GB/24GB GPU, + p99=2800ms, throughput=3 req/s, error rate=15% + - nim-llama-prod (Llama 3.1 8B on NIM): not running, no metrics (empty/error) + - sentiment-classifier: running well, 4GB/24GB GPU, p99=45ms, throughput=150 req/s, + error rate=0.1% +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("observability") + +# ── Mock metrics data ────────────────────────────────────────────────────── + +# text-gen-legacy: OOMKilled, metrics from before crash +MODEL_METRICS = { + "ml-production": { + "text-gen-legacy": { + "status": "OOMKilled", + "latency_ms": {"p50": 1200, "p95": 2100, "p99": 2800}, + "throughput_req_per_sec": 3.0, + "error_rate_percent": 15.0, + "input_tokens_per_sec": 45, + "output_tokens_per_sec": 12, + "total_requests_24h": 259200, # 3 * 86400 + }, + "nim-llama-prod": None, # not running, no metrics + "sentiment-classifier": { + "status": "Running", + "latency_ms": {"p50": 18, "p95": 38, "p99": 45}, + "throughput_req_per_sec": 150.0, + "error_rate_percent": 0.1, + "input_tokens_per_sec": 1200, + "output_tokens_per_sec": 50, + "total_requests_24h": 12960000, + }, + }, +} + +GPU_UTILIZATION = { + "ml-production": [ + { + "pod": "text-gen-legacy-predictor-00001-abc12", + "model": "text-gen-legacy", + "gpu_memory_used_gb": 22.0, + "gpu_memory_total_gb": 24.0, + "gpu_memory_utilization_percent": 91.7, + "gpu_compute_utilization_percent": 35.0, + "status": "OOMKilled", + }, + { + "pod": "sentiment-classifier-predictor-00001-xyz99", + "model": "sentiment-classifier", + "gpu_memory_used_gb": 4.0, + "gpu_memory_total_gb": 24.0, + "gpu_memory_utilization_percent": 16.7, + "gpu_compute_utilization_percent": 42.0, + "status": "Running", + }, + # nim-llama-prod: no pod + ], +} + +RESOURCE_USAGE = { + "ml-production": [ + { + "pod": "text-gen-legacy-predictor-00001-abc12", + "model": "text-gen-legacy", + "cpu_request": "4", + "cpu_limit": "4", + "memory_request": "16Gi", + "memory_limit": "16Gi", + "cpu_actual_usage": "3.2", + "memory_actual_usage_mib": 16384, + "status": "CrashLoopBackOff", + }, + { + "pod": "sentiment-classifier-predictor-00001-xyz99", + "model": "sentiment-classifier", + "cpu_request": "2", + "cpu_limit": "4", + "memory_request": "8Gi", + "memory_limit": "16Gi", + "cpu_actual_usage": "1.1", + "memory_actual_usage_mib": 4096, + "status": "Running", + }, + ], +} + +PROMETHEUS_ALERTS = { + "ml-production": [ + { + "name": "InferenceServiceOOMKilled", + "severity": "critical", + "state": "firing", + "summary": "text-gen-legacy predictor pod OOMKilled", + "description": "Container kserve-container was OOMKilled (exit code 137). " + "GPU memory exhausted during KV cache allocation.", + "labels": { + "inference_service": "text-gen-legacy", + "namespace": "ml-production", + }, + }, + { + "name": "HighInferenceLatency", + "severity": "warning", + "state": "firing", + "summary": "text-gen-legacy p99 latency > 2000ms", + "description": "Inference latency p99 is 2800ms, exceeding threshold of 2000ms.", + "labels": { + "inference_service": "text-gen-legacy", + "namespace": "ml-production", + }, + }, + { + "name": "HighErrorRate", + "severity": "warning", + "state": "firing", + "summary": "text-gen-legacy error rate 15%", + "description": "Inference error rate is 15%, exceeding threshold of 5%.", + "labels": { + "inference_service": "text-gen-legacy", + "namespace": "ml-production", + }, + }, + ], +} + + +# ── Tools ────────────────────────────────────────────────────────────────── + + +@mcp.tool() +def query_model_metrics( + model_name: str, + namespace: str, + metric_type: str = "all", +) -> str: + """Query inference metrics for a model. Returns latency (p50/p95/p99), throughput + (requests/sec), error rates, and token counts. + + metric_type: 'all', 'latency', 'throughput', 'errors', or 'tokens' + """ + ns_data = MODEL_METRICS.get(namespace) + if not ns_data: + return json.dumps({"error": f"Namespace '{namespace}' not found"}, indent=2) + + metrics = ns_data.get(model_name) + if metrics is None: + return json.dumps({ + "error": f"No metrics for model '{model_name}' in namespace '{namespace}'. " + "Model may not be running (e.g., nim-llama-prod has no pods).", + "model_name": model_name, + "namespace": namespace, + }, indent=2) + + result = { + "model_name": model_name, + "namespace": namespace, + "status": metrics["status"], + } + + if metric_type in ("all", "latency"): + result["latency_ms"] = metrics["latency_ms"] + if metric_type in ("all", "throughput"): + result["throughput_req_per_sec"] = metrics["throughput_req_per_sec"] + result["total_requests_24h"] = metrics.get("total_requests_24h") + if metric_type in ("all", "errors"): + result["error_rate_percent"] = metrics["error_rate_percent"] + if metric_type in ("all", "tokens"): + result["input_tokens_per_sec"] = metrics["input_tokens_per_sec"] + result["output_tokens_per_sec"] = metrics["output_tokens_per_sec"] + + return json.dumps(result, indent=2) + + +@mcp.tool() +def query_gpu_utilization(namespace: str) -> str: + """Query GPU memory used/total and compute utilization per inference pod.""" + pods = GPU_UTILIZATION.get(namespace, []) + if not pods: + return json.dumps({ + "namespace": namespace, + "pods": [], + "message": "No GPU-backed inference pods found in namespace.", + }, indent=2) + return json.dumps({ + "namespace": namespace, + "pods": pods, + }, indent=2) + + +@mcp.tool() +def query_resource_usage(namespace: str) -> str: + """Query actual CPU/memory usage vs requests/limits for inference pods.""" + pods = RESOURCE_USAGE.get(namespace, []) + if not pods: + return json.dumps({ + "namespace": namespace, + "pods": [], + "message": "No inference pods found in namespace.", + }, indent=2) + return json.dumps({ + "namespace": namespace, + "pods": pods, + }, indent=2) + + +@mcp.tool() +def list_prometheus_alerts(namespace: str) -> str: + """List firing Prometheus alerts related to inference services in the namespace.""" + alerts = PROMETHEUS_ALERTS.get(namespace, []) + return json.dumps({ + "namespace": namespace, + "alerts": alerts, + "firing_count": len(alerts), + }, indent=2) + + +@mcp.tool() +def get_model_performance_summary(namespace: str) -> str: + """Get aggregated performance data across all models in the namespace.""" + ns_data = MODEL_METRICS.get(namespace) + if not ns_data: + return json.dumps({"error": f"Namespace '{namespace}' not found"}, indent=2) + + models = [] + for name, metrics in ns_data.items(): + if metrics is None: + models.append({ + "model_name": name, + "status": "NotRunning", + "error": "No metrics available (pod not created or not running)", + }) + else: + models.append({ + "model_name": name, + "status": metrics["status"], + "latency_p99_ms": metrics["latency_ms"]["p99"], + "throughput_req_per_sec": metrics["throughput_req_per_sec"], + "error_rate_percent": metrics["error_rate_percent"], + }) + + return json.dumps({ + "namespace": namespace, + "models": models, + }, indent=2) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..e7a4d11c --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +"""Mock OpenShift MCP server for SkillsBench rh-ai-engineer task. + +Simulates Kubernetes resource CRUD, pod management, logs, and events. + +Key scenario elements: +- LimitRange in namespaces: min CPU=100m, min memory=128Mi + (conflicts with KServe sidecar containers hardcoded at 10m CPU/15Mi memory) +- GPU node with custom taint ai-workload=true:NoSchedule +- NIM Account CR in ml-production: not ready (NGC credentials invalid) +- text-gen-legacy pods: OOMKilled (max-model-len=32768 on A10G) +- nim-llama-prod: no pods created (Account CR not ready) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + +# ── Cluster state ──────────────────────────────────────────────────────── + +GPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "gpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + "nvidia.com/gpu.present": "true", + "nvidia.com/gpu.product": "NVIDIA-A10G", + }, + }, + "spec": { + "taints": [ + { + "key": "ai-workload", + "value": "true", + "effect": "NoSchedule", + }, + ], + }, + "status": { + "allocatable": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "capacity": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "conditions": [ + {"type": "Ready", "status": "True"}, + ], + }, +} + +CPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "cpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + }, + }, + "spec": {"taints": []}, + "status": { + "allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "capacity": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +MASTER_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "master-1", + "labels": { + "node-role.kubernetes.io/master": "", + "node-role.kubernetes.io/control-plane": "", + }, + }, + "spec": { + "taints": [ + {"key": "node-role.kubernetes.io/master", "effect": "NoSchedule"}, + ], + }, + "status": { + "allocatable": {"cpu": "8", "memory": "32Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +ALL_NODES = [GPU_NODE, CPU_NODE, MASTER_NODE] + +# LimitRange applied by cluster policy to all DS project namespaces +NAMESPACE_LIMITRANGE = { + "apiVersion": "v1", + "kind": "LimitRange", + "metadata": { + "name": "default-limits", + }, + "spec": { + "limits": [ + { + "type": "Container", + "default": { + "cpu": "2", + "memory": "4Gi", + }, + "defaultRequest": { + "cpu": "500m", + "memory": "256Mi", + }, + "min": { + "cpu": "100m", + "memory": "128Mi", + }, + "max": { + "cpu": "32", + "memory": "128Gi", + }, + }, + ], + }, +} + +NIM_ACCOUNT_CR = { + "apiVersion": "nim.opendatahub.io/v1", + "kind": "Account", + "metadata": { + "name": "nim-account", + "namespace": "ml-production", + }, + "spec": { + "apiKeySecret": { + "name": "ngc-api-key", + }, + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "NGCCredentialsInvalid", + "message": "NGC API key validation failed: 401 Unauthorized. " + "The API key in secret 'ngc-api-key' is expired or invalid. " + "Re-create the secret with a valid NGC API key from " + "https://ngc.nvidia.com/setup/api-key and restart the " + "Account reconciliation.", + "lastTransitionTime": "2026-03-14T12:00:00Z", + }, + ], + "nimPullSecretStatus": "Failed", + "nimConfigStatus": "Pending", + }, +} + +SERVING_RUNTIME_VLLM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "vllm-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "vLLM", "version": "1", "autoSelect": True}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "quay.io/modh/vllm:rhoai-2.16", + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + }, + ], + }, +} + +SERVING_RUNTIME_NIM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "nim-serving-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "NIM", "version": "1"}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "nvcr.io/nim/meta/llama-3.1-8b-instruct:latest", + "ports": [{"containerPort": 8000, "protocol": "TCP"}], + "env": [ + {"name": "NGC_API_KEY", "valueFrom": { + "secretKeyRef": {"name": "ngc-api-key", "key": "api_key"}, + }}, + ], + }, + ], + }, +} + +PODS_BY_NAMESPACE = { + "ml-production": [ + { + "name": "text-gen-legacy-predictor-00001-abc12", + "namespace": "ml-production", + "status": "CrashLoopBackOff", + "restarts": 5, + "node": "gpu-worker-1", + "containers": [ + { + "name": "kserve-container", + "state": "waiting", + "reason": "CrashLoopBackOff", + "last_termination_reason": "OOMKilled", + "last_termination_exit_code": 137, + }, + ], + "labels": { + "serving.kserve.io/inferenceservice": "text-gen-legacy", + }, + "gpu": "1", + }, + # nim-llama-prod: NO pods created (Account CR not ready) + ], +} + +POD_LOGS = { + "text-gen-legacy-predictor-00001-abc12": ( + "INFO 2026-03-01 10:00:00 vllm_engine.py:125] vLLM engine starting...\n" + "INFO 2026-03-01 10:00:01 config.py:89] Model: mistralai/Mistral-7B-Instruct-v0.3\n" + "INFO 2026-03-01 10:00:01 config.py:92] max_model_len = 32768\n" + "INFO 2026-03-01 10:00:02 gpu_executor.py:45] GPU 0: NVIDIA A10G (24576 MiB)\n" + "INFO 2026-03-01 10:00:03 model_runner.py:88] Loading model weights...\n" + "INFO 2026-03-01 10:00:15 model_runner.py:112] Model weights loaded: 13.5 GiB\n" + "INFO 2026-03-01 10:00:15 worker.py:201] Allocating KV cache...\n" + "ERROR 2026-03-01 10:00:16 worker.py:215] torch.cuda.OutOfMemoryError: " + "CUDA out of memory. Tried to allocate 28.5 GiB for KV cache but only " + "10.1 GiB available after loading model weights (13.5 GiB).\n" + "ERROR 2026-03-01 10:00:16 vllm_engine.py:178] Engine failed to start\n" + "Traceback (most recent call last):\n" + " File \"/opt/vllm/vllm/engine/engine.py\", line 175, in start\n" + " self._init_kv_cache()\n" + " File \"/opt/vllm/vllm/worker/worker.py\", line 215, in _init_kv_cache\n" + " raise torch.cuda.OutOfMemoryError(msg)\n" + "torch.cuda.OutOfMemoryError: CUDA out of memory\n" + ), +} + +EVENTS_BY_NAMESPACE = { + "ml-production": [ + { + "type": "Warning", + "reason": "BackOff", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Back-off restarting failed container kserve-container in pod " + "text-gen-legacy-predictor-00001-abc12", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Warning", + "reason": "OOMKilled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Container kserve-container was OOMKilled (exit code 137). " + "GPU memory exhausted during KV cache allocation.", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Normal", + "reason": "Scheduled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Successfully assigned ml-production/" + "text-gen-legacy-predictor-00001-abc12 to gpu-worker-1", + "count": 1, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-02-28T08:00:00Z", + }, + { + "type": "Warning", + "reason": "NIMAccountNotReady", + "object": "InferenceService/nim-llama-prod", + "message": "NIM Account 'nim-account' in namespace 'ml-production' " + "is not ready", + "count": 12, + "first_timestamp": "2026-03-14T12:00:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + { + "type": "Warning", + "reason": "ImagePullBackOff", + "object": "InferenceService/nim-llama-prod", + "message": "Failed to pull image 'nvcr.io/nim/meta/llama-3.1-8b-instruct:" + "latest': unauthorized: authentication required", + "count": 8, + "first_timestamp": "2026-03-14T12:05:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + ], +} + + +# ── Resource tools ─────────────────────────────────────────────────────── + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: str = "", +) -> str: + """Get a single Kubernetes resource by apiVersion, kind, and name.""" + if kind == "Node": + for node in ALL_NODES: + if node["metadata"]["name"] == name: + return json.dumps(node, indent=2) + raise ValueError(f"Node '{name}' not found") + + if kind == "ServingRuntime": + if name == "vllm-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_VLLM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + if name == "nim-serving-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_NIM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + raise ValueError(f"ServingRuntime '{name}' not found in namespace '{namespace}'") + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps(lr, indent=2) + + if kind == "Account" and "nim" in apiVersion.lower(): + if namespace == "ml-production" and name == "nim-account": + return json.dumps(NIM_ACCOUNT_CR, indent=2) + raise ValueError( + f"Account '{name}' not found in namespace '{namespace}'" + ) + + if kind == "ClusterVersion" and apiVersion == "config.openshift.io/v1": + return json.dumps({ + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterVersion", + "metadata": {"name": "version"}, + "status": {"desired": {"version": "4.16.3"}}, + }) + + raise ValueError(f"Resource {apiVersion}/{kind}/{name} not found") + + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: str = "", + labelSelector: str = "", +) -> str: + """List Kubernetes resources by apiVersion and kind.""" + if kind == "Node": + nodes = ALL_NODES + if labelSelector: + parts = labelSelector.split("=", 1) + key = parts[0] + value = parts[1] if len(parts) > 1 else "" + nodes = [ + n for n in nodes + if n["metadata"]["labels"].get(key) == value + ] + return json.dumps(nodes, indent=2) + + if kind == "Service" and apiVersion == "serving.knative.dev/v1": + return json.dumps({ + "kind": "ServiceList", + "apiVersion": "serving.knative.dev/v1", + "items": [], + "metadata": {}, + }) + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps({ + "kind": "LimitRangeList", + "items": [lr], + }) + + if kind == "InferenceService": + return json.dumps({ + "kind": "InferenceServiceList", + "items": [], + }) + + raise ValueError(f"Unsupported list: {apiVersion}/{kind}") + + +@mcp.tool() +def pods_list( + namespace: str, + labelSelector: str = "", +) -> str: + """List pods in a namespace with optional label selector.""" + pods = PODS_BY_NAMESPACE.get(namespace, []) + + if labelSelector: + key, _, value = labelSelector.partition("=") + pods = [p for p in pods if p.get("labels", {}).get(key) == value] + + results = [] + for pod in pods: + results.append({ + "name": pod["name"], + "namespace": pod["namespace"], + "status": pod["status"], + "restarts": pod.get("restarts", 0), + "node": pod.get("node", ""), + "containers": pod.get("containers", []), + "gpu": pod.get("gpu", "0"), + }) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def pods_log( + namespace: str, + name: str, + container: str = "", +) -> str: + """Get logs from a pod container.""" + logs = POD_LOGS.get(name) + if logs is None: + raise ValueError(f"Pod '{name}' not found in namespace '{namespace}'") + return logs + + +@mcp.tool() +def events_list(namespace: str) -> str: + """List events in a namespace.""" + events = EVENTS_BY_NAMESPACE.get(namespace, []) + return json.dumps(events, indent=2) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-rhoai-mcp.py b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-rhoai-mcp.py new file mode 100644 index 00000000..0ae9e4cb --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/mcp-servers/mock-rhoai-mcp.py @@ -0,0 +1,780 @@ +#!/usr/bin/env python3 +"""Mock RHOAI MCP server for SkillsBench rh-ai-engineer task. + +Simulates Red Hat OpenShift AI operations: Data Science Projects, +model serving, data connections, serving runtimes, inference services. + +Scenario: +- ml-production: existing project with two broken deployments + - text-gen-legacy: vLLM OOMKilled (max-model-len=32768 on A10G) + - nim-llama-prod: NIM failing (Account CR not ready, NGC creds invalid) +- fraud-detection: does not exist yet (agent creates it) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("rhoai") + +# ── In-memory state ────────────────────────────────────────────────────── + +PROJECTS = { + "ml-production": { + "name": "ml-production", + "display_name": "ML Production", + "description": "Production ML workloads", + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": "single", + "pipeline_server": True, + }, +} + +DATA_CONNECTIONS = { + "ml-production": [ + { + "name": "prod-model-store", + "type": "S3", + "bucket": "ml-models-prod", + "endpoint": "https://s3.us-east-1.amazonaws.com", + "region": "us-east-1", + }, + ], +} + +SERVING_RUNTIMES = { + "__platform_templates__": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "REST", + "supported_model_formats": [ + {"name": "vLLM", "version": "1", "autoSelect": True} + ], + }, + { + "name": "caikit-tgis-runtime", + "display_name": "Caikit+TGIS ServingRuntime", + "model_formats": ["caikit"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "gRPC", + }, + ], + "ml-production": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + { + "name": "nim-serving-runtime", + "display_name": "NVIDIA NIM ServingRuntime", + "model_formats": ["NIM"], + "requires_instantiation": False, + "source": "nim-account", + "api_protocol": "REST", + }, + { + "name": "ovms-1", + "display_name": "OpenVINO Model Server", + "model_formats": ["openvino_ir", "onnx"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + ], +} + +INFERENCE_SERVICES = { + "ml-production": { + "text-gen-legacy": { + "name": "text-gen-legacy", + "namespace": "ml-production", + "runtime": "vllm-runtime", + "model_format": "vLLM", + "storage_uri": "hf://mistralai/Mistral-7B-Instruct-v0.3", + "display_name": "Mistral 7B Legacy", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "16Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "PredictorFailed", + "message": "Predictor pod is not ready", + }, + { + "type": "PredictorReady", + "status": "False", + "reason": "ContainerCrashLoop", + "message": "Container kserve-container terminated: " + "OOMKilled (exit code 137). 5 restarts.", + }, + { + "type": "IngressReady", + "status": "True", + "reason": "IngressReady", + "message": "Ingress is ready", + }, + ], + "age": "3d", + }, + "nim-llama-prod": { + "name": "nim-llama-prod", + "namespace": "ml-production", + "runtime": "nim-serving-runtime", + "model_format": "NIM", + "storage_uri": "nim://meta/llama-3.1-8b-instruct", + "display_name": "Llama 3.1 8B (NIM)", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "32Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "RuntimeNotReady", + "message": "ServingRuntime 'nim-serving-runtime' " + "is not in ready state", + }, + { + "type": "PredictorReady", + "status": "Unknown", + "reason": "PodNotCreated", + "message": "Predictor pod has not been created. " + "Waiting for ServingRuntime to become ready.", + }, + { + "type": "IngressReady", + "status": "Unknown", + "reason": "PredictorNotReady", + "message": "Waiting for predictor to become ready", + }, + ], + "age": "1d", + }, + }, +} + +DEPLOYED_MODELS = {} + +WORKBENCHES = { + "ml-production": [ + { + "name": "data-exploration-nb", + "display_name": "Data Exploration", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Running", + "cpu_request": "1", + "memory_request": "8Gi", + "gpu_count": 0, + "pvc_name": "data-exploration-nb-pvc", + "pvc_size": "20Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-10T09:00:00Z", + }, + { + "name": "model-training-nb", + "display_name": "Model Training", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Stopped", + "cpu_request": "4", + "memory_request": "16Gi", + "gpu_count": 1, + "pvc_name": "model-training-nb-pvc", + "pvc_size": "50Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-15T14:00:00Z", + }, + ], +} + +PIPELINE_SERVERS = { + "ml-production": { + "configured": True, + "data_connection": "prod-model-store", + "status": "Ready", + "database": "MariaDB", + }, +} + +NOTEBOOK_IMAGES = [ + {"name": "jupyter-pytorch-ubi9-python-3.9-2024.1", "display_name": "PyTorch 2024.1", "packages": ["torch", "transformers"]}, + {"name": "jupyter-tensorflow-ubi9-python-3.9-2024.1", "display_name": "TensorFlow 2024.1", "packages": ["tensorflow"]}, + {"name": "jupyter-datascience-ubi9-python-3.9-2024.1", "display_name": "Standard Data Science", "packages": ["pandas", "scikit-learn"]}, + {"name": "jupyter-minimal-ubi9-python-3.9-2024.1", "display_name": "Minimal Python", "packages": []}, +] + + +# ── Project tools ──────────────────────────────────────────────────────── + + +@mcp.tool() +def list_data_science_projects() -> str: + """List all RHOAI Data Science Projects on the cluster.""" + projects = [] + for name, proj in PROJECTS.items(): + projects.append({ + "name": name, + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + }) + return json.dumps(projects, indent=2) + + +@mcp.tool() +def create_data_science_project( + name: str, + display_name: str, + description: str = "", +) -> str: + """Create a new RHOAI Data Science Project (namespace with dashboard labels).""" + if name in PROJECTS: + raise ValueError( + f"Project '{name}' already exists. Choose a different name " + "or configure the existing project." + ) + if not name.replace("-", "").replace("_", "").isalnum() or len(name) > 63: + raise ValueError( + f"Invalid project name '{name}'. Must be DNS-compatible: " + "lowercase alphanumeric and hyphens, max 63 chars." + ) + + PROJECTS[name] = { + "name": name, + "display_name": display_name, + "description": description, + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": None, + "pipeline_server": False, + } + DATA_CONNECTIONS[name] = [] + SERVING_RUNTIMES[name] = [] + INFERENCE_SERVICES[name] = {} + + return json.dumps({ + "status": "created", + "name": name, + "display_name": display_name, + "namespace": name, + "labels": {"opendatahub.io/dashboard": "true"}, + }) + + +@mcp.tool() +def get_project_details(name: str) -> str: + """Get detailed information about an RHOAI Data Science Project.""" + if name not in PROJECTS: + raise ValueError(f"Project '{name}' not found") + proj = PROJECTS[name] + dc_count = len(DATA_CONNECTIONS.get(name, [])) + isvc_count = len(INFERENCE_SERVICES.get(name, {})) + return json.dumps({ + "name": proj["name"], + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "labels": proj["labels"], + "data_connections": dc_count, + "inference_services": isvc_count, + "model_serving_mode": proj.get("model_serving_mode"), + "pipeline_server": proj.get("pipeline_server", False), + }) + + +@mcp.tool() +def get_project_status(namespace: str) -> str: + """Get comprehensive status of an RHOAI Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Project '{namespace}' not found") + proj = PROJECTS[namespace] + dcs = DATA_CONNECTIONS.get(namespace, []) + isvcs = INFERENCE_SERVICES.get(namespace, {}) + return json.dumps({ + "namespace": namespace, + "display_name": proj["display_name"], + "status": "Active", + "components": { + "data_connections": len(dcs), + "inference_services": len(isvcs), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + "pipeline_server": "configured" if proj.get("pipeline_server") else "not configured", + }, + }) + + +# ── Data connection tools ──────────────────────────────────────────────── + + +@mcp.tool() +def create_s3_data_connection( + namespace: str, + name: str, + bucket: str, + endpoint: str, + access_key: str, + secret_key: str, + region: str = "", +) -> str: + """Create an S3-compatible data connection in an RHOAI project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + existing = DATA_CONNECTIONS.get(namespace, []) + if any(dc["name"] == name for dc in existing): + raise ValueError( + f"Data connection '{name}' already exists in namespace '{namespace}'" + ) + + dc = { + "name": name, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + "region": region, + } + DATA_CONNECTIONS.setdefault(namespace, []).append(dc) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + }) + + +@mcp.tool() +def list_data_connections(namespace: str) -> str: + """List data connections in an RHOAI project namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + return json.dumps(dcs, indent=2) + + +# ── Model serving tools ───────────────────────────────────────────────── + + +@mcp.tool() +def set_model_serving_mode(namespace: str, mode: str) -> str: + """Enable model serving on a Data Science Project (single or multi mode).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + if mode not in ("single", "multi"): + raise ValueError(f"Invalid mode '{mode}'. Must be 'single' or 'multi'.") + + PROJECTS[namespace]["model_serving_mode"] = mode + + if not SERVING_RUNTIMES.get(namespace): + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + SERVING_RUNTIMES[namespace] = [ + {**t, "requires_instantiation": False, "source": "existing"} + for t in templates + ] + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "mode": mode, + }) + + +@mcp.tool() +def list_serving_runtimes( + namespace: str, + include_templates: bool = False, +) -> str: + """List available ServingRuntimes in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + runtimes = list(SERVING_RUNTIMES.get(namespace, [])) + if include_templates: + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + existing_names = {r["name"] for r in runtimes} + for t in templates: + if t["name"] not in existing_names: + runtimes.append(t) + + return json.dumps(runtimes, indent=2) + + +# ── Inference service tools ────────────────────────────────────────────── + + +@mcp.tool() +def deploy_model( + name: str, + namespace: str, + runtime: str, + model_format: str, + storage_uri: str, + display_name: str = "", + min_replicas: int = 1, + max_replicas: int = 1, + cpu_request: str = "1", + cpu_limit: str = "2", + memory_request: str = "4Gi", + memory_limit: str = "8Gi", + gpu_count: int = 0, +) -> str: + """Deploy an AI/ML model as a KServe InferenceService.""" + if namespace not in PROJECTS: + raise ValueError( + f"Namespace '{namespace}' is not a Data Science Project. " + "Create one via create_data_science_project first." + ) + + ns_runtimes = SERVING_RUNTIMES.get(namespace, []) + runtime_names = [r["name"] for r in ns_runtimes] + if runtime not in runtime_names: + available = ", ".join(runtime_names) or "none" + raise ValueError( + f"ServingRuntime '{runtime}' not found in namespace '{namespace}'. " + f"Available runtimes: {available}" + ) + + endpoint = f"https://{name}-{namespace}.apps.ocp-cluster.example.com" + isvc = { + "name": name, + "namespace": namespace, + "runtime": runtime, + "model_format": model_format, + "storage_uri": storage_uri, + "display_name": display_name or name, + "gpu_count": gpu_count, + "cpu_request": cpu_request, + "memory_request": memory_request, + "min_replicas": min_replicas, + "max_replicas": max_replicas, + "ready": True, + "url": endpoint, + "conditions": [ + {"type": "Ready", "status": "True", "reason": "Ready", "message": ""}, + {"type": "PredictorReady", "status": "True", "reason": "PodReady", "message": ""}, + {"type": "IngressReady", "status": "True", "reason": "IngressReady", "message": ""}, + ], + "age": "0s", + } + + INFERENCE_SERVICES.setdefault(namespace, {})[name] = isvc + DEPLOYED_MODELS[f"{namespace}/{name}"] = isvc + + return json.dumps({ + "status": "deployed", + "name": name, + "namespace": namespace, + "runtime": runtime, + "endpoint": endpoint, + "ready": True, + }) + + +@mcp.tool() +def list_inference_services( + namespace: str, + verbosity: str = "standard", +) -> str: + """List deployed InferenceServices in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + isvcs = INFERENCE_SERVICES.get(namespace, {}) + results = [] + for isvc_name, isvc in isvcs.items(): + entry = { + "name": isvc["name"], + "runtime": isvc["runtime"], + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "age": isvc.get("age", ""), + } + if verbosity == "full": + entry["conditions"] = isvc.get("conditions", []) + entry["storage_uri"] = isvc.get("storage_uri", "") + entry["gpu_count"] = isvc.get("gpu_count", 0) + results.append(entry) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def get_inference_service( + name: str, + namespace: str, + verbosity: str = "standard", +) -> str: + """Get detailed status of a specific InferenceService.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + + isvc = isvcs[name] + result = { + "name": isvc["name"], + "namespace": isvc["namespace"], + "runtime": isvc["runtime"], + "model_format": isvc.get("model_format", ""), + "storage_uri": isvc.get("storage_uri", ""), + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "conditions": isvc.get("conditions", []), + "gpu_count": isvc.get("gpu_count", 0), + "replicas": {"min": isvc.get("min_replicas", 1), "max": isvc.get("max_replicas", 1)}, + "resources": { + "cpu_request": isvc.get("cpu_request", "1"), + "memory_request": isvc.get("memory_request", "4Gi"), + "memory_limit": isvc.get("memory_limit", "8Gi"), + }, + "age": isvc.get("age", ""), + } + return json.dumps(result, indent=2) + + +@mcp.tool() +def get_model_endpoint(name: str, namespace: str) -> str: + """Get the inference endpoint URL for a deployed model.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + isvc = isvcs[name] + if not isvc["ready"]: + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": "", + "error": "InferenceService is not ready. Check conditions for details.", + }) + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": isvc["url"], + }) + + +# ── Workbench tools ────────────────────────────────────────────────────── + + +@mcp.tool() +def list_workbenches(namespace: str) -> str: + """List workbenches (Jupyter notebooks) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + return json.dumps(wbs, indent=2) + + +@mcp.tool() +def create_workbench( + namespace: str, + name: str, + display_name: str = "", + image: str = "jupyter-datascience-ubi9-python-3.9-2024.1", + cpu_request: str = "1", + memory_request: str = "4Gi", + gpu_count: int = 0, + pvc_size: str = "20Gi", +) -> str: + """Create a new workbench (Jupyter notebook) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + valid_images = [img["name"] for img in NOTEBOOK_IMAGES] + if image not in valid_images: + raise ValueError( + f"Image '{image}' not found. Available: {', '.join(valid_images)}" + ) + + wb = { + "name": name, + "display_name": display_name or name, + "image": image, + "status": "Running", + "cpu_request": cpu_request, + "memory_request": memory_request, + "gpu_count": gpu_count, + "pvc_name": f"{name}-pvc", + "pvc_size": pvc_size, + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-03-02T12:00:00Z", + } + WORKBENCHES.setdefault(namespace, []).append(wb) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "image": image, + "pvc": f"{name}-pvc", + }) + + +@mcp.tool() +def stop_workbench(namespace: str, name: str) -> str: + """Stop a running workbench (preserves data).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Stopped" + return json.dumps({"status": "stopped", "name": name, "namespace": namespace}) + + +@mcp.tool() +def start_workbench(namespace: str, name: str) -> str: + """Start a stopped workbench.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Running" + return json.dumps({"status": "running", "name": name, "namespace": namespace}) + + +@mcp.tool() +def delete_workbench(namespace: str, name: str) -> str: + """Delete a workbench. WARNING: PVC data may be lost if not backed up.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wbs.remove(wb) + return json.dumps({ + "status": "deleted", + "name": name, + "namespace": namespace, + "warning": "Associated PVC data has been deleted", + }) + + +@mcp.tool() +def list_notebook_images() -> str: + """List available notebook images for workbench creation.""" + return json.dumps(NOTEBOOK_IMAGES, indent=2) + + +# ── Pipeline server tools ─────────────────────────────────────────────── + + +@mcp.tool() +def configure_pipeline_server( + namespace: str, + data_connection: str, + database: str = "MariaDB", +) -> str: + """Configure a pipeline server for a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + if not any(dc["name"] == data_connection for dc in dcs): + available = [dc["name"] for dc in dcs] + raise ValueError( + f"Data connection '{data_connection}' not found. Available: {available}" + ) + + PIPELINE_SERVERS[namespace] = { + "configured": True, + "data_connection": data_connection, + "status": "Ready", + "database": database, + } + PROJECTS[namespace]["pipeline_server"] = True + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "data_connection": data_connection, + "database": database, + }) + + +@mcp.tool() +def get_pipeline_server_status(namespace: str) -> str: + """Get the status of the pipeline server in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + ps = PIPELINE_SERVERS.get(namespace) + if not ps: + return json.dumps({"namespace": namespace, "configured": False}) + return json.dumps({ + "namespace": namespace, + "configured": ps["configured"], + "data_connection": ps["data_connection"], + "status": ps["status"], + "database": ps["database"], + }) + + +# ── Serving runtime creation ──────────────────────────────────────────── + + +@mcp.tool() +def create_serving_runtime( + namespace: str, + name: str, + display_name: str = "", + model_formats: list = None, + container_image: str = "", + container_port: int = 8080, + multi_model: bool = False, + api_protocol: str = "REST", +) -> str: + """Create a custom ServingRuntime in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + if not model_formats: + raise ValueError("model_formats must specify at least one model format") + + runtime = { + "name": name, + "display_name": display_name or name, + "model_formats": model_formats, + "requires_instantiation": False, + "source": "custom", + "api_protocol": api_protocol, + "container_image": container_image, + "container_port": container_port, + "multi_model": multi_model, + } + SERVING_RUNTIMES.setdefault(namespace, []).append(runtime) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "model_formats": model_formats, + }) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/instruction.md b/evaluation/without_skills/rh-ai-engineer__ai-observability/instruction.md new file mode 100644 index 00000000..f76c1829 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ai-observability/instruction.md @@ -0,0 +1,13 @@ +# AI Observability Task + +You are an AI engineer on Red Hat OpenShift AI. Your team has deployed several inference services, but has no visibility into how they are performing or whether resources are sized correctly. + +## Requirements +- Assess the current state of deployed inference services and their resource consumption +- Define a metrics strategy covering: inference latency, throughput, error rates, and GPU memory utilization +- Identify any models that appear over-provisioned or under-provisioned based on current usage +- Recommend specific resource adjustments (CPU, memory, GPU, replicas) with justification + +Document your observability strategy and resource recommendations in `/root/report.md`. + +Use MCP tools to interact with the platform. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/solution/solve.sh b/evaluation/without_skills/rh-ai-engineer__ai-observability/solution/solve.sh new file mode 100644 index 00000000..d319c204 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ai-observability/solution/solve.sh @@ -0,0 +1,23 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# AI Observability Report + +## Model: fraud-detection + +### GPU Utilization (last 15m) +- GPU memory: 65% utilized +- GPU compute: 45% utilized +- Recommendation: GPU is underutilized, consider right-sizing + +### Model Latency +- P50: 120ms +- P99: 450ms + +### Right-Sizing +- Current: 1x A100 80GB +- Recommended: 1x A100 40GB (sufficient for workload) + +### Advanced Observability +- execute_promql for custom metrics (e.g., vllm:request_success:ratio) +- query_tempo_tool for trace latency on slow requests +REPORT_EOF diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/task.toml b/evaluation/without_skills/rh-ai-engineer__ai-observability/task.toml new file mode 100644 index 00000000..130dece1 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ai-observability/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-ai-engineer__ai-observability" +name = "rh-ai-engineer AI Observability Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-ai-engineer", "ai-observability", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/tests/llm_judge.py b/evaluation/without_skills/rh-ai-engineer__ai-observability/tests/llm_judge.py new file mode 100644 index 00000000..28b8d6a0 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ai-observability/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "observability_stack_awareness", + "file": "/root/report.md", + "question": "Does the report recommend specific observability tools from the OpenShift AI ecosystem — specifically Tempo for distributed tracing AND Korrel8r for cross-domain correlation — as part of the observability strategy?", + "reference": "A skilled report should mention BOTH Tempo (for distributed tracing of inference requests) and Korrel8r (for correlating signals across logs, metrics, traces, and alerts). These are specific tools the skill teaches. A report that only recommends generic Prometheus/Grafana monitoring without Tempo or Korrel8r is insufficient." + }, + { + "id": "gpu_metric_specificity", + "file": "/root/report.md", + "question": "Does the report reference DCGM (Data Center GPU Manager) metric names (like DCGM_FI_DEV_FB_USED or DCGM_FI_DEV_GPU_UTIL) for GPU monitoring, rather than generic nvidia_gpu_memory metric names?", + "reference": "A skilled report uses DCGM-specific metric names (DCGM_FI_DEV_*) which are the actual metrics exposed by the GPU operator on OpenShift. Using generic names like nvidia_gpu_memory_used_bytes suggests the agent doesn't know the specific metric naming convention." + }, + { + "id": "vllm_tuning_specificity", + "file": "/root/report.md", + "question": "Does the report recommend specific vLLM configuration parameters (like --max-model-len, --gpu-memory-utilization, or tensor parallelism) for resolving GPU memory issues, rather than only recommending generic resource increases?", + "reference": "A skilled report should mention vLLM-specific tuning args like --max-model-len to limit KV cache size, --gpu-memory-utilization to control memory allocation, or tensor parallelism for multi-GPU distribution. Only recommending 'increase memory to 32Gi' without vLLM-specific configuration is insufficient." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/tests/test.sh b/evaluation/without_skills/rh-ai-engineer__ai-observability/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ai-observability/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/tests/test_outputs.py b/evaluation/without_skills/rh-ai-engineer__ai-observability/tests/test_outputs.py new file mode 100644 index 00000000..eb3755b2 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ai-observability/tests/test_outputs.py @@ -0,0 +1,91 @@ +""" +Tests for rh-ai-engineer__ai-observability per-skill evaluation. +Baseline tests: any competent agent should pass. +Skill-dependent tests: based on empirical gaps between skilled and unskilled agent outputs. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ["monitor", "metric", "observ", "inference"]), ( + "report should mention monitoring or observability" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 200, "report should have substantial content" + + +class TestSkillDependent: + def test_tempo_distributed_tracing(self): + """Skill teaches Tempo for distributed tracing of inference requests. + Without skill, agents don't mention Tempo at all.""" + c = read_report().lower() + assert any(t in c for t in ["tempo", "distributed trac"]), ( + "should recommend Tempo for distributed tracing" + ) + + def test_korrel8r_correlation(self): + """Skill teaches Korrel8r for cross-domain signal correlation. + Without skill, agents don't know about Korrel8r.""" + c = read_report().lower() + assert any(t in c for t in ["korrel8r", "cross-domain correlation"]), ( + "should mention Korrel8r for cross-domain correlation" + ) + + def test_dcgm_gpu_metric_names(self): + """Skill teaches DCGM-specific GPU metric names (DCGM_FI_DEV_*). + Without skill, agents use generic nvidia_gpu_memory_* names.""" + c = read_report() + assert any(t in c for t in ["DCGM_FI_DEV", "dcgm_fi_dev", "DCGM"]), ( + "should reference DCGM GPU metric names (not generic nvidia_gpu_*)" + ) + + def test_opentelemetry_instrumentation(self): + """Skill teaches OpenTelemetry for trace instrumentation on inference endpoints. + Without skill, agents don't mention OpenTelemetry.""" + c = read_report().lower() + assert any(t in c for t in ["opentelemetry", "otel"]), ( + "should recommend OpenTelemetry instrumentation" + ) + + def test_vllm_tuning_args(self): + """Skill teaches vLLM CLI args for memory management. + Without skill, agents recommend generic resource increases but not vLLM-specific tuning.""" + c = read_report().lower() + assert any(t in c for t in [ + "max-model-len", "max_model_len", "gpu-memory-utilization", + "gpu_memory_utilization", "tensor parallel", "tensor_parallel", + ]), "should mention vLLM-specific configuration args for resource tuning" + + def test_latency_percentiles(self): + """Both agents should report latency percentiles (easy test).""" + c = read_report().lower() + assert any(t in c for t in ["p50", "p95", "p99"]), ( + "should report latency with percentiles" + ) + + def test_tensor_parallel_size_tuning(self): + """Docs teach reducing --tensor-parallel-size as GPU scheduling triage step, + and OOM mitigation via --max-model-len and quantized models (AWQ/GPTQ/FP8). + Without docs, agents don't know these vLLM tuning parameters.""" + c = read_report().lower() + assert any(t in c for t in [ + "tensor-parallel-size", "tensor_parallel_size", "tensor parallel", + "awq", "gptq", "fp8", "quantiz", + ]), "should address tensor-parallel-size and quantization for GPU tuning" diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/Dockerfile b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/Dockerfile new file mode 100644 index 00000000..aac4c84e --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/Dockerfile @@ -0,0 +1,67 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "rhoai": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-rhoai-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/common-issues.md b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/examples/model-deploy.md b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/examples/model-deploy.md new file mode 100644 index 00000000..dcd79e44 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/examples/model-deploy.md @@ -0,0 +1,160 @@ +# /model-deploy Examples + +Example interactions demonstrating the model-deploy skill workflow. + +## Example 1: Deploy Llama 3.1 8B with vLLM + +``` +User: "Deploy Llama 3.1 8B on my cluster" + +Agent: [Invokes /model-deploy skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand runtime capabilities. + +Step 1 of 10: Gathering deployment information. + +| Setting | Value | Source | +|---------|-------|--------| +| Model | Llama 3.1 8B Instruct | user input | +| Runtime | TBD | auto-detect | +| Namespace | ? | needs input | +| Model Source | ? | needs input | +| Deployment Mode | Serverless | default | + +What namespace should I deploy to, and where are the model weights stored? +(e.g., S3 bucket, OCI registry, PVC) + +User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" + +Agent: Step 2 of 10: Determining runtime. + +I consulted supported-runtimes.md to understand runtime capabilities. + +Based on Llama 3.1 8B (open-source HuggingFace model): + +Recommended runtime: vLLM +Reason: Default runtime for open-source LLMs. Broad compatibility, + OpenAI-compatible API, no additional setup required. + +Proceed with vLLM? (yes / no / suggest alternative) + +User: "yes" + +Agent: Step 3 of 10: Looking up hardware profile. + +I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. + +| Requirement | Value | +|-------------|-------| +| GPUs | 1x A10G/L4/A100/H100 | +| VRAM | 16GB | +| Key Args | --max-model-len=4096 --tool-call-parser hermes | + +Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). +Ensure your cluster has at least 1 GPU with 16GB VRAM. + +Step 5 of 10: Skipped (NIM platform check not needed for vLLM). + +Step 6-7 of 10: Generating InferenceService YAML. + +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: llama-3-1-8b + namespace: ai-models + annotations: + serving.kserve.io/deploymentMode: Serverless +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: vllm-runtime + storageUri: s3://my-bucket/llama-3.1-8b + resources: + limits: + nvidia.com/gpu: "1" + requests: + cpu: "4" + memory: "16Gi" + args: + - --max-model-len=4096 + - --tool-call-parser=hermes + +| Setting | Value | +|---------|-------| +| Model | llama-3-1-8b | +| Runtime | vllm-runtime | +| GPUs | 1 | +| Namespace | ai-models | +| Mode | Serverless | + +Proceed with creating this InferenceService? (yes/no/modify) + +User: "yes" + +Agent: Step 8 of 10: Creating InferenceService... + [Creates via resources_create_or_update] + + InferenceService created. Monitoring rollout... + +Step 9 of 10: Monitoring rollout. + +| Stage | Status | +|-------|--------| +| Pod Scheduled | Complete | +| Image Pulled | Complete | +| Container Started | Complete | +| Model Loaded | Complete | +| Ready | Complete | + +Step 10 of 10: Deployment complete! + +Model Deployment Successful + +Model: llama-3-1-8b +Runtime: vLLM +Namespace: ai-models +GPUs: 1 + +Inference Endpoint: + URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com + API: OpenAI-compatible REST + +Quick Test: + curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' + +Next Steps: + Monitor performance: "/ai-observability llama-3-1-8b" +``` + +## Example 2: Deploy a Model with NIM (NIM Not Set Up) + +``` +User: "Deploy Llama 3.1 70B with NIM" + +Agent: [Invokes /model-deploy skill] + ... + Step 2: User requested NIM runtime. + + Step 5: Checking NIM platform... + +NIM platform is not set up in namespace ai-models. + +The NIM runtime requires a one-time platform setup. Would you like to: +1. Run /nim-setup now to configure the NIM platform +2. Switch to vLLM runtime instead (no setup required) +3. Abort deployment + +User: "1" + +Agent: [Invokes /nim-setup skill — walks through NGC credential setup] + ... + NIM Platform Setup Complete. + + Resuming model deployment... + [Continues from Step 6 with NIM runtime] +``` diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/examples/nim-setup.md b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/examples/nim-setup.md new file mode 100644 index 00000000..ccfc1a5d --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/examples/nim-setup.md @@ -0,0 +1,115 @@ +# /nim-setup Examples + +Example interactions demonstrating the nim-setup skill workflow. + +## Example 1: First-Time NIM Setup + +``` +User: "Set up NIM on my cluster" + +Agent: [Invokes /nim-setup skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand NIM platform requirements. + +Let me verify the prerequisites for NIM setup. + +Step 1 of 7: Checking GPU Operator and NFD Operator... + +GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) +NFD Operator: nfd.v4.19.0 (Succeeded) + +Both operators are installed and healthy. + +Step 2 of 7: I need your NGC credentials. + +To set up NIM, I need your NVIDIA NGC API key. +You can generate one at: https://ngc.nvidia.com/setup/api-key + +Please provide: +1. Your NGC API key +2. The target namespace for NIM resources + +User: "Here's my key: [user provides NGC API key], namespace: ai-models" + +Agent: Step 3 of 7: Creating NGC image pull secret... + +I'll create a docker-registry secret for pulling NIM images from nvcr.io: + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-image-pull-secret + namespace: ai-models + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] + +Should I create this image pull secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + Image pull secret created successfully. + +Step 4 of 7: Creating NGC API key secret... + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-api-key + namespace: ai-models + type: Opaque + stringData: + NGC_API_KEY: "[REDACTED]" + +Should I create this API key secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + API key secret created successfully. + +Step 5 of 7: Creating NIM Account CR... + + apiVersion: nim.opendatahub.io/v1 + kind: Account + metadata: + name: nim-account + namespace: ai-models + spec: + apiKeySecret: + name: ngc-api-key + imagePullSecret: + name: ngc-image-pull-secret + +Should I create this NIM Account CR in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates Account CR via resources_create_or_update] + Account CR created successfully. + +Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) + +User: "no" + +Agent: Step 7 of 7: Validating NIM platform readiness... + +NIM Platform Setup Complete + +Account CR: `nim-account` (namespace: ai-models) + Status: Ready + +NGC Credentials: + Image Pull Secret: ngc-image-pull-secret [created] + API Key Secret: ngc-api-key [created] + +NIM ServingRuntimes Available: + - nim-llama-3-1-8b-instruct + - nim-llama-3-1-70b-instruct + - nim-mistral-7b-instruct + +Next Steps: + To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" +``` diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/live-doc-lookup.md b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/references/known-model-profiles.md b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/references/known-model-profiles.md new file mode 100644 index 00000000..40148981 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/references/known-model-profiles.md @@ -0,0 +1,83 @@ +--- +title: Known Model Hardware Profiles +category: references +tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] +semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] +use_cases: [model-deploy, debug-inference, ai-observability] +last_updated: 2026-03-10 +--- + +# Known Model Hardware Profiles + +Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). + +**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. + +## Model Source Conventions + +Each model lists a recommended `storageUri` with its authentication requirements: +- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** +- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). +- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. + +When the user does not specify a model source, use the `hf://` URI listed in the profile below. + +## Llama 3.x (Meta) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | +| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | +| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | +| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | + +- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) +- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly + +## Granite 3.x (IBM/Red Hat) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | +| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | + +- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) +- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser granite --chat-template` +- Red Hat-supported model family on RHOAI + +## Mixtral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | +| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) +- Mixture-of-Experts architecture: only ~13B/45B parameters active per token + +## Mistral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | +| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) + +## When a Model Is Not Listed + +If the requested model is not in this file, the agent MUST use the live doc lookup protocol: + +1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure +2. Fetch hardware specs from the appropriate source +3. Report findings to the user before proceeding with deployment + +Common cases requiring live lookup: +- Newly released models (after this file's last update) +- Domain-specific fine-tuned models +- Models with custom quantization +- NIM-specific optimized profiles diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/references/supported-runtimes.md b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/references/supported-runtimes.md new file mode 100644 index 00000000..9d018651 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/references/supported-runtimes.md @@ -0,0 +1,104 @@ +--- +title: Supported Serving Runtimes +category: references +tags: [runtimes, vllm, nim, caikit, tgis, serving] +semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] +use_cases: [model-deploy, serving-runtime-config, nim-setup] +last_updated: 2026-02-26 +--- + +# Supported Serving Runtimes + +This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. + +## Runtime Comparison + +| Runtime | API Type | Model Format | GPU Support | License | Setup Required | +|---------|----------|-------------|-------------|---------|----------------| +| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | +| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | +| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | +| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | + +## vLLM + +**Default runtime for most open-source models.** + +- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) +- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized +- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) +- **Model source**: S3-compatible storage, OCI registry, PVC, URI +- **Key features**: + - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) + - PagedAttention for efficient memory management + - Tensor parallelism for multi-GPU inference + - Tool/function calling support (`--tool-call-parser`) + - Continuous batching for high throughput +- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name + +## NVIDIA NIM + +**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** + +- **API**: OpenAI-compatible REST +- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog +- **GPU support**: NVIDIA only (requires specific GPU models per model profile) +- **Model source**: NGC model catalog (pulled automatically via NGC credentials) +- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) +- **Key features**: + - TensorRT-LLM optimization for lower latency + - Pre-compiled model engines (no compilation on first load) + - Optimized and generic profiles per GPU type + - Automatic model download from NGC +- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. +- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM +- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. +- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state + +## Caikit+TGIS + +**Red Hat's Caikit format with Text Generation Inference Server.** + +- **API**: gRPC (not REST) +- **Model formats**: Caikit format (requires conversion from HuggingFace) +- **GPU support**: NVIDIA +- **Model source**: S3-compatible storage +- **Key features**: + - Red Hat-supported runtime + - gRPC API for streaming inference + - Integrated with RHOAI model serving platform +- **When to choose**: When the model is already in Caikit format or when gRPC API is required. +- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name + +## Custom Runtimes + +**User-provided ServingRuntime CRs for unsupported frameworks.** + +- **API**: Defined by the custom runtime +- **Model formats**: Defined by the custom runtime +- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. +- **How to create**: Use `/serving-runtime-config` skill +- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility + +## Runtime Selection Decision Tree + +``` +Is the user's preferred runtime explicitly stated? +├── Yes → Use that runtime +└── No → Continue + +Is the model available in the NGC NIM catalog? +├── Yes → Suggest NIM (with vLLM as fallback) +│ Note: Requires /nim-setup and NVIDIA GPUs +└── No → Continue + +Is the model in Caikit format? +├── Yes → Caikit+TGIS +└── No → Continue + +Is the model a standard open-source LLM (HuggingFace-compatible)? +├── Yes → vLLM (default) +└── No → Custom runtime via /serving-runtime-config +``` diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/skill-conventions.md b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..e7a4d11c --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +"""Mock OpenShift MCP server for SkillsBench rh-ai-engineer task. + +Simulates Kubernetes resource CRUD, pod management, logs, and events. + +Key scenario elements: +- LimitRange in namespaces: min CPU=100m, min memory=128Mi + (conflicts with KServe sidecar containers hardcoded at 10m CPU/15Mi memory) +- GPU node with custom taint ai-workload=true:NoSchedule +- NIM Account CR in ml-production: not ready (NGC credentials invalid) +- text-gen-legacy pods: OOMKilled (max-model-len=32768 on A10G) +- nim-llama-prod: no pods created (Account CR not ready) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + +# ── Cluster state ──────────────────────────────────────────────────────── + +GPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "gpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + "nvidia.com/gpu.present": "true", + "nvidia.com/gpu.product": "NVIDIA-A10G", + }, + }, + "spec": { + "taints": [ + { + "key": "ai-workload", + "value": "true", + "effect": "NoSchedule", + }, + ], + }, + "status": { + "allocatable": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "capacity": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "conditions": [ + {"type": "Ready", "status": "True"}, + ], + }, +} + +CPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "cpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + }, + }, + "spec": {"taints": []}, + "status": { + "allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "capacity": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +MASTER_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "master-1", + "labels": { + "node-role.kubernetes.io/master": "", + "node-role.kubernetes.io/control-plane": "", + }, + }, + "spec": { + "taints": [ + {"key": "node-role.kubernetes.io/master", "effect": "NoSchedule"}, + ], + }, + "status": { + "allocatable": {"cpu": "8", "memory": "32Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +ALL_NODES = [GPU_NODE, CPU_NODE, MASTER_NODE] + +# LimitRange applied by cluster policy to all DS project namespaces +NAMESPACE_LIMITRANGE = { + "apiVersion": "v1", + "kind": "LimitRange", + "metadata": { + "name": "default-limits", + }, + "spec": { + "limits": [ + { + "type": "Container", + "default": { + "cpu": "2", + "memory": "4Gi", + }, + "defaultRequest": { + "cpu": "500m", + "memory": "256Mi", + }, + "min": { + "cpu": "100m", + "memory": "128Mi", + }, + "max": { + "cpu": "32", + "memory": "128Gi", + }, + }, + ], + }, +} + +NIM_ACCOUNT_CR = { + "apiVersion": "nim.opendatahub.io/v1", + "kind": "Account", + "metadata": { + "name": "nim-account", + "namespace": "ml-production", + }, + "spec": { + "apiKeySecret": { + "name": "ngc-api-key", + }, + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "NGCCredentialsInvalid", + "message": "NGC API key validation failed: 401 Unauthorized. " + "The API key in secret 'ngc-api-key' is expired or invalid. " + "Re-create the secret with a valid NGC API key from " + "https://ngc.nvidia.com/setup/api-key and restart the " + "Account reconciliation.", + "lastTransitionTime": "2026-03-14T12:00:00Z", + }, + ], + "nimPullSecretStatus": "Failed", + "nimConfigStatus": "Pending", + }, +} + +SERVING_RUNTIME_VLLM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "vllm-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "vLLM", "version": "1", "autoSelect": True}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "quay.io/modh/vllm:rhoai-2.16", + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + }, + ], + }, +} + +SERVING_RUNTIME_NIM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "nim-serving-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "NIM", "version": "1"}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "nvcr.io/nim/meta/llama-3.1-8b-instruct:latest", + "ports": [{"containerPort": 8000, "protocol": "TCP"}], + "env": [ + {"name": "NGC_API_KEY", "valueFrom": { + "secretKeyRef": {"name": "ngc-api-key", "key": "api_key"}, + }}, + ], + }, + ], + }, +} + +PODS_BY_NAMESPACE = { + "ml-production": [ + { + "name": "text-gen-legacy-predictor-00001-abc12", + "namespace": "ml-production", + "status": "CrashLoopBackOff", + "restarts": 5, + "node": "gpu-worker-1", + "containers": [ + { + "name": "kserve-container", + "state": "waiting", + "reason": "CrashLoopBackOff", + "last_termination_reason": "OOMKilled", + "last_termination_exit_code": 137, + }, + ], + "labels": { + "serving.kserve.io/inferenceservice": "text-gen-legacy", + }, + "gpu": "1", + }, + # nim-llama-prod: NO pods created (Account CR not ready) + ], +} + +POD_LOGS = { + "text-gen-legacy-predictor-00001-abc12": ( + "INFO 2026-03-01 10:00:00 vllm_engine.py:125] vLLM engine starting...\n" + "INFO 2026-03-01 10:00:01 config.py:89] Model: mistralai/Mistral-7B-Instruct-v0.3\n" + "INFO 2026-03-01 10:00:01 config.py:92] max_model_len = 32768\n" + "INFO 2026-03-01 10:00:02 gpu_executor.py:45] GPU 0: NVIDIA A10G (24576 MiB)\n" + "INFO 2026-03-01 10:00:03 model_runner.py:88] Loading model weights...\n" + "INFO 2026-03-01 10:00:15 model_runner.py:112] Model weights loaded: 13.5 GiB\n" + "INFO 2026-03-01 10:00:15 worker.py:201] Allocating KV cache...\n" + "ERROR 2026-03-01 10:00:16 worker.py:215] torch.cuda.OutOfMemoryError: " + "CUDA out of memory. Tried to allocate 28.5 GiB for KV cache but only " + "10.1 GiB available after loading model weights (13.5 GiB).\n" + "ERROR 2026-03-01 10:00:16 vllm_engine.py:178] Engine failed to start\n" + "Traceback (most recent call last):\n" + " File \"/opt/vllm/vllm/engine/engine.py\", line 175, in start\n" + " self._init_kv_cache()\n" + " File \"/opt/vllm/vllm/worker/worker.py\", line 215, in _init_kv_cache\n" + " raise torch.cuda.OutOfMemoryError(msg)\n" + "torch.cuda.OutOfMemoryError: CUDA out of memory\n" + ), +} + +EVENTS_BY_NAMESPACE = { + "ml-production": [ + { + "type": "Warning", + "reason": "BackOff", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Back-off restarting failed container kserve-container in pod " + "text-gen-legacy-predictor-00001-abc12", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Warning", + "reason": "OOMKilled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Container kserve-container was OOMKilled (exit code 137). " + "GPU memory exhausted during KV cache allocation.", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Normal", + "reason": "Scheduled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Successfully assigned ml-production/" + "text-gen-legacy-predictor-00001-abc12 to gpu-worker-1", + "count": 1, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-02-28T08:00:00Z", + }, + { + "type": "Warning", + "reason": "NIMAccountNotReady", + "object": "InferenceService/nim-llama-prod", + "message": "NIM Account 'nim-account' in namespace 'ml-production' " + "is not ready", + "count": 12, + "first_timestamp": "2026-03-14T12:00:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + { + "type": "Warning", + "reason": "ImagePullBackOff", + "object": "InferenceService/nim-llama-prod", + "message": "Failed to pull image 'nvcr.io/nim/meta/llama-3.1-8b-instruct:" + "latest': unauthorized: authentication required", + "count": 8, + "first_timestamp": "2026-03-14T12:05:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + ], +} + + +# ── Resource tools ─────────────────────────────────────────────────────── + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: str = "", +) -> str: + """Get a single Kubernetes resource by apiVersion, kind, and name.""" + if kind == "Node": + for node in ALL_NODES: + if node["metadata"]["name"] == name: + return json.dumps(node, indent=2) + raise ValueError(f"Node '{name}' not found") + + if kind == "ServingRuntime": + if name == "vllm-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_VLLM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + if name == "nim-serving-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_NIM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + raise ValueError(f"ServingRuntime '{name}' not found in namespace '{namespace}'") + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps(lr, indent=2) + + if kind == "Account" and "nim" in apiVersion.lower(): + if namespace == "ml-production" and name == "nim-account": + return json.dumps(NIM_ACCOUNT_CR, indent=2) + raise ValueError( + f"Account '{name}' not found in namespace '{namespace}'" + ) + + if kind == "ClusterVersion" and apiVersion == "config.openshift.io/v1": + return json.dumps({ + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterVersion", + "metadata": {"name": "version"}, + "status": {"desired": {"version": "4.16.3"}}, + }) + + raise ValueError(f"Resource {apiVersion}/{kind}/{name} not found") + + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: str = "", + labelSelector: str = "", +) -> str: + """List Kubernetes resources by apiVersion and kind.""" + if kind == "Node": + nodes = ALL_NODES + if labelSelector: + parts = labelSelector.split("=", 1) + key = parts[0] + value = parts[1] if len(parts) > 1 else "" + nodes = [ + n for n in nodes + if n["metadata"]["labels"].get(key) == value + ] + return json.dumps(nodes, indent=2) + + if kind == "Service" and apiVersion == "serving.knative.dev/v1": + return json.dumps({ + "kind": "ServiceList", + "apiVersion": "serving.knative.dev/v1", + "items": [], + "metadata": {}, + }) + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps({ + "kind": "LimitRangeList", + "items": [lr], + }) + + if kind == "InferenceService": + return json.dumps({ + "kind": "InferenceServiceList", + "items": [], + }) + + raise ValueError(f"Unsupported list: {apiVersion}/{kind}") + + +@mcp.tool() +def pods_list( + namespace: str, + labelSelector: str = "", +) -> str: + """List pods in a namespace with optional label selector.""" + pods = PODS_BY_NAMESPACE.get(namespace, []) + + if labelSelector: + key, _, value = labelSelector.partition("=") + pods = [p for p in pods if p.get("labels", {}).get(key) == value] + + results = [] + for pod in pods: + results.append({ + "name": pod["name"], + "namespace": pod["namespace"], + "status": pod["status"], + "restarts": pod.get("restarts", 0), + "node": pod.get("node", ""), + "containers": pod.get("containers", []), + "gpu": pod.get("gpu", "0"), + }) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def pods_log( + namespace: str, + name: str, + container: str = "", +) -> str: + """Get logs from a pod container.""" + logs = POD_LOGS.get(name) + if logs is None: + raise ValueError(f"Pod '{name}' not found in namespace '{namespace}'") + return logs + + +@mcp.tool() +def events_list(namespace: str) -> str: + """List events in a namespace.""" + events = EVENTS_BY_NAMESPACE.get(namespace, []) + return json.dumps(events, indent=2) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/mcp-servers/mock-rhoai-mcp.py b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/mcp-servers/mock-rhoai-mcp.py new file mode 100644 index 00000000..0ae9e4cb --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/mcp-servers/mock-rhoai-mcp.py @@ -0,0 +1,780 @@ +#!/usr/bin/env python3 +"""Mock RHOAI MCP server for SkillsBench rh-ai-engineer task. + +Simulates Red Hat OpenShift AI operations: Data Science Projects, +model serving, data connections, serving runtimes, inference services. + +Scenario: +- ml-production: existing project with two broken deployments + - text-gen-legacy: vLLM OOMKilled (max-model-len=32768 on A10G) + - nim-llama-prod: NIM failing (Account CR not ready, NGC creds invalid) +- fraud-detection: does not exist yet (agent creates it) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("rhoai") + +# ── In-memory state ────────────────────────────────────────────────────── + +PROJECTS = { + "ml-production": { + "name": "ml-production", + "display_name": "ML Production", + "description": "Production ML workloads", + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": "single", + "pipeline_server": True, + }, +} + +DATA_CONNECTIONS = { + "ml-production": [ + { + "name": "prod-model-store", + "type": "S3", + "bucket": "ml-models-prod", + "endpoint": "https://s3.us-east-1.amazonaws.com", + "region": "us-east-1", + }, + ], +} + +SERVING_RUNTIMES = { + "__platform_templates__": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "REST", + "supported_model_formats": [ + {"name": "vLLM", "version": "1", "autoSelect": True} + ], + }, + { + "name": "caikit-tgis-runtime", + "display_name": "Caikit+TGIS ServingRuntime", + "model_formats": ["caikit"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "gRPC", + }, + ], + "ml-production": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + { + "name": "nim-serving-runtime", + "display_name": "NVIDIA NIM ServingRuntime", + "model_formats": ["NIM"], + "requires_instantiation": False, + "source": "nim-account", + "api_protocol": "REST", + }, + { + "name": "ovms-1", + "display_name": "OpenVINO Model Server", + "model_formats": ["openvino_ir", "onnx"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + ], +} + +INFERENCE_SERVICES = { + "ml-production": { + "text-gen-legacy": { + "name": "text-gen-legacy", + "namespace": "ml-production", + "runtime": "vllm-runtime", + "model_format": "vLLM", + "storage_uri": "hf://mistralai/Mistral-7B-Instruct-v0.3", + "display_name": "Mistral 7B Legacy", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "16Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "PredictorFailed", + "message": "Predictor pod is not ready", + }, + { + "type": "PredictorReady", + "status": "False", + "reason": "ContainerCrashLoop", + "message": "Container kserve-container terminated: " + "OOMKilled (exit code 137). 5 restarts.", + }, + { + "type": "IngressReady", + "status": "True", + "reason": "IngressReady", + "message": "Ingress is ready", + }, + ], + "age": "3d", + }, + "nim-llama-prod": { + "name": "nim-llama-prod", + "namespace": "ml-production", + "runtime": "nim-serving-runtime", + "model_format": "NIM", + "storage_uri": "nim://meta/llama-3.1-8b-instruct", + "display_name": "Llama 3.1 8B (NIM)", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "32Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "RuntimeNotReady", + "message": "ServingRuntime 'nim-serving-runtime' " + "is not in ready state", + }, + { + "type": "PredictorReady", + "status": "Unknown", + "reason": "PodNotCreated", + "message": "Predictor pod has not been created. " + "Waiting for ServingRuntime to become ready.", + }, + { + "type": "IngressReady", + "status": "Unknown", + "reason": "PredictorNotReady", + "message": "Waiting for predictor to become ready", + }, + ], + "age": "1d", + }, + }, +} + +DEPLOYED_MODELS = {} + +WORKBENCHES = { + "ml-production": [ + { + "name": "data-exploration-nb", + "display_name": "Data Exploration", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Running", + "cpu_request": "1", + "memory_request": "8Gi", + "gpu_count": 0, + "pvc_name": "data-exploration-nb-pvc", + "pvc_size": "20Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-10T09:00:00Z", + }, + { + "name": "model-training-nb", + "display_name": "Model Training", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Stopped", + "cpu_request": "4", + "memory_request": "16Gi", + "gpu_count": 1, + "pvc_name": "model-training-nb-pvc", + "pvc_size": "50Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-15T14:00:00Z", + }, + ], +} + +PIPELINE_SERVERS = { + "ml-production": { + "configured": True, + "data_connection": "prod-model-store", + "status": "Ready", + "database": "MariaDB", + }, +} + +NOTEBOOK_IMAGES = [ + {"name": "jupyter-pytorch-ubi9-python-3.9-2024.1", "display_name": "PyTorch 2024.1", "packages": ["torch", "transformers"]}, + {"name": "jupyter-tensorflow-ubi9-python-3.9-2024.1", "display_name": "TensorFlow 2024.1", "packages": ["tensorflow"]}, + {"name": "jupyter-datascience-ubi9-python-3.9-2024.1", "display_name": "Standard Data Science", "packages": ["pandas", "scikit-learn"]}, + {"name": "jupyter-minimal-ubi9-python-3.9-2024.1", "display_name": "Minimal Python", "packages": []}, +] + + +# ── Project tools ──────────────────────────────────────────────────────── + + +@mcp.tool() +def list_data_science_projects() -> str: + """List all RHOAI Data Science Projects on the cluster.""" + projects = [] + for name, proj in PROJECTS.items(): + projects.append({ + "name": name, + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + }) + return json.dumps(projects, indent=2) + + +@mcp.tool() +def create_data_science_project( + name: str, + display_name: str, + description: str = "", +) -> str: + """Create a new RHOAI Data Science Project (namespace with dashboard labels).""" + if name in PROJECTS: + raise ValueError( + f"Project '{name}' already exists. Choose a different name " + "or configure the existing project." + ) + if not name.replace("-", "").replace("_", "").isalnum() or len(name) > 63: + raise ValueError( + f"Invalid project name '{name}'. Must be DNS-compatible: " + "lowercase alphanumeric and hyphens, max 63 chars." + ) + + PROJECTS[name] = { + "name": name, + "display_name": display_name, + "description": description, + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": None, + "pipeline_server": False, + } + DATA_CONNECTIONS[name] = [] + SERVING_RUNTIMES[name] = [] + INFERENCE_SERVICES[name] = {} + + return json.dumps({ + "status": "created", + "name": name, + "display_name": display_name, + "namespace": name, + "labels": {"opendatahub.io/dashboard": "true"}, + }) + + +@mcp.tool() +def get_project_details(name: str) -> str: + """Get detailed information about an RHOAI Data Science Project.""" + if name not in PROJECTS: + raise ValueError(f"Project '{name}' not found") + proj = PROJECTS[name] + dc_count = len(DATA_CONNECTIONS.get(name, [])) + isvc_count = len(INFERENCE_SERVICES.get(name, {})) + return json.dumps({ + "name": proj["name"], + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "labels": proj["labels"], + "data_connections": dc_count, + "inference_services": isvc_count, + "model_serving_mode": proj.get("model_serving_mode"), + "pipeline_server": proj.get("pipeline_server", False), + }) + + +@mcp.tool() +def get_project_status(namespace: str) -> str: + """Get comprehensive status of an RHOAI Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Project '{namespace}' not found") + proj = PROJECTS[namespace] + dcs = DATA_CONNECTIONS.get(namespace, []) + isvcs = INFERENCE_SERVICES.get(namespace, {}) + return json.dumps({ + "namespace": namespace, + "display_name": proj["display_name"], + "status": "Active", + "components": { + "data_connections": len(dcs), + "inference_services": len(isvcs), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + "pipeline_server": "configured" if proj.get("pipeline_server") else "not configured", + }, + }) + + +# ── Data connection tools ──────────────────────────────────────────────── + + +@mcp.tool() +def create_s3_data_connection( + namespace: str, + name: str, + bucket: str, + endpoint: str, + access_key: str, + secret_key: str, + region: str = "", +) -> str: + """Create an S3-compatible data connection in an RHOAI project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + existing = DATA_CONNECTIONS.get(namespace, []) + if any(dc["name"] == name for dc in existing): + raise ValueError( + f"Data connection '{name}' already exists in namespace '{namespace}'" + ) + + dc = { + "name": name, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + "region": region, + } + DATA_CONNECTIONS.setdefault(namespace, []).append(dc) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + }) + + +@mcp.tool() +def list_data_connections(namespace: str) -> str: + """List data connections in an RHOAI project namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + return json.dumps(dcs, indent=2) + + +# ── Model serving tools ───────────────────────────────────────────────── + + +@mcp.tool() +def set_model_serving_mode(namespace: str, mode: str) -> str: + """Enable model serving on a Data Science Project (single or multi mode).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + if mode not in ("single", "multi"): + raise ValueError(f"Invalid mode '{mode}'. Must be 'single' or 'multi'.") + + PROJECTS[namespace]["model_serving_mode"] = mode + + if not SERVING_RUNTIMES.get(namespace): + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + SERVING_RUNTIMES[namespace] = [ + {**t, "requires_instantiation": False, "source": "existing"} + for t in templates + ] + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "mode": mode, + }) + + +@mcp.tool() +def list_serving_runtimes( + namespace: str, + include_templates: bool = False, +) -> str: + """List available ServingRuntimes in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + runtimes = list(SERVING_RUNTIMES.get(namespace, [])) + if include_templates: + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + existing_names = {r["name"] for r in runtimes} + for t in templates: + if t["name"] not in existing_names: + runtimes.append(t) + + return json.dumps(runtimes, indent=2) + + +# ── Inference service tools ────────────────────────────────────────────── + + +@mcp.tool() +def deploy_model( + name: str, + namespace: str, + runtime: str, + model_format: str, + storage_uri: str, + display_name: str = "", + min_replicas: int = 1, + max_replicas: int = 1, + cpu_request: str = "1", + cpu_limit: str = "2", + memory_request: str = "4Gi", + memory_limit: str = "8Gi", + gpu_count: int = 0, +) -> str: + """Deploy an AI/ML model as a KServe InferenceService.""" + if namespace not in PROJECTS: + raise ValueError( + f"Namespace '{namespace}' is not a Data Science Project. " + "Create one via create_data_science_project first." + ) + + ns_runtimes = SERVING_RUNTIMES.get(namespace, []) + runtime_names = [r["name"] for r in ns_runtimes] + if runtime not in runtime_names: + available = ", ".join(runtime_names) or "none" + raise ValueError( + f"ServingRuntime '{runtime}' not found in namespace '{namespace}'. " + f"Available runtimes: {available}" + ) + + endpoint = f"https://{name}-{namespace}.apps.ocp-cluster.example.com" + isvc = { + "name": name, + "namespace": namespace, + "runtime": runtime, + "model_format": model_format, + "storage_uri": storage_uri, + "display_name": display_name or name, + "gpu_count": gpu_count, + "cpu_request": cpu_request, + "memory_request": memory_request, + "min_replicas": min_replicas, + "max_replicas": max_replicas, + "ready": True, + "url": endpoint, + "conditions": [ + {"type": "Ready", "status": "True", "reason": "Ready", "message": ""}, + {"type": "PredictorReady", "status": "True", "reason": "PodReady", "message": ""}, + {"type": "IngressReady", "status": "True", "reason": "IngressReady", "message": ""}, + ], + "age": "0s", + } + + INFERENCE_SERVICES.setdefault(namespace, {})[name] = isvc + DEPLOYED_MODELS[f"{namespace}/{name}"] = isvc + + return json.dumps({ + "status": "deployed", + "name": name, + "namespace": namespace, + "runtime": runtime, + "endpoint": endpoint, + "ready": True, + }) + + +@mcp.tool() +def list_inference_services( + namespace: str, + verbosity: str = "standard", +) -> str: + """List deployed InferenceServices in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + isvcs = INFERENCE_SERVICES.get(namespace, {}) + results = [] + for isvc_name, isvc in isvcs.items(): + entry = { + "name": isvc["name"], + "runtime": isvc["runtime"], + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "age": isvc.get("age", ""), + } + if verbosity == "full": + entry["conditions"] = isvc.get("conditions", []) + entry["storage_uri"] = isvc.get("storage_uri", "") + entry["gpu_count"] = isvc.get("gpu_count", 0) + results.append(entry) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def get_inference_service( + name: str, + namespace: str, + verbosity: str = "standard", +) -> str: + """Get detailed status of a specific InferenceService.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + + isvc = isvcs[name] + result = { + "name": isvc["name"], + "namespace": isvc["namespace"], + "runtime": isvc["runtime"], + "model_format": isvc.get("model_format", ""), + "storage_uri": isvc.get("storage_uri", ""), + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "conditions": isvc.get("conditions", []), + "gpu_count": isvc.get("gpu_count", 0), + "replicas": {"min": isvc.get("min_replicas", 1), "max": isvc.get("max_replicas", 1)}, + "resources": { + "cpu_request": isvc.get("cpu_request", "1"), + "memory_request": isvc.get("memory_request", "4Gi"), + "memory_limit": isvc.get("memory_limit", "8Gi"), + }, + "age": isvc.get("age", ""), + } + return json.dumps(result, indent=2) + + +@mcp.tool() +def get_model_endpoint(name: str, namespace: str) -> str: + """Get the inference endpoint URL for a deployed model.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + isvc = isvcs[name] + if not isvc["ready"]: + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": "", + "error": "InferenceService is not ready. Check conditions for details.", + }) + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": isvc["url"], + }) + + +# ── Workbench tools ────────────────────────────────────────────────────── + + +@mcp.tool() +def list_workbenches(namespace: str) -> str: + """List workbenches (Jupyter notebooks) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + return json.dumps(wbs, indent=2) + + +@mcp.tool() +def create_workbench( + namespace: str, + name: str, + display_name: str = "", + image: str = "jupyter-datascience-ubi9-python-3.9-2024.1", + cpu_request: str = "1", + memory_request: str = "4Gi", + gpu_count: int = 0, + pvc_size: str = "20Gi", +) -> str: + """Create a new workbench (Jupyter notebook) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + valid_images = [img["name"] for img in NOTEBOOK_IMAGES] + if image not in valid_images: + raise ValueError( + f"Image '{image}' not found. Available: {', '.join(valid_images)}" + ) + + wb = { + "name": name, + "display_name": display_name or name, + "image": image, + "status": "Running", + "cpu_request": cpu_request, + "memory_request": memory_request, + "gpu_count": gpu_count, + "pvc_name": f"{name}-pvc", + "pvc_size": pvc_size, + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-03-02T12:00:00Z", + } + WORKBENCHES.setdefault(namespace, []).append(wb) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "image": image, + "pvc": f"{name}-pvc", + }) + + +@mcp.tool() +def stop_workbench(namespace: str, name: str) -> str: + """Stop a running workbench (preserves data).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Stopped" + return json.dumps({"status": "stopped", "name": name, "namespace": namespace}) + + +@mcp.tool() +def start_workbench(namespace: str, name: str) -> str: + """Start a stopped workbench.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Running" + return json.dumps({"status": "running", "name": name, "namespace": namespace}) + + +@mcp.tool() +def delete_workbench(namespace: str, name: str) -> str: + """Delete a workbench. WARNING: PVC data may be lost if not backed up.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wbs.remove(wb) + return json.dumps({ + "status": "deleted", + "name": name, + "namespace": namespace, + "warning": "Associated PVC data has been deleted", + }) + + +@mcp.tool() +def list_notebook_images() -> str: + """List available notebook images for workbench creation.""" + return json.dumps(NOTEBOOK_IMAGES, indent=2) + + +# ── Pipeline server tools ─────────────────────────────────────────────── + + +@mcp.tool() +def configure_pipeline_server( + namespace: str, + data_connection: str, + database: str = "MariaDB", +) -> str: + """Configure a pipeline server for a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + if not any(dc["name"] == data_connection for dc in dcs): + available = [dc["name"] for dc in dcs] + raise ValueError( + f"Data connection '{data_connection}' not found. Available: {available}" + ) + + PIPELINE_SERVERS[namespace] = { + "configured": True, + "data_connection": data_connection, + "status": "Ready", + "database": database, + } + PROJECTS[namespace]["pipeline_server"] = True + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "data_connection": data_connection, + "database": database, + }) + + +@mcp.tool() +def get_pipeline_server_status(namespace: str) -> str: + """Get the status of the pipeline server in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + ps = PIPELINE_SERVERS.get(namespace) + if not ps: + return json.dumps({"namespace": namespace, "configured": False}) + return json.dumps({ + "namespace": namespace, + "configured": ps["configured"], + "data_connection": ps["data_connection"], + "status": ps["status"], + "database": ps["database"], + }) + + +# ── Serving runtime creation ──────────────────────────────────────────── + + +@mcp.tool() +def create_serving_runtime( + namespace: str, + name: str, + display_name: str = "", + model_formats: list = None, + container_image: str = "", + container_port: int = 8080, + multi_model: bool = False, + api_protocol: str = "REST", +) -> str: + """Create a custom ServingRuntime in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + if not model_formats: + raise ValueError("model_formats must specify at least one model format") + + runtime = { + "name": name, + "display_name": display_name or name, + "model_formats": model_formats, + "requires_instantiation": False, + "source": "custom", + "api_protocol": api_protocol, + "container_image": container_image, + "container_port": container_port, + "multi_model": multi_model, + } + SERVING_RUNTIMES.setdefault(namespace, []).append(runtime) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "model_formats": model_formats, + }) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/instruction.md b/evaluation/without_skills/rh-ai-engineer__debug-inference/instruction.md new file mode 100644 index 00000000..11b9268d --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__debug-inference/instruction.md @@ -0,0 +1,13 @@ +# Inference Debugging Task + +You are an AI engineer on Red Hat OpenShift AI. There are failing model inference deployments in the `ml-production` namespace that need debugging. + +## Requirements +- List all InferenceServices in the `ml-production` namespace and identify which ones are not ready +- For each failing InferenceService, diagnose the root cause: check status conditions, pod state, container logs, events, and related resources (ServingRuntime, Account CRs) +- Recommend a specific fix for each failing deployment +- Document your methodology and the diagnostic steps you followed + +Use MCP tools to interact with the platform. Write your complete findings and recommendations in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/solution/solve.sh b/evaluation/without_skills/rh-ai-engineer__debug-inference/solution/solve.sh new file mode 100644 index 00000000..6b94e02f --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__debug-inference/solution/solve.sh @@ -0,0 +1,36 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Inference Debug Report + +## Diagnosis Categories (get_inference_service verbosity full) + +### 1. ServingRuntime ✓ +ServingRuntime CR exists and is valid + +### 2. Pod Scheduling ✗ +Pod cannot be scheduled — check kserve-container logs (pods_log container=kserve-container) +Label selector: serving.kserve.io/inferenceservice + +### 3. Container Start +KServe sidecar containers may conflict with LimitRange + +### 4. Model Loading +Check model download and initialization + +### 5. GPU Access +Verify GPU allocation and CUDA compatibility + +### 6. Endpoint Health +Check InferenceService URL and readiness (PredictorReady, IngressReady conditions) + +## Events +events_list filtered by namespace for pod/InferenceService events + +## NIM Deployments +For NIM: Check Account CR (nim.opendatahub.io) for NGC credential errors + +## Observability (optional) +- korrel8r_get_correlated for cross-domain signals +- query_tempo_tool for trace latency +- execute_promql for custom metrics +REPORT_EOF diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/task.toml b/evaluation/without_skills/rh-ai-engineer__debug-inference/task.toml new file mode 100644 index 00000000..44c5ea2a --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__debug-inference/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-ai-engineer__debug-inference" +name = "rh-ai-engineer Inference Debugging Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-ai-engineer", "debug-inference", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/tests/llm_judge.py b/evaluation/without_skills/rh-ai-engineer__debug-inference/tests/llm_judge.py new file mode 100644 index 00000000..748256ca --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__debug-inference/tests/llm_judge.py @@ -0,0 +1,114 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "kserve_conditions_awareness", + "file": "/root/report.md", + "question": "Does the report explicitly name the KServe InferenceService status conditions — specifically PredictorReady and IngressReady — and present them in a structured conditions table with Status/Reason/Message columns?", + "reference": "A skilled report should present a conditions table showing PredictorReady and IngressReady as distinct conditions with their status (True/False), reason, and message. Simply reporting 'CrashLoopBackOff' or 'pod failing' without naming the specific KServe conditions is insufficient." + }, + { + "id": "kserve_container_specificity", + "file": "/root/report.md", + "question": "Does the report mention 'kserve-container' by name as the specific container to inspect for logs, and reference the serving.kserve.io/inferenceservice label selector as the method for discovering predictor pods?", + "reference": "A skilled report should mention 'kserve-container' as the container name for log inspection and reference the serving.kserve.io/inferenceservice label selector for pod discovery. Generically saying 'check pod logs' or 'look at the container' without these specific KServe identifiers is insufficient." + }, + { + "id": "nim_account_cr_pattern", + "file": "/root/report.md", + "question": "Does the report prescribe creating a NIM Account custom resource (kind: Account) as the credential management mechanism for NVIDIA NIM, rather than only manually creating docker-registry secrets and patching service accounts?", + "reference": "A skilled report creates a NIM Account CR (kind: Account, apiVersion: nvidia.com/v1alpha1) with ngcSecret reference and imagePullSecret auto-creation. An unskilled report manually creates docker-registry secrets and patches service accounts without using the Account CR pattern." + }, + { + "id": "ngc_credential_expiry", + "file": "/root/report.md", + "question": "Does the report identify NGC API key or pull-secret expiry as a possible root cause for image pull failures in NIM deployments, and recommend checking the secret's expiration date?", + "reference": "A skilled report checks whether the NGC pull-secret has expired as a diagnosis step for ImagePullBackOff. An unskilled report treats image pull failures generically without considering credential expiry." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/tests/test.sh b/evaluation/without_skills/rh-ai-engineer__debug-inference/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__debug-inference/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/tests/test_outputs.py b/evaluation/without_skills/rh-ai-engineer__debug-inference/tests/test_outputs.py new file mode 100644 index 00000000..60f73901 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__debug-inference/tests/test_outputs.py @@ -0,0 +1,98 @@ +""" +Tests for rh-ai-engineer__debug-inference per-skill evaluation. +Baseline tests: any competent agent should pass. +Skill-dependent tests: based on empirical gaps between skilled and unskilled agent outputs. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ["inference", "model", "serving", "deploy"]), ( + "report should mention inference" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 200, "report should have substantial content" + + +class TestSkillDependent: + def test_kserve_status_conditions(self): + """Skill teaches presenting PredictorReady and IngressReady as distinct KServe conditions. + Without skill, agents report generic pod status (CrashLoopBackOff) without naming these conditions.""" + c = read_report().lower() + assert any(t in c for t in [ + "predictorready", "predictor ready", "predictor_ready", + "ingressready", "ingress ready", "ingress_ready", + ]), "should name KServe status conditions (PredictorReady, IngressReady)" + + def test_kserve_container_name(self): + """Skill teaches 'kserve-container' as the specific container for log inspection. + Without skill, agents check logs generically without naming this container.""" + c = read_report().lower() + assert "kserve-container" in c or "kserve container" in c, ( + "should mention kserve-container by name as the container to inspect" + ) + + def test_label_selector_methodology(self): + """Skill teaches using serving.kserve.io/inferenceservice label to find predictor pods. + Without skill, agents discover pods through generic namespace listing.""" + c = read_report().lower() + assert any(t in c for t in [ + "serving.kserve.io", "kserve.io/inferenceservice", + ]), "should reference the KServe label selector for predictor pod discovery" + + def test_account_cr_awareness(self): + """Skill teaches NIM Account CR as the credential management mechanism. + Without skill, agents manually create docker-registry secrets and + patch service accounts instead of using the Account custom resource.""" + c = read_report() + assert any(t in c for t in [ + "Account CR", "kind: Account", "Account resource", + "Account custom resource", + ]) or "account cr" in c.lower(), ( + "should reference NIM Account CR as credential management mechanism" + ) + + def test_nim_api_version(self): + """Skill teaches the nvidia.com API group for NIM Account and ngcSecret + field for NGC credential binding. Without skill, agents create + generic secrets without the Account CR pattern.""" + c = read_report().lower() + assert any(t in c for t in [ + "nvidia.com/v1alpha1", "ngcsecret", "ngc_api_key", + ]) or ("account" in c and "api" in c and "nvidia" in c), ( + "should reference NIM Account API version or NGC secret binding" + ) + + def test_root_cause_with_remediation(self): + """Both agents should link diagnosis to fix — easy test.""" + c = read_report().lower() + has_diagnosis = any(t in c for t in ["oom", "memory", "crash", "fail"]) + has_fix = any(t in c for t in ["fix", "recommend", "solution", "increase", "reduce"]) + assert has_diagnosis and has_fix, "should link diagnosis to recommended fix" + + def test_ngc_pull_secret_expiry(self): + """Docs teach NGC pull-secret expiry as a common issue, and + 'Insufficient nvidia.com/gpu' as GPU scheduling error signature. + Without docs, agents miss these specific failure patterns.""" + c = read_report().lower() + assert any(t in c for t in [ + "ngc", "pull-secret", "pull secret", "expir", + "insufficient nvidia.com/gpu", "nvidia.com/gpu", + ]), "should address NGC pull-secret expiry or GPU scheduling errors" diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/Dockerfile b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/Dockerfile new file mode 100644 index 00000000..aac4c84e --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/Dockerfile @@ -0,0 +1,67 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "rhoai": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-rhoai-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/common-issues.md b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/model-deploy.md b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/model-deploy.md new file mode 100644 index 00000000..dcd79e44 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/model-deploy.md @@ -0,0 +1,160 @@ +# /model-deploy Examples + +Example interactions demonstrating the model-deploy skill workflow. + +## Example 1: Deploy Llama 3.1 8B with vLLM + +``` +User: "Deploy Llama 3.1 8B on my cluster" + +Agent: [Invokes /model-deploy skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand runtime capabilities. + +Step 1 of 10: Gathering deployment information. + +| Setting | Value | Source | +|---------|-------|--------| +| Model | Llama 3.1 8B Instruct | user input | +| Runtime | TBD | auto-detect | +| Namespace | ? | needs input | +| Model Source | ? | needs input | +| Deployment Mode | Serverless | default | + +What namespace should I deploy to, and where are the model weights stored? +(e.g., S3 bucket, OCI registry, PVC) + +User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" + +Agent: Step 2 of 10: Determining runtime. + +I consulted supported-runtimes.md to understand runtime capabilities. + +Based on Llama 3.1 8B (open-source HuggingFace model): + +Recommended runtime: vLLM +Reason: Default runtime for open-source LLMs. Broad compatibility, + OpenAI-compatible API, no additional setup required. + +Proceed with vLLM? (yes / no / suggest alternative) + +User: "yes" + +Agent: Step 3 of 10: Looking up hardware profile. + +I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. + +| Requirement | Value | +|-------------|-------| +| GPUs | 1x A10G/L4/A100/H100 | +| VRAM | 16GB | +| Key Args | --max-model-len=4096 --tool-call-parser hermes | + +Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). +Ensure your cluster has at least 1 GPU with 16GB VRAM. + +Step 5 of 10: Skipped (NIM platform check not needed for vLLM). + +Step 6-7 of 10: Generating InferenceService YAML. + +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: llama-3-1-8b + namespace: ai-models + annotations: + serving.kserve.io/deploymentMode: Serverless +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: vllm-runtime + storageUri: s3://my-bucket/llama-3.1-8b + resources: + limits: + nvidia.com/gpu: "1" + requests: + cpu: "4" + memory: "16Gi" + args: + - --max-model-len=4096 + - --tool-call-parser=hermes + +| Setting | Value | +|---------|-------| +| Model | llama-3-1-8b | +| Runtime | vllm-runtime | +| GPUs | 1 | +| Namespace | ai-models | +| Mode | Serverless | + +Proceed with creating this InferenceService? (yes/no/modify) + +User: "yes" + +Agent: Step 8 of 10: Creating InferenceService... + [Creates via resources_create_or_update] + + InferenceService created. Monitoring rollout... + +Step 9 of 10: Monitoring rollout. + +| Stage | Status | +|-------|--------| +| Pod Scheduled | Complete | +| Image Pulled | Complete | +| Container Started | Complete | +| Model Loaded | Complete | +| Ready | Complete | + +Step 10 of 10: Deployment complete! + +Model Deployment Successful + +Model: llama-3-1-8b +Runtime: vLLM +Namespace: ai-models +GPUs: 1 + +Inference Endpoint: + URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com + API: OpenAI-compatible REST + +Quick Test: + curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' + +Next Steps: + Monitor performance: "/ai-observability llama-3-1-8b" +``` + +## Example 2: Deploy a Model with NIM (NIM Not Set Up) + +``` +User: "Deploy Llama 3.1 70B with NIM" + +Agent: [Invokes /model-deploy skill] + ... + Step 2: User requested NIM runtime. + + Step 5: Checking NIM platform... + +NIM platform is not set up in namespace ai-models. + +The NIM runtime requires a one-time platform setup. Would you like to: +1. Run /nim-setup now to configure the NIM platform +2. Switch to vLLM runtime instead (no setup required) +3. Abort deployment + +User: "1" + +Agent: [Invokes /nim-setup skill — walks through NGC credential setup] + ... + NIM Platform Setup Complete. + + Resuming model deployment... + [Continues from Step 6 with NIM runtime] +``` diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/nim-setup.md b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/nim-setup.md new file mode 100644 index 00000000..ccfc1a5d --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/nim-setup.md @@ -0,0 +1,115 @@ +# /nim-setup Examples + +Example interactions demonstrating the nim-setup skill workflow. + +## Example 1: First-Time NIM Setup + +``` +User: "Set up NIM on my cluster" + +Agent: [Invokes /nim-setup skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand NIM platform requirements. + +Let me verify the prerequisites for NIM setup. + +Step 1 of 7: Checking GPU Operator and NFD Operator... + +GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) +NFD Operator: nfd.v4.19.0 (Succeeded) + +Both operators are installed and healthy. + +Step 2 of 7: I need your NGC credentials. + +To set up NIM, I need your NVIDIA NGC API key. +You can generate one at: https://ngc.nvidia.com/setup/api-key + +Please provide: +1. Your NGC API key +2. The target namespace for NIM resources + +User: "Here's my key: [user provides NGC API key], namespace: ai-models" + +Agent: Step 3 of 7: Creating NGC image pull secret... + +I'll create a docker-registry secret for pulling NIM images from nvcr.io: + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-image-pull-secret + namespace: ai-models + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] + +Should I create this image pull secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + Image pull secret created successfully. + +Step 4 of 7: Creating NGC API key secret... + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-api-key + namespace: ai-models + type: Opaque + stringData: + NGC_API_KEY: "[REDACTED]" + +Should I create this API key secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + API key secret created successfully. + +Step 5 of 7: Creating NIM Account CR... + + apiVersion: nim.opendatahub.io/v1 + kind: Account + metadata: + name: nim-account + namespace: ai-models + spec: + apiKeySecret: + name: ngc-api-key + imagePullSecret: + name: ngc-image-pull-secret + +Should I create this NIM Account CR in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates Account CR via resources_create_or_update] + Account CR created successfully. + +Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) + +User: "no" + +Agent: Step 7 of 7: Validating NIM platform readiness... + +NIM Platform Setup Complete + +Account CR: `nim-account` (namespace: ai-models) + Status: Ready + +NGC Credentials: + Image Pull Secret: ngc-image-pull-secret [created] + API Key Secret: ngc-api-key [created] + +NIM ServingRuntimes Available: + - nim-llama-3-1-8b-instruct + - nim-llama-3-1-70b-instruct + - nim-mistral-7b-instruct + +Next Steps: + To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" +``` diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/live-doc-lookup.md b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/known-model-profiles.md b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/known-model-profiles.md new file mode 100644 index 00000000..40148981 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/known-model-profiles.md @@ -0,0 +1,83 @@ +--- +title: Known Model Hardware Profiles +category: references +tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] +semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] +use_cases: [model-deploy, debug-inference, ai-observability] +last_updated: 2026-03-10 +--- + +# Known Model Hardware Profiles + +Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). + +**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. + +## Model Source Conventions + +Each model lists a recommended `storageUri` with its authentication requirements: +- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** +- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). +- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. + +When the user does not specify a model source, use the `hf://` URI listed in the profile below. + +## Llama 3.x (Meta) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | +| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | +| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | +| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | + +- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) +- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly + +## Granite 3.x (IBM/Red Hat) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | +| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | + +- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) +- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser granite --chat-template` +- Red Hat-supported model family on RHOAI + +## Mixtral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | +| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) +- Mixture-of-Experts architecture: only ~13B/45B parameters active per token + +## Mistral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | +| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) + +## When a Model Is Not Listed + +If the requested model is not in this file, the agent MUST use the live doc lookup protocol: + +1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure +2. Fetch hardware specs from the appropriate source +3. Report findings to the user before proceeding with deployment + +Common cases requiring live lookup: +- Newly released models (after this file's last update) +- Domain-specific fine-tuned models +- Models with custom quantization +- NIM-specific optimized profiles diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/supported-runtimes.md b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/supported-runtimes.md new file mode 100644 index 00000000..9d018651 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/supported-runtimes.md @@ -0,0 +1,104 @@ +--- +title: Supported Serving Runtimes +category: references +tags: [runtimes, vllm, nim, caikit, tgis, serving] +semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] +use_cases: [model-deploy, serving-runtime-config, nim-setup] +last_updated: 2026-02-26 +--- + +# Supported Serving Runtimes + +This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. + +## Runtime Comparison + +| Runtime | API Type | Model Format | GPU Support | License | Setup Required | +|---------|----------|-------------|-------------|---------|----------------| +| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | +| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | +| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | +| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | + +## vLLM + +**Default runtime for most open-source models.** + +- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) +- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized +- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) +- **Model source**: S3-compatible storage, OCI registry, PVC, URI +- **Key features**: + - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) + - PagedAttention for efficient memory management + - Tensor parallelism for multi-GPU inference + - Tool/function calling support (`--tool-call-parser`) + - Continuous batching for high throughput +- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name + +## NVIDIA NIM + +**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** + +- **API**: OpenAI-compatible REST +- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog +- **GPU support**: NVIDIA only (requires specific GPU models per model profile) +- **Model source**: NGC model catalog (pulled automatically via NGC credentials) +- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) +- **Key features**: + - TensorRT-LLM optimization for lower latency + - Pre-compiled model engines (no compilation on first load) + - Optimized and generic profiles per GPU type + - Automatic model download from NGC +- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. +- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM +- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. +- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state + +## Caikit+TGIS + +**Red Hat's Caikit format with Text Generation Inference Server.** + +- **API**: gRPC (not REST) +- **Model formats**: Caikit format (requires conversion from HuggingFace) +- **GPU support**: NVIDIA +- **Model source**: S3-compatible storage +- **Key features**: + - Red Hat-supported runtime + - gRPC API for streaming inference + - Integrated with RHOAI model serving platform +- **When to choose**: When the model is already in Caikit format or when gRPC API is required. +- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name + +## Custom Runtimes + +**User-provided ServingRuntime CRs for unsupported frameworks.** + +- **API**: Defined by the custom runtime +- **Model formats**: Defined by the custom runtime +- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. +- **How to create**: Use `/serving-runtime-config` skill +- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility + +## Runtime Selection Decision Tree + +``` +Is the user's preferred runtime explicitly stated? +├── Yes → Use that runtime +└── No → Continue + +Is the model available in the NGC NIM catalog? +├── Yes → Suggest NIM (with vLLM as fallback) +│ Note: Requires /nim-setup and NVIDIA GPUs +└── No → Continue + +Is the model in Caikit format? +├── Yes → Caikit+TGIS +└── No → Continue + +Is the model a standard open-source LLM (HuggingFace-compatible)? +├── Yes → vLLM (default) +└── No → Custom runtime via /serving-runtime-config +``` diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/skill-conventions.md b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..e7a4d11c --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +"""Mock OpenShift MCP server for SkillsBench rh-ai-engineer task. + +Simulates Kubernetes resource CRUD, pod management, logs, and events. + +Key scenario elements: +- LimitRange in namespaces: min CPU=100m, min memory=128Mi + (conflicts with KServe sidecar containers hardcoded at 10m CPU/15Mi memory) +- GPU node with custom taint ai-workload=true:NoSchedule +- NIM Account CR in ml-production: not ready (NGC credentials invalid) +- text-gen-legacy pods: OOMKilled (max-model-len=32768 on A10G) +- nim-llama-prod: no pods created (Account CR not ready) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + +# ── Cluster state ──────────────────────────────────────────────────────── + +GPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "gpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + "nvidia.com/gpu.present": "true", + "nvidia.com/gpu.product": "NVIDIA-A10G", + }, + }, + "spec": { + "taints": [ + { + "key": "ai-workload", + "value": "true", + "effect": "NoSchedule", + }, + ], + }, + "status": { + "allocatable": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "capacity": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "conditions": [ + {"type": "Ready", "status": "True"}, + ], + }, +} + +CPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "cpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + }, + }, + "spec": {"taints": []}, + "status": { + "allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "capacity": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +MASTER_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "master-1", + "labels": { + "node-role.kubernetes.io/master": "", + "node-role.kubernetes.io/control-plane": "", + }, + }, + "spec": { + "taints": [ + {"key": "node-role.kubernetes.io/master", "effect": "NoSchedule"}, + ], + }, + "status": { + "allocatable": {"cpu": "8", "memory": "32Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +ALL_NODES = [GPU_NODE, CPU_NODE, MASTER_NODE] + +# LimitRange applied by cluster policy to all DS project namespaces +NAMESPACE_LIMITRANGE = { + "apiVersion": "v1", + "kind": "LimitRange", + "metadata": { + "name": "default-limits", + }, + "spec": { + "limits": [ + { + "type": "Container", + "default": { + "cpu": "2", + "memory": "4Gi", + }, + "defaultRequest": { + "cpu": "500m", + "memory": "256Mi", + }, + "min": { + "cpu": "100m", + "memory": "128Mi", + }, + "max": { + "cpu": "32", + "memory": "128Gi", + }, + }, + ], + }, +} + +NIM_ACCOUNT_CR = { + "apiVersion": "nim.opendatahub.io/v1", + "kind": "Account", + "metadata": { + "name": "nim-account", + "namespace": "ml-production", + }, + "spec": { + "apiKeySecret": { + "name": "ngc-api-key", + }, + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "NGCCredentialsInvalid", + "message": "NGC API key validation failed: 401 Unauthorized. " + "The API key in secret 'ngc-api-key' is expired or invalid. " + "Re-create the secret with a valid NGC API key from " + "https://ngc.nvidia.com/setup/api-key and restart the " + "Account reconciliation.", + "lastTransitionTime": "2026-03-14T12:00:00Z", + }, + ], + "nimPullSecretStatus": "Failed", + "nimConfigStatus": "Pending", + }, +} + +SERVING_RUNTIME_VLLM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "vllm-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "vLLM", "version": "1", "autoSelect": True}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "quay.io/modh/vllm:rhoai-2.16", + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + }, + ], + }, +} + +SERVING_RUNTIME_NIM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "nim-serving-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "NIM", "version": "1"}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "nvcr.io/nim/meta/llama-3.1-8b-instruct:latest", + "ports": [{"containerPort": 8000, "protocol": "TCP"}], + "env": [ + {"name": "NGC_API_KEY", "valueFrom": { + "secretKeyRef": {"name": "ngc-api-key", "key": "api_key"}, + }}, + ], + }, + ], + }, +} + +PODS_BY_NAMESPACE = { + "ml-production": [ + { + "name": "text-gen-legacy-predictor-00001-abc12", + "namespace": "ml-production", + "status": "CrashLoopBackOff", + "restarts": 5, + "node": "gpu-worker-1", + "containers": [ + { + "name": "kserve-container", + "state": "waiting", + "reason": "CrashLoopBackOff", + "last_termination_reason": "OOMKilled", + "last_termination_exit_code": 137, + }, + ], + "labels": { + "serving.kserve.io/inferenceservice": "text-gen-legacy", + }, + "gpu": "1", + }, + # nim-llama-prod: NO pods created (Account CR not ready) + ], +} + +POD_LOGS = { + "text-gen-legacy-predictor-00001-abc12": ( + "INFO 2026-03-01 10:00:00 vllm_engine.py:125] vLLM engine starting...\n" + "INFO 2026-03-01 10:00:01 config.py:89] Model: mistralai/Mistral-7B-Instruct-v0.3\n" + "INFO 2026-03-01 10:00:01 config.py:92] max_model_len = 32768\n" + "INFO 2026-03-01 10:00:02 gpu_executor.py:45] GPU 0: NVIDIA A10G (24576 MiB)\n" + "INFO 2026-03-01 10:00:03 model_runner.py:88] Loading model weights...\n" + "INFO 2026-03-01 10:00:15 model_runner.py:112] Model weights loaded: 13.5 GiB\n" + "INFO 2026-03-01 10:00:15 worker.py:201] Allocating KV cache...\n" + "ERROR 2026-03-01 10:00:16 worker.py:215] torch.cuda.OutOfMemoryError: " + "CUDA out of memory. Tried to allocate 28.5 GiB for KV cache but only " + "10.1 GiB available after loading model weights (13.5 GiB).\n" + "ERROR 2026-03-01 10:00:16 vllm_engine.py:178] Engine failed to start\n" + "Traceback (most recent call last):\n" + " File \"/opt/vllm/vllm/engine/engine.py\", line 175, in start\n" + " self._init_kv_cache()\n" + " File \"/opt/vllm/vllm/worker/worker.py\", line 215, in _init_kv_cache\n" + " raise torch.cuda.OutOfMemoryError(msg)\n" + "torch.cuda.OutOfMemoryError: CUDA out of memory\n" + ), +} + +EVENTS_BY_NAMESPACE = { + "ml-production": [ + { + "type": "Warning", + "reason": "BackOff", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Back-off restarting failed container kserve-container in pod " + "text-gen-legacy-predictor-00001-abc12", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Warning", + "reason": "OOMKilled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Container kserve-container was OOMKilled (exit code 137). " + "GPU memory exhausted during KV cache allocation.", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Normal", + "reason": "Scheduled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Successfully assigned ml-production/" + "text-gen-legacy-predictor-00001-abc12 to gpu-worker-1", + "count": 1, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-02-28T08:00:00Z", + }, + { + "type": "Warning", + "reason": "NIMAccountNotReady", + "object": "InferenceService/nim-llama-prod", + "message": "NIM Account 'nim-account' in namespace 'ml-production' " + "is not ready", + "count": 12, + "first_timestamp": "2026-03-14T12:00:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + { + "type": "Warning", + "reason": "ImagePullBackOff", + "object": "InferenceService/nim-llama-prod", + "message": "Failed to pull image 'nvcr.io/nim/meta/llama-3.1-8b-instruct:" + "latest': unauthorized: authentication required", + "count": 8, + "first_timestamp": "2026-03-14T12:05:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + ], +} + + +# ── Resource tools ─────────────────────────────────────────────────────── + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: str = "", +) -> str: + """Get a single Kubernetes resource by apiVersion, kind, and name.""" + if kind == "Node": + for node in ALL_NODES: + if node["metadata"]["name"] == name: + return json.dumps(node, indent=2) + raise ValueError(f"Node '{name}' not found") + + if kind == "ServingRuntime": + if name == "vllm-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_VLLM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + if name == "nim-serving-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_NIM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + raise ValueError(f"ServingRuntime '{name}' not found in namespace '{namespace}'") + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps(lr, indent=2) + + if kind == "Account" and "nim" in apiVersion.lower(): + if namespace == "ml-production" and name == "nim-account": + return json.dumps(NIM_ACCOUNT_CR, indent=2) + raise ValueError( + f"Account '{name}' not found in namespace '{namespace}'" + ) + + if kind == "ClusterVersion" and apiVersion == "config.openshift.io/v1": + return json.dumps({ + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterVersion", + "metadata": {"name": "version"}, + "status": {"desired": {"version": "4.16.3"}}, + }) + + raise ValueError(f"Resource {apiVersion}/{kind}/{name} not found") + + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: str = "", + labelSelector: str = "", +) -> str: + """List Kubernetes resources by apiVersion and kind.""" + if kind == "Node": + nodes = ALL_NODES + if labelSelector: + parts = labelSelector.split("=", 1) + key = parts[0] + value = parts[1] if len(parts) > 1 else "" + nodes = [ + n for n in nodes + if n["metadata"]["labels"].get(key) == value + ] + return json.dumps(nodes, indent=2) + + if kind == "Service" and apiVersion == "serving.knative.dev/v1": + return json.dumps({ + "kind": "ServiceList", + "apiVersion": "serving.knative.dev/v1", + "items": [], + "metadata": {}, + }) + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps({ + "kind": "LimitRangeList", + "items": [lr], + }) + + if kind == "InferenceService": + return json.dumps({ + "kind": "InferenceServiceList", + "items": [], + }) + + raise ValueError(f"Unsupported list: {apiVersion}/{kind}") + + +@mcp.tool() +def pods_list( + namespace: str, + labelSelector: str = "", +) -> str: + """List pods in a namespace with optional label selector.""" + pods = PODS_BY_NAMESPACE.get(namespace, []) + + if labelSelector: + key, _, value = labelSelector.partition("=") + pods = [p for p in pods if p.get("labels", {}).get(key) == value] + + results = [] + for pod in pods: + results.append({ + "name": pod["name"], + "namespace": pod["namespace"], + "status": pod["status"], + "restarts": pod.get("restarts", 0), + "node": pod.get("node", ""), + "containers": pod.get("containers", []), + "gpu": pod.get("gpu", "0"), + }) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def pods_log( + namespace: str, + name: str, + container: str = "", +) -> str: + """Get logs from a pod container.""" + logs = POD_LOGS.get(name) + if logs is None: + raise ValueError(f"Pod '{name}' not found in namespace '{namespace}'") + return logs + + +@mcp.tool() +def events_list(namespace: str) -> str: + """List events in a namespace.""" + events = EVENTS_BY_NAMESPACE.get(namespace, []) + return json.dumps(events, indent=2) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/mcp-servers/mock-rhoai-mcp.py b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/mcp-servers/mock-rhoai-mcp.py new file mode 100644 index 00000000..9b072b37 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/mcp-servers/mock-rhoai-mcp.py @@ -0,0 +1,796 @@ +#!/usr/bin/env python3 +"""Mock RHOAI MCP server for SkillsBench rh-ai-engineer task. + +Simulates Red Hat OpenShift AI operations: Data Science Projects, +model serving, data connections, serving runtimes, inference services. + +Scenario: +- ml-production: existing project with two broken deployments + - text-gen-legacy: vLLM OOMKilled (max-model-len=32768 on A10G) + - nim-llama-prod: NIM failing (Account CR not ready, NGC creds invalid) +- fraud-detection: does not exist yet (agent creates it) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("rhoai") + +# ── In-memory state ────────────────────────────────────────────────────── + +PROJECTS = { + "ml-production": { + "name": "ml-production", + "display_name": "ML Production", + "description": "Production ML workloads", + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": "single", + "pipeline_server": True, + }, +} + +DATA_CONNECTIONS = { + "ml-production": [ + { + "name": "prod-model-store", + "type": "S3", + "bucket": "ml-models-prod", + "endpoint": "https://s3.us-east-1.amazonaws.com", + "region": "us-east-1", + }, + ], +} + +SERVING_RUNTIMES = { + "__platform_templates__": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "REST", + "supported_model_formats": [ + {"name": "vLLM", "version": "1", "autoSelect": True} + ], + }, + { + "name": "caikit-tgis-runtime", + "display_name": "Caikit+TGIS ServingRuntime", + "model_formats": ["caikit"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "gRPC", + }, + ], + "ml-production": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + { + "name": "nim-serving-runtime", + "display_name": "NVIDIA NIM ServingRuntime", + "model_formats": ["NIM"], + "requires_instantiation": False, + "source": "nim-account", + "api_protocol": "REST", + }, + { + "name": "ovms-1", + "display_name": "OpenVINO Model Server", + "model_formats": ["openvino_ir", "onnx"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + ], +} + +INFERENCE_SERVICES = { + "ml-production": { + "text-gen-legacy": { + "name": "text-gen-legacy", + "namespace": "ml-production", + "runtime": "vllm-runtime", + "model_format": "vLLM", + "storage_uri": "hf://mistralai/Mistral-7B-Instruct-v0.3", + "display_name": "Mistral 7B Legacy", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "16Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "PredictorFailed", + "message": "Predictor pod is not ready", + }, + { + "type": "PredictorReady", + "status": "False", + "reason": "ContainerCrashLoop", + "message": "Container kserve-container terminated: " + "OOMKilled (exit code 137). 5 restarts.", + }, + { + "type": "IngressReady", + "status": "True", + "reason": "IngressReady", + "message": "Ingress is ready", + }, + ], + "age": "3d", + }, + "nim-llama-prod": { + "name": "nim-llama-prod", + "namespace": "ml-production", + "runtime": "nim-serving-runtime", + "model_format": "NIM", + "storage_uri": "nim://meta/llama-3.1-8b-instruct", + "display_name": "Llama 3.1 8B (NIM)", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "32Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "RuntimeNotReady", + "message": "ServingRuntime 'nim-serving-runtime' " + "is not in ready state", + }, + { + "type": "PredictorReady", + "status": "Unknown", + "reason": "PodNotCreated", + "message": "Predictor pod has not been created. " + "Waiting for ServingRuntime to become ready.", + }, + { + "type": "IngressReady", + "status": "Unknown", + "reason": "PredictorNotReady", + "message": "Waiting for predictor to become ready", + }, + ], + "age": "1d", + }, + }, +} + +DEPLOYED_MODELS = {} + +WORKBENCHES = { + "ml-production": [ + { + "name": "data-exploration-nb", + "display_name": "Data Exploration", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Running", + "cpu_request": "1", + "memory_request": "8Gi", + "gpu_count": 0, + "pvc_name": "data-exploration-nb-pvc", + "pvc_size": "20Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-10T09:00:00Z", + }, + { + "name": "model-training-nb", + "display_name": "Model Training", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Stopped", + "cpu_request": "4", + "memory_request": "16Gi", + "gpu_count": 1, + "pvc_name": "model-training-nb-pvc", + "pvc_size": "50Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-15T14:00:00Z", + }, + ], +} + +PIPELINE_SERVERS = { + "ml-production": { + "configured": True, + "data_connection": "prod-model-store", + "status": "Ready", + "database": "MariaDB", + }, +} + +NOTEBOOK_IMAGES = [ + {"name": "jupyter-pytorch-ubi9-python-3.9-2024.1", "display_name": "PyTorch 2024.1", "packages": ["torch", "transformers"]}, + {"name": "jupyter-tensorflow-ubi9-python-3.9-2024.1", "display_name": "TensorFlow 2024.1", "packages": ["tensorflow"]}, + {"name": "jupyter-datascience-ubi9-python-3.9-2024.1", "display_name": "Standard Data Science", "packages": ["pandas", "scikit-learn"]}, + {"name": "jupyter-minimal-ubi9-python-3.9-2024.1", "display_name": "Minimal Python", "packages": []}, +] + + +# ── Project tools ──────────────────────────────────────────────────────── + + +@mcp.tool() +def list_data_science_projects() -> str: + """List all RHOAI Data Science Projects on the cluster.""" + projects = [] + for name, proj in PROJECTS.items(): + projects.append({ + "name": name, + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + }) + return json.dumps(projects, indent=2) + + +@mcp.tool() +def create_data_science_project( + name: str, + display_name: str, + description: str = "", +) -> str: + """Create a new RHOAI Data Science Project (namespace with dashboard labels).""" + if name in PROJECTS: + raise ValueError( + f"Project '{name}' already exists. Choose a different name " + "or configure the existing project." + ) + if not name.replace("-", "").replace("_", "").isalnum() or len(name) > 63: + raise ValueError( + f"Invalid project name '{name}'. Must be DNS-compatible: " + "lowercase alphanumeric and hyphens, max 63 chars." + ) + + PROJECTS[name] = { + "name": name, + "display_name": display_name, + "description": description, + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": None, + "pipeline_server": False, + } + DATA_CONNECTIONS[name] = [] + SERVING_RUNTIMES[name] = [] + INFERENCE_SERVICES[name] = {} + + return json.dumps({ + "status": "created", + "name": name, + "display_name": display_name, + "namespace": name, + "labels": {"opendatahub.io/dashboard": "true"}, + }) + + +@mcp.tool() +def get_project_details(name: str) -> str: + """Get detailed information about an RHOAI Data Science Project.""" + if name not in PROJECTS: + raise ValueError(f"Project '{name}' not found") + proj = PROJECTS[name] + dc_count = len(DATA_CONNECTIONS.get(name, [])) + isvc_count = len(INFERENCE_SERVICES.get(name, {})) + return json.dumps({ + "name": proj["name"], + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "labels": proj["labels"], + "data_connections": dc_count, + "inference_services": isvc_count, + "model_serving_mode": proj.get("model_serving_mode"), + "pipeline_server": proj.get("pipeline_server", False), + }) + + +@mcp.tool() +def get_project_status(namespace: str) -> str: + """Get comprehensive status of an RHOAI Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Project '{namespace}' not found") + proj = PROJECTS[namespace] + dcs = DATA_CONNECTIONS.get(namespace, []) + isvcs = INFERENCE_SERVICES.get(namespace, {}) + return json.dumps({ + "namespace": namespace, + "display_name": proj["display_name"], + "status": "Active", + "components": { + "data_connections": len(dcs), + "inference_services": len(isvcs), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + "pipeline_server": "configured" if proj.get("pipeline_server") else "not configured", + }, + }) + + +# ── Data connection tools ──────────────────────────────────────────────── + + +@mcp.tool() +def create_s3_data_connection( + namespace: str, + name: str, + bucket: str, + endpoint: str, + access_key: str, + secret_key: str, + region: str = "", +) -> str: + """Create an S3-compatible data connection in an RHOAI project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + existing = DATA_CONNECTIONS.get(namespace, []) + if any(dc["name"] == name for dc in existing): + raise ValueError( + f"Data connection '{name}' already exists in namespace '{namespace}'" + ) + + dc = { + "name": name, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + "region": region, + } + DATA_CONNECTIONS.setdefault(namespace, []).append(dc) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + }) + + +@mcp.tool() +def list_data_connections(namespace: str) -> str: + """List data connections in an RHOAI project namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + return json.dumps(dcs, indent=2) + + +# ── Model serving tools ───────────────────────────────────────────────── + + +@mcp.tool() +def set_model_serving_mode(namespace: str, mode: str) -> str: + """Enable model serving on a Data Science Project (single or multi mode).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + if mode not in ("single", "multi"): + raise ValueError(f"Invalid mode '{mode}'. Must be 'single' or 'multi'.") + + PROJECTS[namespace]["model_serving_mode"] = mode + + if not SERVING_RUNTIMES.get(namespace): + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + SERVING_RUNTIMES[namespace] = [ + {**t, "requires_instantiation": False, "source": "existing"} + for t in templates + ] + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "mode": mode, + }) + + +@mcp.tool() +def list_serving_runtimes( + namespace: str, + include_templates: bool = False, +) -> str: + """List available ServingRuntimes in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + runtimes = list(SERVING_RUNTIMES.get(namespace, [])) + if include_templates: + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + existing_names = {r["name"] for r in runtimes} + for t in templates: + if t["name"] not in existing_names: + runtimes.append(t) + + return json.dumps(runtimes, indent=2) + + +# ── Inference service tools ────────────────────────────────────────────── + + +@mcp.tool() +def deploy_model( + name: str, + namespace: str, + runtime: str, + model_format: str, + storage_uri: str, + display_name: str = "", + min_replicas: int = 1, + max_replicas: int = 1, + cpu_request: str = "1", + cpu_limit: str = "2", + memory_request: str = "4Gi", + memory_limit: str = "8Gi", + gpu_count: int = 0, +) -> str: + """Deploy an AI/ML model as a KServe InferenceService.""" + if namespace not in PROJECTS: + raise ValueError( + f"Namespace '{namespace}' is not a Data Science Project. " + "Create one via create_data_science_project first." + ) + + ns_runtimes = SERVING_RUNTIMES.get(namespace, []) + runtime_names = [r["name"] for r in ns_runtimes] + if runtime not in runtime_names: + available = ", ".join(runtime_names) or "none" + raise ValueError( + f"ServingRuntime '{runtime}' not found in namespace '{namespace}'. " + f"Available runtimes: {available}" + ) + + endpoint = f"https://{name}-{namespace}.apps.ocp-cluster.example.com" + isvc = { + "name": name, + "namespace": namespace, + "runtime": runtime, + "model_format": model_format, + "storage_uri": storage_uri, + "display_name": display_name or name, + "gpu_count": gpu_count, + "cpu_request": cpu_request, + "memory_request": memory_request, + "min_replicas": min_replicas, + "max_replicas": max_replicas, + "ready": True, + "url": endpoint, + "conditions": [ + {"type": "Ready", "status": "True", "reason": "Ready", "message": ""}, + {"type": "PredictorReady", "status": "True", "reason": "PodReady", "message": ""}, + {"type": "IngressReady", "status": "True", "reason": "IngressReady", "message": ""}, + ], + "age": "0s", + } + + INFERENCE_SERVICES.setdefault(namespace, {})[name] = isvc + DEPLOYED_MODELS[f"{namespace}/{name}"] = isvc + + return json.dumps({ + "status": "deployed", + "name": name, + "namespace": namespace, + "runtime": runtime, + "endpoint": endpoint, + "ready": True, + }) + + +@mcp.tool() +def list_inference_services( + namespace: str, + verbosity: str = "standard", +) -> str: + """List deployed InferenceServices in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + isvcs = INFERENCE_SERVICES.get(namespace, {}) + results = [] + for isvc_name, isvc in isvcs.items(): + entry = { + "name": isvc["name"], + "runtime": isvc["runtime"], + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "age": isvc.get("age", ""), + } + if verbosity == "full": + entry["conditions"] = isvc.get("conditions", []) + entry["storage_uri"] = isvc.get("storage_uri", "") + entry["gpu_count"] = isvc.get("gpu_count", 0) + results.append(entry) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def get_inference_service( + name: str, + namespace: str, + verbosity: str = "standard", +) -> str: + """Get detailed status of a specific InferenceService.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + + isvc = isvcs[name] + result = { + "name": isvc["name"], + "namespace": isvc["namespace"], + "runtime": isvc["runtime"], + "model_format": isvc.get("model_format", ""), + "storage_uri": isvc.get("storage_uri", ""), + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "conditions": isvc.get("conditions", []), + "gpu_count": isvc.get("gpu_count", 0), + "replicas": {"min": isvc.get("min_replicas", 1), "max": isvc.get("max_replicas", 1)}, + "resources": { + "cpu_request": isvc.get("cpu_request", "1"), + "memory_request": isvc.get("memory_request", "4Gi"), + "memory_limit": isvc.get("memory_limit", "8Gi"), + }, + "age": isvc.get("age", ""), + } + return json.dumps(result, indent=2) + + +@mcp.tool() +def get_model_endpoint(name: str, namespace: str) -> str: + """Get the inference endpoint URL for a deployed model.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + isvc = isvcs[name] + if not isvc["ready"]: + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": "", + "error": "InferenceService is not ready. Check conditions for details.", + }) + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": isvc["url"], + }) + + +# ── Workbench tools ────────────────────────────────────────────────────── + + +@mcp.tool() +def list_workbenches(namespace: str) -> str: + """List workbenches (Jupyter notebooks) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + return json.dumps(wbs, indent=2) + + +@mcp.tool() +def create_workbench( + namespace: str, + name: str, + display_name: str = "", + image: str = "jupyter-datascience-ubi9-python-3.9-2024.1", + cpu_request: str = "1", + memory_request: str = "4Gi", + gpu_count: int = 0, + pvc_size: str = "20Gi", +) -> str: + """Create a new workbench (Jupyter notebook) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + valid_images = [img["name"] for img in NOTEBOOK_IMAGES] + if image not in valid_images: + raise ValueError( + f"Image '{image}' not found. Available: {', '.join(valid_images)}" + ) + + wb = { + "name": name, + "display_name": display_name or name, + "image": image, + "status": "Running", + "cpu_request": cpu_request, + "memory_request": memory_request, + "gpu_count": gpu_count, + "pvc_name": f"{name}-pvc", + "pvc_size": pvc_size, + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-03-02T12:00:00Z", + } + WORKBENCHES.setdefault(namespace, []).append(wb) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "image": image, + "pvc": f"{name}-pvc", + }) + + +@mcp.tool() +def stop_workbench(namespace: str, name: str) -> str: + """Stop a running workbench (preserves data).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Stopped" + return json.dumps({"status": "stopped", "name": name, "namespace": namespace}) + + +@mcp.tool() +def start_workbench(namespace: str, name: str) -> str: + """Start a stopped workbench.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Running" + return json.dumps({"status": "running", "name": name, "namespace": namespace}) + + +@mcp.tool() +def delete_workbench(namespace: str, name: str) -> str: + """Delete a workbench. WARNING: PVC data may be lost if not backed up.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wbs.remove(wb) + return json.dumps({ + "status": "deleted", + "name": name, + "namespace": namespace, + "warning": "Associated PVC data has been deleted", + }) + + +@mcp.tool() +def list_notebook_images() -> str: + """List available notebook images for workbench creation.""" + return json.dumps(NOTEBOOK_IMAGES, indent=2) + + +# ── Pipeline server tools ─────────────────────────────────────────────── + + +@mcp.tool() +def configure_pipeline_server( + namespace: str, + data_connection: str, + database: str = "MariaDB", +) -> str: + """Configure a pipeline server for a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + if not any(dc["name"] == data_connection for dc in dcs): + available = [dc["name"] for dc in dcs] + raise ValueError( + f"Data connection '{data_connection}' not found. Available: {available}" + ) + + PIPELINE_SERVERS[namespace] = { + "configured": True, + "data_connection": data_connection, + "status": "Ready", + "database": database, + } + PROJECTS[namespace]["pipeline_server"] = True + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "data_connection": data_connection, + "database": database, + }) + + +@mcp.tool() +def get_pipeline_server_status(namespace: str) -> str: + """Get the status of the pipeline server in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + ps = PIPELINE_SERVERS.get(namespace) + if not ps: + return json.dumps({"namespace": namespace, "configured": False}) + return json.dumps({ + "namespace": namespace, + "configured": ps["configured"], + "data_connection": ps["data_connection"], + "status": ps["status"], + "database": ps["database"], + }) + + +@mcp.tool() +def setup_pipeline_server( + namespace: str, + data_connection: str, + database: str = "MariaDB", +) -> str: + """Alias for configure_pipeline_server. Configure a pipeline server for a Data Science Project.""" + return configure_pipeline_server(namespace, data_connection, database) + + +@mcp.tool() +def get_pipeline_status(namespace: str) -> str: + """Alias for get_pipeline_server_status. Get the status of the pipeline server.""" + return get_pipeline_server_status(namespace) + + +# ── Serving runtime creation ──────────────────────────────────────────── + + +@mcp.tool() +def create_serving_runtime( + namespace: str, + name: str, + display_name: str = "", + model_formats: list = None, + container_image: str = "", + container_port: int = 8080, + multi_model: bool = False, + api_protocol: str = "REST", +) -> str: + """Create a custom ServingRuntime in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + if not model_formats: + raise ValueError("model_formats must specify at least one model format") + + runtime = { + "name": name, + "display_name": display_name or name, + "model_formats": model_formats, + "requires_instantiation": False, + "source": "custom", + "api_protocol": api_protocol, + "container_image": container_image, + "container_port": container_port, + "multi_model": multi_model, + } + SERVING_RUNTIMES.setdefault(namespace, []).append(runtime) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "model_formats": model_formats, + }) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/instruction.md b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/instruction.md new file mode 100644 index 00000000..03edef95 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/instruction.md @@ -0,0 +1,20 @@ +# Data Science Project Setup Task + +You are an AI engineer on Red Hat OpenShift AI. Your team needs a new project to host a fraud detection model, with access to an S3 model store and model serving enabled. + +## Requirements +- Create a new data science project named `fraud-detection` with a descriptive display name and appropriate labels +- Configure an S3-compatible data connection for model artifact storage using these details: + - Connection name: `model-store` + - Bucket: `fraud-models` + - Endpoint: `https://s3.us-east-1.amazonaws.com` + - Access key: `AKIA1234EXAMPLE` + - Secret key: `wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY` + - Region: `us-east-1` +- Enable model serving on the project and confirm which serving runtimes become available +- Configure the pipeline server using the data connection you created +- Verify the project is fully configured and ready for model deployment + +Document your setup steps, the final project configuration, and any issues encountered in `/root/report.md`. Redact any credentials in the report. + +Use MCP tools to interact with the platform. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/solution/solve.sh b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/solution/solve.sh new file mode 100644 index 00000000..91e7d226 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/solution/solve.sh @@ -0,0 +1,32 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Data Science Project Setup + +## Project: fraud-detection + +### Namespace Configuration +- Label: `opendatahub.io/dashboard: "true"` (required for RHOAI dashboard visibility) +- Model serving mode: `single` (one model per namespace) + +### S3 Data Connection +- Name: model-artifacts +- Bucket: fraud-detection-models +- Endpoint: https://s3.amazonaws.com +- Access key / Secret key configured (credentials REDACTED in display) +- Region: us-east-1 + +### Pipeline Server +- Data connection: model-artifacts (required for pipeline artifact storage) +- Pipeline server uses data connection for artifacts + +### Project Status (get_project_status) +| Component | Status | +|-----------|--------| +| Project | fraud-detection (created) | +| Data connections | 1 configured | +| Pipeline server | configured | +| Model serving | single mode enabled | + +### Validation +- list_data_science_projects: checked for duplicate project name before create +REPORT_EOF diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/task.toml b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/task.toml new file mode 100644 index 00000000..83c47b69 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-ai-engineer__ds-project-setup" +name = "rh-ai-engineer Data Science Project Setup Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-ai-engineer", "ds-project-setup", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/tests/llm_judge.py b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/tests/llm_judge.py new file mode 100644 index 00000000..2d6a34c6 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/tests/llm_judge.py @@ -0,0 +1,102 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "partial_credential_redaction", + "file": "/root/report.md", + "question": "Does the report display S3 credentials using partial redaction — showing the first few characters of the access key (like AKIA****) with the rest masked — rather than using placeholder values or full redaction?", + "reference": "A skilled report shows credentials partially redacted: the first few characters of the access key visible followed by asterisks (e.g., 'AKIA****'), and the secret key fully masked ('********'). Using PLACEHOLDER_ACCESS_KEY or completely hiding the access key is insufficient — partial redaction allows verification without exposing the full credential." + }, + { + "id": "secret_manifest_structure", + "file": "/root/report.md", + "question": "Does the report include a Kubernetes Secret manifest (with kind: Secret, apiVersion, metadata, and data fields) showing how the S3 data connection is stored as a K8s resource, rather than just describing the connection narratively?", + "reference": "A skilled report shows the actual K8s Secret YAML structure with kind: Secret, metadata (namespace, name, labels), and data fields containing base64-encoded values. An unskilled report describes the data connection configuration narratively without showing the underlying K8s resource structure." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/tests/test.sh b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/tests/test_outputs.py b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/tests/test_outputs.py new file mode 100644 index 00000000..8978be1d --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/tests/test_outputs.py @@ -0,0 +1,113 @@ +""" +Tests for rh-ai-engineer__ds-project-setup per-skill evaluation. +Baseline tests: any competent agent should pass. +Skill-dependent tests: based on empirical gaps between skilled and unskilled agent outputs. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ["project", "data science", "namespace"]), ( + "report should mention the project" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 200, "report should have substantial content" + + +class TestSkillDependent: + def test_data_connection_secret_keys(self): + """Skill teaches RHOAI data connections are stored as K8s Secrets with specific + key names: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_S3_BUCKET, + AWS_S3_ENDPOINT. Without skill, agents describe connections abstractly.""" + c = read_report() + aws_keys = ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_S3_BUCKET", + "AWS_S3_ENDPOINT", "AWS_DEFAULT_REGION"] + mentioned = sum(1 for k in aws_keys if k in c) + assert mentioned >= 2, ( + "should reference specific RHOAI data connection secret key names (AWS_*)" + ) + + def test_credential_partial_redaction(self): + """Skill teaches showing first 4 chars + **** for credentials (e.g., AKIA****). + Without skill, agents use PLACEHOLDER values or full redaction.""" + c = read_report() + has_partial = any(t in c for t in [ + "AKIA****", "AKIA*", "wJal****", "wJal*", + "1234****", "1234*", + ]) + has_stars_with_prefix = "****" in c and any(t in c for t in ["AKIA", "akia"]) + assert has_partial or has_stars_with_prefix, ( + "should use partial credential redaction (first chars visible + ****)" + ) + + def test_k8s_secret_yaml_manifest(self): + """Skill teaches showing the K8s Secret manifest structure for data connections. + Without skill, agents describe connections narratively without YAML.""" + c = read_report() + has_secret_kind = "kind: Secret" in c or "kind:Secret" in c + has_secret_ref = "Secret" in c and ("apiVersion" in c or "metadata" in c) + assert has_secret_kind or has_secret_ref, ( + "should include K8s Secret manifest structure for data connection" + ) + + def test_pipeline_server_with_data_connection(self): + """Skill teaches pipeline server requires a data connection (prerequisite chain). + Without skill, agents skip pipeline server or configure it generically.""" + c = read_report().lower() + has_pipeline = any(t in c for t in ["pipeline server", "pipeline"]) + has_linkage = any(t in c for t in [ + "data connection", "model-store", "artifact storage", + "s3 bucket", "data_connection", + ]) + pipeline_configured = "pipeline" in c and "configured" in c and "not configured" not in c + assert has_pipeline and (has_linkage or pipeline_configured), ( + "should configure pipeline server linked to a data connection" + ) + + def test_base64_secret_values(self): + """Skill teaches showing actual base64-encoded secret values in K8s + Secret YAML manifests. Without skill, agents show credentials in + plain text or fully redacted format.""" + c = read_report() + import re + has_base64 = bool(re.search(r'[A-Za-z0-9+/]{12,}={0,2}', c)) + has_opaque = "Opaque" in c + assert has_base64 or has_opaque, ( + "should include base64-encoded values or Opaque secret type in K8s manifest" + ) + + def test_model_serving_mode(self): + """Both agents should configure model serving — easy test.""" + c = read_report().lower() + assert any(t in c for t in [ + "single", "multi", "model serving", "serving mode", + ]), "should configure model serving mode" + + def test_runtime_selection_context(self): + """Docs teach decision context across runtimes: vLLM (PagedAttention), + NIM (TensorRT-LLM, no compilation), Caikit+TGIS (gRPC-only). + Without docs, agents don't provide runtime comparison context.""" + c = read_report().lower() + assert any(t in c for t in [ + "pagedattention", "paged attention", "tensorrt", "grpc", + "caikit", "vllm", "nim", + ]) and any(t in c for t in ["runtime", "serving", "comparison", "select"]), ( + "should compare runtimes with technical characteristics" + ) diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/Dockerfile b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/Dockerfile new file mode 100644 index 00000000..aac4c84e --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/Dockerfile @@ -0,0 +1,67 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "rhoai": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-rhoai-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/common-issues.md b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/examples/model-deploy.md b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/examples/model-deploy.md new file mode 100644 index 00000000..dcd79e44 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/examples/model-deploy.md @@ -0,0 +1,160 @@ +# /model-deploy Examples + +Example interactions demonstrating the model-deploy skill workflow. + +## Example 1: Deploy Llama 3.1 8B with vLLM + +``` +User: "Deploy Llama 3.1 8B on my cluster" + +Agent: [Invokes /model-deploy skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand runtime capabilities. + +Step 1 of 10: Gathering deployment information. + +| Setting | Value | Source | +|---------|-------|--------| +| Model | Llama 3.1 8B Instruct | user input | +| Runtime | TBD | auto-detect | +| Namespace | ? | needs input | +| Model Source | ? | needs input | +| Deployment Mode | Serverless | default | + +What namespace should I deploy to, and where are the model weights stored? +(e.g., S3 bucket, OCI registry, PVC) + +User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" + +Agent: Step 2 of 10: Determining runtime. + +I consulted supported-runtimes.md to understand runtime capabilities. + +Based on Llama 3.1 8B (open-source HuggingFace model): + +Recommended runtime: vLLM +Reason: Default runtime for open-source LLMs. Broad compatibility, + OpenAI-compatible API, no additional setup required. + +Proceed with vLLM? (yes / no / suggest alternative) + +User: "yes" + +Agent: Step 3 of 10: Looking up hardware profile. + +I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. + +| Requirement | Value | +|-------------|-------| +| GPUs | 1x A10G/L4/A100/H100 | +| VRAM | 16GB | +| Key Args | --max-model-len=4096 --tool-call-parser hermes | + +Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). +Ensure your cluster has at least 1 GPU with 16GB VRAM. + +Step 5 of 10: Skipped (NIM platform check not needed for vLLM). + +Step 6-7 of 10: Generating InferenceService YAML. + +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: llama-3-1-8b + namespace: ai-models + annotations: + serving.kserve.io/deploymentMode: Serverless +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: vllm-runtime + storageUri: s3://my-bucket/llama-3.1-8b + resources: + limits: + nvidia.com/gpu: "1" + requests: + cpu: "4" + memory: "16Gi" + args: + - --max-model-len=4096 + - --tool-call-parser=hermes + +| Setting | Value | +|---------|-------| +| Model | llama-3-1-8b | +| Runtime | vllm-runtime | +| GPUs | 1 | +| Namespace | ai-models | +| Mode | Serverless | + +Proceed with creating this InferenceService? (yes/no/modify) + +User: "yes" + +Agent: Step 8 of 10: Creating InferenceService... + [Creates via resources_create_or_update] + + InferenceService created. Monitoring rollout... + +Step 9 of 10: Monitoring rollout. + +| Stage | Status | +|-------|--------| +| Pod Scheduled | Complete | +| Image Pulled | Complete | +| Container Started | Complete | +| Model Loaded | Complete | +| Ready | Complete | + +Step 10 of 10: Deployment complete! + +Model Deployment Successful + +Model: llama-3-1-8b +Runtime: vLLM +Namespace: ai-models +GPUs: 1 + +Inference Endpoint: + URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com + API: OpenAI-compatible REST + +Quick Test: + curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' + +Next Steps: + Monitor performance: "/ai-observability llama-3-1-8b" +``` + +## Example 2: Deploy a Model with NIM (NIM Not Set Up) + +``` +User: "Deploy Llama 3.1 70B with NIM" + +Agent: [Invokes /model-deploy skill] + ... + Step 2: User requested NIM runtime. + + Step 5: Checking NIM platform... + +NIM platform is not set up in namespace ai-models. + +The NIM runtime requires a one-time platform setup. Would you like to: +1. Run /nim-setup now to configure the NIM platform +2. Switch to vLLM runtime instead (no setup required) +3. Abort deployment + +User: "1" + +Agent: [Invokes /nim-setup skill — walks through NGC credential setup] + ... + NIM Platform Setup Complete. + + Resuming model deployment... + [Continues from Step 6 with NIM runtime] +``` diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/examples/nim-setup.md b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/examples/nim-setup.md new file mode 100644 index 00000000..ccfc1a5d --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/examples/nim-setup.md @@ -0,0 +1,115 @@ +# /nim-setup Examples + +Example interactions demonstrating the nim-setup skill workflow. + +## Example 1: First-Time NIM Setup + +``` +User: "Set up NIM on my cluster" + +Agent: [Invokes /nim-setup skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand NIM platform requirements. + +Let me verify the prerequisites for NIM setup. + +Step 1 of 7: Checking GPU Operator and NFD Operator... + +GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) +NFD Operator: nfd.v4.19.0 (Succeeded) + +Both operators are installed and healthy. + +Step 2 of 7: I need your NGC credentials. + +To set up NIM, I need your NVIDIA NGC API key. +You can generate one at: https://ngc.nvidia.com/setup/api-key + +Please provide: +1. Your NGC API key +2. The target namespace for NIM resources + +User: "Here's my key: [user provides NGC API key], namespace: ai-models" + +Agent: Step 3 of 7: Creating NGC image pull secret... + +I'll create a docker-registry secret for pulling NIM images from nvcr.io: + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-image-pull-secret + namespace: ai-models + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] + +Should I create this image pull secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + Image pull secret created successfully. + +Step 4 of 7: Creating NGC API key secret... + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-api-key + namespace: ai-models + type: Opaque + stringData: + NGC_API_KEY: "[REDACTED]" + +Should I create this API key secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + API key secret created successfully. + +Step 5 of 7: Creating NIM Account CR... + + apiVersion: nim.opendatahub.io/v1 + kind: Account + metadata: + name: nim-account + namespace: ai-models + spec: + apiKeySecret: + name: ngc-api-key + imagePullSecret: + name: ngc-image-pull-secret + +Should I create this NIM Account CR in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates Account CR via resources_create_or_update] + Account CR created successfully. + +Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) + +User: "no" + +Agent: Step 7 of 7: Validating NIM platform readiness... + +NIM Platform Setup Complete + +Account CR: `nim-account` (namespace: ai-models) + Status: Ready + +NGC Credentials: + Image Pull Secret: ngc-image-pull-secret [created] + API Key Secret: ngc-api-key [created] + +NIM ServingRuntimes Available: + - nim-llama-3-1-8b-instruct + - nim-llama-3-1-70b-instruct + - nim-mistral-7b-instruct + +Next Steps: + To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" +``` diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/live-doc-lookup.md b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/references/known-model-profiles.md b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/references/known-model-profiles.md new file mode 100644 index 00000000..40148981 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/references/known-model-profiles.md @@ -0,0 +1,83 @@ +--- +title: Known Model Hardware Profiles +category: references +tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] +semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] +use_cases: [model-deploy, debug-inference, ai-observability] +last_updated: 2026-03-10 +--- + +# Known Model Hardware Profiles + +Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). + +**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. + +## Model Source Conventions + +Each model lists a recommended `storageUri` with its authentication requirements: +- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** +- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). +- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. + +When the user does not specify a model source, use the `hf://` URI listed in the profile below. + +## Llama 3.x (Meta) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | +| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | +| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | +| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | + +- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) +- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly + +## Granite 3.x (IBM/Red Hat) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | +| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | + +- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) +- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser granite --chat-template` +- Red Hat-supported model family on RHOAI + +## Mixtral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | +| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) +- Mixture-of-Experts architecture: only ~13B/45B parameters active per token + +## Mistral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | +| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) + +## When a Model Is Not Listed + +If the requested model is not in this file, the agent MUST use the live doc lookup protocol: + +1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure +2. Fetch hardware specs from the appropriate source +3. Report findings to the user before proceeding with deployment + +Common cases requiring live lookup: +- Newly released models (after this file's last update) +- Domain-specific fine-tuned models +- Models with custom quantization +- NIM-specific optimized profiles diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/references/supported-runtimes.md b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/references/supported-runtimes.md new file mode 100644 index 00000000..9d018651 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/references/supported-runtimes.md @@ -0,0 +1,104 @@ +--- +title: Supported Serving Runtimes +category: references +tags: [runtimes, vllm, nim, caikit, tgis, serving] +semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] +use_cases: [model-deploy, serving-runtime-config, nim-setup] +last_updated: 2026-02-26 +--- + +# Supported Serving Runtimes + +This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. + +## Runtime Comparison + +| Runtime | API Type | Model Format | GPU Support | License | Setup Required | +|---------|----------|-------------|-------------|---------|----------------| +| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | +| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | +| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | +| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | + +## vLLM + +**Default runtime for most open-source models.** + +- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) +- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized +- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) +- **Model source**: S3-compatible storage, OCI registry, PVC, URI +- **Key features**: + - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) + - PagedAttention for efficient memory management + - Tensor parallelism for multi-GPU inference + - Tool/function calling support (`--tool-call-parser`) + - Continuous batching for high throughput +- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name + +## NVIDIA NIM + +**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** + +- **API**: OpenAI-compatible REST +- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog +- **GPU support**: NVIDIA only (requires specific GPU models per model profile) +- **Model source**: NGC model catalog (pulled automatically via NGC credentials) +- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) +- **Key features**: + - TensorRT-LLM optimization for lower latency + - Pre-compiled model engines (no compilation on first load) + - Optimized and generic profiles per GPU type + - Automatic model download from NGC +- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. +- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM +- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. +- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state + +## Caikit+TGIS + +**Red Hat's Caikit format with Text Generation Inference Server.** + +- **API**: gRPC (not REST) +- **Model formats**: Caikit format (requires conversion from HuggingFace) +- **GPU support**: NVIDIA +- **Model source**: S3-compatible storage +- **Key features**: + - Red Hat-supported runtime + - gRPC API for streaming inference + - Integrated with RHOAI model serving platform +- **When to choose**: When the model is already in Caikit format or when gRPC API is required. +- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name + +## Custom Runtimes + +**User-provided ServingRuntime CRs for unsupported frameworks.** + +- **API**: Defined by the custom runtime +- **Model formats**: Defined by the custom runtime +- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. +- **How to create**: Use `/serving-runtime-config` skill +- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility + +## Runtime Selection Decision Tree + +``` +Is the user's preferred runtime explicitly stated? +├── Yes → Use that runtime +└── No → Continue + +Is the model available in the NGC NIM catalog? +├── Yes → Suggest NIM (with vLLM as fallback) +│ Note: Requires /nim-setup and NVIDIA GPUs +└── No → Continue + +Is the model in Caikit format? +├── Yes → Caikit+TGIS +└── No → Continue + +Is the model a standard open-source LLM (HuggingFace-compatible)? +├── Yes → vLLM (default) +└── No → Custom runtime via /serving-runtime-config +``` diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/skill-conventions.md b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..e7a4d11c --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +"""Mock OpenShift MCP server for SkillsBench rh-ai-engineer task. + +Simulates Kubernetes resource CRUD, pod management, logs, and events. + +Key scenario elements: +- LimitRange in namespaces: min CPU=100m, min memory=128Mi + (conflicts with KServe sidecar containers hardcoded at 10m CPU/15Mi memory) +- GPU node with custom taint ai-workload=true:NoSchedule +- NIM Account CR in ml-production: not ready (NGC credentials invalid) +- text-gen-legacy pods: OOMKilled (max-model-len=32768 on A10G) +- nim-llama-prod: no pods created (Account CR not ready) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + +# ── Cluster state ──────────────────────────────────────────────────────── + +GPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "gpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + "nvidia.com/gpu.present": "true", + "nvidia.com/gpu.product": "NVIDIA-A10G", + }, + }, + "spec": { + "taints": [ + { + "key": "ai-workload", + "value": "true", + "effect": "NoSchedule", + }, + ], + }, + "status": { + "allocatable": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "capacity": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "conditions": [ + {"type": "Ready", "status": "True"}, + ], + }, +} + +CPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "cpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + }, + }, + "spec": {"taints": []}, + "status": { + "allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "capacity": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +MASTER_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "master-1", + "labels": { + "node-role.kubernetes.io/master": "", + "node-role.kubernetes.io/control-plane": "", + }, + }, + "spec": { + "taints": [ + {"key": "node-role.kubernetes.io/master", "effect": "NoSchedule"}, + ], + }, + "status": { + "allocatable": {"cpu": "8", "memory": "32Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +ALL_NODES = [GPU_NODE, CPU_NODE, MASTER_NODE] + +# LimitRange applied by cluster policy to all DS project namespaces +NAMESPACE_LIMITRANGE = { + "apiVersion": "v1", + "kind": "LimitRange", + "metadata": { + "name": "default-limits", + }, + "spec": { + "limits": [ + { + "type": "Container", + "default": { + "cpu": "2", + "memory": "4Gi", + }, + "defaultRequest": { + "cpu": "500m", + "memory": "256Mi", + }, + "min": { + "cpu": "100m", + "memory": "128Mi", + }, + "max": { + "cpu": "32", + "memory": "128Gi", + }, + }, + ], + }, +} + +NIM_ACCOUNT_CR = { + "apiVersion": "nim.opendatahub.io/v1", + "kind": "Account", + "metadata": { + "name": "nim-account", + "namespace": "ml-production", + }, + "spec": { + "apiKeySecret": { + "name": "ngc-api-key", + }, + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "NGCCredentialsInvalid", + "message": "NGC API key validation failed: 401 Unauthorized. " + "The API key in secret 'ngc-api-key' is expired or invalid. " + "Re-create the secret with a valid NGC API key from " + "https://ngc.nvidia.com/setup/api-key and restart the " + "Account reconciliation.", + "lastTransitionTime": "2026-03-14T12:00:00Z", + }, + ], + "nimPullSecretStatus": "Failed", + "nimConfigStatus": "Pending", + }, +} + +SERVING_RUNTIME_VLLM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "vllm-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "vLLM", "version": "1", "autoSelect": True}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "quay.io/modh/vllm:rhoai-2.16", + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + }, + ], + }, +} + +SERVING_RUNTIME_NIM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "nim-serving-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "NIM", "version": "1"}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "nvcr.io/nim/meta/llama-3.1-8b-instruct:latest", + "ports": [{"containerPort": 8000, "protocol": "TCP"}], + "env": [ + {"name": "NGC_API_KEY", "valueFrom": { + "secretKeyRef": {"name": "ngc-api-key", "key": "api_key"}, + }}, + ], + }, + ], + }, +} + +PODS_BY_NAMESPACE = { + "ml-production": [ + { + "name": "text-gen-legacy-predictor-00001-abc12", + "namespace": "ml-production", + "status": "CrashLoopBackOff", + "restarts": 5, + "node": "gpu-worker-1", + "containers": [ + { + "name": "kserve-container", + "state": "waiting", + "reason": "CrashLoopBackOff", + "last_termination_reason": "OOMKilled", + "last_termination_exit_code": 137, + }, + ], + "labels": { + "serving.kserve.io/inferenceservice": "text-gen-legacy", + }, + "gpu": "1", + }, + # nim-llama-prod: NO pods created (Account CR not ready) + ], +} + +POD_LOGS = { + "text-gen-legacy-predictor-00001-abc12": ( + "INFO 2026-03-01 10:00:00 vllm_engine.py:125] vLLM engine starting...\n" + "INFO 2026-03-01 10:00:01 config.py:89] Model: mistralai/Mistral-7B-Instruct-v0.3\n" + "INFO 2026-03-01 10:00:01 config.py:92] max_model_len = 32768\n" + "INFO 2026-03-01 10:00:02 gpu_executor.py:45] GPU 0: NVIDIA A10G (24576 MiB)\n" + "INFO 2026-03-01 10:00:03 model_runner.py:88] Loading model weights...\n" + "INFO 2026-03-01 10:00:15 model_runner.py:112] Model weights loaded: 13.5 GiB\n" + "INFO 2026-03-01 10:00:15 worker.py:201] Allocating KV cache...\n" + "ERROR 2026-03-01 10:00:16 worker.py:215] torch.cuda.OutOfMemoryError: " + "CUDA out of memory. Tried to allocate 28.5 GiB for KV cache but only " + "10.1 GiB available after loading model weights (13.5 GiB).\n" + "ERROR 2026-03-01 10:00:16 vllm_engine.py:178] Engine failed to start\n" + "Traceback (most recent call last):\n" + " File \"/opt/vllm/vllm/engine/engine.py\", line 175, in start\n" + " self._init_kv_cache()\n" + " File \"/opt/vllm/vllm/worker/worker.py\", line 215, in _init_kv_cache\n" + " raise torch.cuda.OutOfMemoryError(msg)\n" + "torch.cuda.OutOfMemoryError: CUDA out of memory\n" + ), +} + +EVENTS_BY_NAMESPACE = { + "ml-production": [ + { + "type": "Warning", + "reason": "BackOff", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Back-off restarting failed container kserve-container in pod " + "text-gen-legacy-predictor-00001-abc12", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Warning", + "reason": "OOMKilled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Container kserve-container was OOMKilled (exit code 137). " + "GPU memory exhausted during KV cache allocation.", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Normal", + "reason": "Scheduled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Successfully assigned ml-production/" + "text-gen-legacy-predictor-00001-abc12 to gpu-worker-1", + "count": 1, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-02-28T08:00:00Z", + }, + { + "type": "Warning", + "reason": "NIMAccountNotReady", + "object": "InferenceService/nim-llama-prod", + "message": "NIM Account 'nim-account' in namespace 'ml-production' " + "is not ready", + "count": 12, + "first_timestamp": "2026-03-14T12:00:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + { + "type": "Warning", + "reason": "ImagePullBackOff", + "object": "InferenceService/nim-llama-prod", + "message": "Failed to pull image 'nvcr.io/nim/meta/llama-3.1-8b-instruct:" + "latest': unauthorized: authentication required", + "count": 8, + "first_timestamp": "2026-03-14T12:05:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + ], +} + + +# ── Resource tools ─────────────────────────────────────────────────────── + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: str = "", +) -> str: + """Get a single Kubernetes resource by apiVersion, kind, and name.""" + if kind == "Node": + for node in ALL_NODES: + if node["metadata"]["name"] == name: + return json.dumps(node, indent=2) + raise ValueError(f"Node '{name}' not found") + + if kind == "ServingRuntime": + if name == "vllm-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_VLLM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + if name == "nim-serving-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_NIM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + raise ValueError(f"ServingRuntime '{name}' not found in namespace '{namespace}'") + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps(lr, indent=2) + + if kind == "Account" and "nim" in apiVersion.lower(): + if namespace == "ml-production" and name == "nim-account": + return json.dumps(NIM_ACCOUNT_CR, indent=2) + raise ValueError( + f"Account '{name}' not found in namespace '{namespace}'" + ) + + if kind == "ClusterVersion" and apiVersion == "config.openshift.io/v1": + return json.dumps({ + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterVersion", + "metadata": {"name": "version"}, + "status": {"desired": {"version": "4.16.3"}}, + }) + + raise ValueError(f"Resource {apiVersion}/{kind}/{name} not found") + + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: str = "", + labelSelector: str = "", +) -> str: + """List Kubernetes resources by apiVersion and kind.""" + if kind == "Node": + nodes = ALL_NODES + if labelSelector: + parts = labelSelector.split("=", 1) + key = parts[0] + value = parts[1] if len(parts) > 1 else "" + nodes = [ + n for n in nodes + if n["metadata"]["labels"].get(key) == value + ] + return json.dumps(nodes, indent=2) + + if kind == "Service" and apiVersion == "serving.knative.dev/v1": + return json.dumps({ + "kind": "ServiceList", + "apiVersion": "serving.knative.dev/v1", + "items": [], + "metadata": {}, + }) + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps({ + "kind": "LimitRangeList", + "items": [lr], + }) + + if kind == "InferenceService": + return json.dumps({ + "kind": "InferenceServiceList", + "items": [], + }) + + raise ValueError(f"Unsupported list: {apiVersion}/{kind}") + + +@mcp.tool() +def pods_list( + namespace: str, + labelSelector: str = "", +) -> str: + """List pods in a namespace with optional label selector.""" + pods = PODS_BY_NAMESPACE.get(namespace, []) + + if labelSelector: + key, _, value = labelSelector.partition("=") + pods = [p for p in pods if p.get("labels", {}).get(key) == value] + + results = [] + for pod in pods: + results.append({ + "name": pod["name"], + "namespace": pod["namespace"], + "status": pod["status"], + "restarts": pod.get("restarts", 0), + "node": pod.get("node", ""), + "containers": pod.get("containers", []), + "gpu": pod.get("gpu", "0"), + }) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def pods_log( + namespace: str, + name: str, + container: str = "", +) -> str: + """Get logs from a pod container.""" + logs = POD_LOGS.get(name) + if logs is None: + raise ValueError(f"Pod '{name}' not found in namespace '{namespace}'") + return logs + + +@mcp.tool() +def events_list(namespace: str) -> str: + """List events in a namespace.""" + events = EVENTS_BY_NAMESPACE.get(namespace, []) + return json.dumps(events, indent=2) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/mcp-servers/mock-rhoai-mcp.py b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/mcp-servers/mock-rhoai-mcp.py new file mode 100644 index 00000000..0ae9e4cb --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/mcp-servers/mock-rhoai-mcp.py @@ -0,0 +1,780 @@ +#!/usr/bin/env python3 +"""Mock RHOAI MCP server for SkillsBench rh-ai-engineer task. + +Simulates Red Hat OpenShift AI operations: Data Science Projects, +model serving, data connections, serving runtimes, inference services. + +Scenario: +- ml-production: existing project with two broken deployments + - text-gen-legacy: vLLM OOMKilled (max-model-len=32768 on A10G) + - nim-llama-prod: NIM failing (Account CR not ready, NGC creds invalid) +- fraud-detection: does not exist yet (agent creates it) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("rhoai") + +# ── In-memory state ────────────────────────────────────────────────────── + +PROJECTS = { + "ml-production": { + "name": "ml-production", + "display_name": "ML Production", + "description": "Production ML workloads", + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": "single", + "pipeline_server": True, + }, +} + +DATA_CONNECTIONS = { + "ml-production": [ + { + "name": "prod-model-store", + "type": "S3", + "bucket": "ml-models-prod", + "endpoint": "https://s3.us-east-1.amazonaws.com", + "region": "us-east-1", + }, + ], +} + +SERVING_RUNTIMES = { + "__platform_templates__": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "REST", + "supported_model_formats": [ + {"name": "vLLM", "version": "1", "autoSelect": True} + ], + }, + { + "name": "caikit-tgis-runtime", + "display_name": "Caikit+TGIS ServingRuntime", + "model_formats": ["caikit"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "gRPC", + }, + ], + "ml-production": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + { + "name": "nim-serving-runtime", + "display_name": "NVIDIA NIM ServingRuntime", + "model_formats": ["NIM"], + "requires_instantiation": False, + "source": "nim-account", + "api_protocol": "REST", + }, + { + "name": "ovms-1", + "display_name": "OpenVINO Model Server", + "model_formats": ["openvino_ir", "onnx"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + ], +} + +INFERENCE_SERVICES = { + "ml-production": { + "text-gen-legacy": { + "name": "text-gen-legacy", + "namespace": "ml-production", + "runtime": "vllm-runtime", + "model_format": "vLLM", + "storage_uri": "hf://mistralai/Mistral-7B-Instruct-v0.3", + "display_name": "Mistral 7B Legacy", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "16Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "PredictorFailed", + "message": "Predictor pod is not ready", + }, + { + "type": "PredictorReady", + "status": "False", + "reason": "ContainerCrashLoop", + "message": "Container kserve-container terminated: " + "OOMKilled (exit code 137). 5 restarts.", + }, + { + "type": "IngressReady", + "status": "True", + "reason": "IngressReady", + "message": "Ingress is ready", + }, + ], + "age": "3d", + }, + "nim-llama-prod": { + "name": "nim-llama-prod", + "namespace": "ml-production", + "runtime": "nim-serving-runtime", + "model_format": "NIM", + "storage_uri": "nim://meta/llama-3.1-8b-instruct", + "display_name": "Llama 3.1 8B (NIM)", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "32Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "RuntimeNotReady", + "message": "ServingRuntime 'nim-serving-runtime' " + "is not in ready state", + }, + { + "type": "PredictorReady", + "status": "Unknown", + "reason": "PodNotCreated", + "message": "Predictor pod has not been created. " + "Waiting for ServingRuntime to become ready.", + }, + { + "type": "IngressReady", + "status": "Unknown", + "reason": "PredictorNotReady", + "message": "Waiting for predictor to become ready", + }, + ], + "age": "1d", + }, + }, +} + +DEPLOYED_MODELS = {} + +WORKBENCHES = { + "ml-production": [ + { + "name": "data-exploration-nb", + "display_name": "Data Exploration", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Running", + "cpu_request": "1", + "memory_request": "8Gi", + "gpu_count": 0, + "pvc_name": "data-exploration-nb-pvc", + "pvc_size": "20Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-10T09:00:00Z", + }, + { + "name": "model-training-nb", + "display_name": "Model Training", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Stopped", + "cpu_request": "4", + "memory_request": "16Gi", + "gpu_count": 1, + "pvc_name": "model-training-nb-pvc", + "pvc_size": "50Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-15T14:00:00Z", + }, + ], +} + +PIPELINE_SERVERS = { + "ml-production": { + "configured": True, + "data_connection": "prod-model-store", + "status": "Ready", + "database": "MariaDB", + }, +} + +NOTEBOOK_IMAGES = [ + {"name": "jupyter-pytorch-ubi9-python-3.9-2024.1", "display_name": "PyTorch 2024.1", "packages": ["torch", "transformers"]}, + {"name": "jupyter-tensorflow-ubi9-python-3.9-2024.1", "display_name": "TensorFlow 2024.1", "packages": ["tensorflow"]}, + {"name": "jupyter-datascience-ubi9-python-3.9-2024.1", "display_name": "Standard Data Science", "packages": ["pandas", "scikit-learn"]}, + {"name": "jupyter-minimal-ubi9-python-3.9-2024.1", "display_name": "Minimal Python", "packages": []}, +] + + +# ── Project tools ──────────────────────────────────────────────────────── + + +@mcp.tool() +def list_data_science_projects() -> str: + """List all RHOAI Data Science Projects on the cluster.""" + projects = [] + for name, proj in PROJECTS.items(): + projects.append({ + "name": name, + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + }) + return json.dumps(projects, indent=2) + + +@mcp.tool() +def create_data_science_project( + name: str, + display_name: str, + description: str = "", +) -> str: + """Create a new RHOAI Data Science Project (namespace with dashboard labels).""" + if name in PROJECTS: + raise ValueError( + f"Project '{name}' already exists. Choose a different name " + "or configure the existing project." + ) + if not name.replace("-", "").replace("_", "").isalnum() or len(name) > 63: + raise ValueError( + f"Invalid project name '{name}'. Must be DNS-compatible: " + "lowercase alphanumeric and hyphens, max 63 chars." + ) + + PROJECTS[name] = { + "name": name, + "display_name": display_name, + "description": description, + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": None, + "pipeline_server": False, + } + DATA_CONNECTIONS[name] = [] + SERVING_RUNTIMES[name] = [] + INFERENCE_SERVICES[name] = {} + + return json.dumps({ + "status": "created", + "name": name, + "display_name": display_name, + "namespace": name, + "labels": {"opendatahub.io/dashboard": "true"}, + }) + + +@mcp.tool() +def get_project_details(name: str) -> str: + """Get detailed information about an RHOAI Data Science Project.""" + if name not in PROJECTS: + raise ValueError(f"Project '{name}' not found") + proj = PROJECTS[name] + dc_count = len(DATA_CONNECTIONS.get(name, [])) + isvc_count = len(INFERENCE_SERVICES.get(name, {})) + return json.dumps({ + "name": proj["name"], + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "labels": proj["labels"], + "data_connections": dc_count, + "inference_services": isvc_count, + "model_serving_mode": proj.get("model_serving_mode"), + "pipeline_server": proj.get("pipeline_server", False), + }) + + +@mcp.tool() +def get_project_status(namespace: str) -> str: + """Get comprehensive status of an RHOAI Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Project '{namespace}' not found") + proj = PROJECTS[namespace] + dcs = DATA_CONNECTIONS.get(namespace, []) + isvcs = INFERENCE_SERVICES.get(namespace, {}) + return json.dumps({ + "namespace": namespace, + "display_name": proj["display_name"], + "status": "Active", + "components": { + "data_connections": len(dcs), + "inference_services": len(isvcs), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + "pipeline_server": "configured" if proj.get("pipeline_server") else "not configured", + }, + }) + + +# ── Data connection tools ──────────────────────────────────────────────── + + +@mcp.tool() +def create_s3_data_connection( + namespace: str, + name: str, + bucket: str, + endpoint: str, + access_key: str, + secret_key: str, + region: str = "", +) -> str: + """Create an S3-compatible data connection in an RHOAI project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + existing = DATA_CONNECTIONS.get(namespace, []) + if any(dc["name"] == name for dc in existing): + raise ValueError( + f"Data connection '{name}' already exists in namespace '{namespace}'" + ) + + dc = { + "name": name, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + "region": region, + } + DATA_CONNECTIONS.setdefault(namespace, []).append(dc) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + }) + + +@mcp.tool() +def list_data_connections(namespace: str) -> str: + """List data connections in an RHOAI project namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + return json.dumps(dcs, indent=2) + + +# ── Model serving tools ───────────────────────────────────────────────── + + +@mcp.tool() +def set_model_serving_mode(namespace: str, mode: str) -> str: + """Enable model serving on a Data Science Project (single or multi mode).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + if mode not in ("single", "multi"): + raise ValueError(f"Invalid mode '{mode}'. Must be 'single' or 'multi'.") + + PROJECTS[namespace]["model_serving_mode"] = mode + + if not SERVING_RUNTIMES.get(namespace): + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + SERVING_RUNTIMES[namespace] = [ + {**t, "requires_instantiation": False, "source": "existing"} + for t in templates + ] + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "mode": mode, + }) + + +@mcp.tool() +def list_serving_runtimes( + namespace: str, + include_templates: bool = False, +) -> str: + """List available ServingRuntimes in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + runtimes = list(SERVING_RUNTIMES.get(namespace, [])) + if include_templates: + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + existing_names = {r["name"] for r in runtimes} + for t in templates: + if t["name"] not in existing_names: + runtimes.append(t) + + return json.dumps(runtimes, indent=2) + + +# ── Inference service tools ────────────────────────────────────────────── + + +@mcp.tool() +def deploy_model( + name: str, + namespace: str, + runtime: str, + model_format: str, + storage_uri: str, + display_name: str = "", + min_replicas: int = 1, + max_replicas: int = 1, + cpu_request: str = "1", + cpu_limit: str = "2", + memory_request: str = "4Gi", + memory_limit: str = "8Gi", + gpu_count: int = 0, +) -> str: + """Deploy an AI/ML model as a KServe InferenceService.""" + if namespace not in PROJECTS: + raise ValueError( + f"Namespace '{namespace}' is not a Data Science Project. " + "Create one via create_data_science_project first." + ) + + ns_runtimes = SERVING_RUNTIMES.get(namespace, []) + runtime_names = [r["name"] for r in ns_runtimes] + if runtime not in runtime_names: + available = ", ".join(runtime_names) or "none" + raise ValueError( + f"ServingRuntime '{runtime}' not found in namespace '{namespace}'. " + f"Available runtimes: {available}" + ) + + endpoint = f"https://{name}-{namespace}.apps.ocp-cluster.example.com" + isvc = { + "name": name, + "namespace": namespace, + "runtime": runtime, + "model_format": model_format, + "storage_uri": storage_uri, + "display_name": display_name or name, + "gpu_count": gpu_count, + "cpu_request": cpu_request, + "memory_request": memory_request, + "min_replicas": min_replicas, + "max_replicas": max_replicas, + "ready": True, + "url": endpoint, + "conditions": [ + {"type": "Ready", "status": "True", "reason": "Ready", "message": ""}, + {"type": "PredictorReady", "status": "True", "reason": "PodReady", "message": ""}, + {"type": "IngressReady", "status": "True", "reason": "IngressReady", "message": ""}, + ], + "age": "0s", + } + + INFERENCE_SERVICES.setdefault(namespace, {})[name] = isvc + DEPLOYED_MODELS[f"{namespace}/{name}"] = isvc + + return json.dumps({ + "status": "deployed", + "name": name, + "namespace": namespace, + "runtime": runtime, + "endpoint": endpoint, + "ready": True, + }) + + +@mcp.tool() +def list_inference_services( + namespace: str, + verbosity: str = "standard", +) -> str: + """List deployed InferenceServices in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + isvcs = INFERENCE_SERVICES.get(namespace, {}) + results = [] + for isvc_name, isvc in isvcs.items(): + entry = { + "name": isvc["name"], + "runtime": isvc["runtime"], + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "age": isvc.get("age", ""), + } + if verbosity == "full": + entry["conditions"] = isvc.get("conditions", []) + entry["storage_uri"] = isvc.get("storage_uri", "") + entry["gpu_count"] = isvc.get("gpu_count", 0) + results.append(entry) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def get_inference_service( + name: str, + namespace: str, + verbosity: str = "standard", +) -> str: + """Get detailed status of a specific InferenceService.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + + isvc = isvcs[name] + result = { + "name": isvc["name"], + "namespace": isvc["namespace"], + "runtime": isvc["runtime"], + "model_format": isvc.get("model_format", ""), + "storage_uri": isvc.get("storage_uri", ""), + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "conditions": isvc.get("conditions", []), + "gpu_count": isvc.get("gpu_count", 0), + "replicas": {"min": isvc.get("min_replicas", 1), "max": isvc.get("max_replicas", 1)}, + "resources": { + "cpu_request": isvc.get("cpu_request", "1"), + "memory_request": isvc.get("memory_request", "4Gi"), + "memory_limit": isvc.get("memory_limit", "8Gi"), + }, + "age": isvc.get("age", ""), + } + return json.dumps(result, indent=2) + + +@mcp.tool() +def get_model_endpoint(name: str, namespace: str) -> str: + """Get the inference endpoint URL for a deployed model.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + isvc = isvcs[name] + if not isvc["ready"]: + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": "", + "error": "InferenceService is not ready. Check conditions for details.", + }) + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": isvc["url"], + }) + + +# ── Workbench tools ────────────────────────────────────────────────────── + + +@mcp.tool() +def list_workbenches(namespace: str) -> str: + """List workbenches (Jupyter notebooks) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + return json.dumps(wbs, indent=2) + + +@mcp.tool() +def create_workbench( + namespace: str, + name: str, + display_name: str = "", + image: str = "jupyter-datascience-ubi9-python-3.9-2024.1", + cpu_request: str = "1", + memory_request: str = "4Gi", + gpu_count: int = 0, + pvc_size: str = "20Gi", +) -> str: + """Create a new workbench (Jupyter notebook) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + valid_images = [img["name"] for img in NOTEBOOK_IMAGES] + if image not in valid_images: + raise ValueError( + f"Image '{image}' not found. Available: {', '.join(valid_images)}" + ) + + wb = { + "name": name, + "display_name": display_name or name, + "image": image, + "status": "Running", + "cpu_request": cpu_request, + "memory_request": memory_request, + "gpu_count": gpu_count, + "pvc_name": f"{name}-pvc", + "pvc_size": pvc_size, + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-03-02T12:00:00Z", + } + WORKBENCHES.setdefault(namespace, []).append(wb) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "image": image, + "pvc": f"{name}-pvc", + }) + + +@mcp.tool() +def stop_workbench(namespace: str, name: str) -> str: + """Stop a running workbench (preserves data).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Stopped" + return json.dumps({"status": "stopped", "name": name, "namespace": namespace}) + + +@mcp.tool() +def start_workbench(namespace: str, name: str) -> str: + """Start a stopped workbench.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Running" + return json.dumps({"status": "running", "name": name, "namespace": namespace}) + + +@mcp.tool() +def delete_workbench(namespace: str, name: str) -> str: + """Delete a workbench. WARNING: PVC data may be lost if not backed up.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wbs.remove(wb) + return json.dumps({ + "status": "deleted", + "name": name, + "namespace": namespace, + "warning": "Associated PVC data has been deleted", + }) + + +@mcp.tool() +def list_notebook_images() -> str: + """List available notebook images for workbench creation.""" + return json.dumps(NOTEBOOK_IMAGES, indent=2) + + +# ── Pipeline server tools ─────────────────────────────────────────────── + + +@mcp.tool() +def configure_pipeline_server( + namespace: str, + data_connection: str, + database: str = "MariaDB", +) -> str: + """Configure a pipeline server for a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + if not any(dc["name"] == data_connection for dc in dcs): + available = [dc["name"] for dc in dcs] + raise ValueError( + f"Data connection '{data_connection}' not found. Available: {available}" + ) + + PIPELINE_SERVERS[namespace] = { + "configured": True, + "data_connection": data_connection, + "status": "Ready", + "database": database, + } + PROJECTS[namespace]["pipeline_server"] = True + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "data_connection": data_connection, + "database": database, + }) + + +@mcp.tool() +def get_pipeline_server_status(namespace: str) -> str: + """Get the status of the pipeline server in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + ps = PIPELINE_SERVERS.get(namespace) + if not ps: + return json.dumps({"namespace": namespace, "configured": False}) + return json.dumps({ + "namespace": namespace, + "configured": ps["configured"], + "data_connection": ps["data_connection"], + "status": ps["status"], + "database": ps["database"], + }) + + +# ── Serving runtime creation ──────────────────────────────────────────── + + +@mcp.tool() +def create_serving_runtime( + namespace: str, + name: str, + display_name: str = "", + model_formats: list = None, + container_image: str = "", + container_port: int = 8080, + multi_model: bool = False, + api_protocol: str = "REST", +) -> str: + """Create a custom ServingRuntime in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + if not model_formats: + raise ValueError("model_formats must specify at least one model format") + + runtime = { + "name": name, + "display_name": display_name or name, + "model_formats": model_formats, + "requires_instantiation": False, + "source": "custom", + "api_protocol": api_protocol, + "container_image": container_image, + "container_port": container_port, + "multi_model": multi_model, + } + SERVING_RUNTIMES.setdefault(namespace, []).append(runtime) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "model_formats": model_formats, + }) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/instruction.md b/evaluation/without_skills/rh-ai-engineer__model-deploy/instruction.md new file mode 100644 index 00000000..44f79a58 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__model-deploy/instruction.md @@ -0,0 +1,15 @@ +# Model Deployment Task + +You are an AI engineer on Red Hat OpenShift AI. Your team has trained models ready to serve and needs them deployed as inference endpoints in the `ml-production` project. + +## Requirements +- Examine the existing project, available serving runtimes, and any existing deployments +- Diagnose any failing deployments: check pod conditions, container status, logs, and events to determine root causes +- For GPU memory issues, provide a VRAM budget analysis showing model weight size, KV cache requirements, and available GPU memory — distinguish GPU VRAM constraints from pod system memory limits +- Before recommending fixes, check the namespace environment for resource policies and GPU node scheduling constraints that could block redeployment +- For each failing deployment, provide a complete KServe InferenceService YAML manifest with your recommended fix +- Produce a deployment plan that addresses all identified issues and gets the models serving successfully + +Document your deployment plan, diagnosed issues, environment validation, and recommended fixes in `/root/report.md`. + +Use MCP tools to interact with the platform. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/solution/solve.sh b/evaluation/without_skills/rh-ai-engineer__model-deploy/solution/solve.sh new file mode 100644 index 00000000..05b7171e --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__model-deploy/solution/solve.sh @@ -0,0 +1,63 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Model Deployment Plan + +## Diagnosed Issues + +### GPU VRAM Budget Analysis +The vLLM OOM is a **GPU VRAM constraint**, not a pod system memory issue: +- Model weights: ~13.5 GiB loaded into GPU +- KV cache allocation: ~28.5 GiB (at default max_model_len=32768) +- Available VRAM after model load: ~10.1 GiB on A10G (24576 MiB total) +- **Root cause**: Default max_model_len=32768 causes KV cache to exhaust GPU VRAM +- **Fix**: Set MAX_MODEL_LEN=4096 or GPU_MEMORY_UTILIZATION=0.85 + +### LimitRange Conflict +- Namespace LimitRange min CPU: 100m +- KServe sidecar containers request: 10m CPU, 15Mi memory +- **CONFLICT**: Sidecar resources below LimitRange minimum +- Fix: Adjust LimitRange or use annotation to override + +### GPU Node Taints +- GPU nodes may have taint ai-app=true:NoSchedule +- Add matching tolerations to InferenceService predictor spec + +### NIMAccount Dependency +- NIM deployments require a NIMAccount CR to be ready before ServingRuntime can pull images +- Check for NIMAccountNotReady condition if ImagePullBackOff occurs + +## Recommended InferenceService YAML + +```yaml +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: llama-3-8b + namespace: ml-production + annotations: + serving.kserve.io/deploymentMode: RawDeployment +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: vllm-cuda-runtime + storageUri: "hf://meta-llama/Llama-3-8B" + resources: + requests: + cpu: "4" + memory: "32Gi" + nvidia.com/gpu: "1" + containers: + - name: kserve-container + env: + - name: MAX_MODEL_LEN + value: "4096" + - name: GPU_MEMORY_UTILIZATION + value: "0.85" +``` + +## Endpoint +- get_model_endpoint for inference URL +- vLLM: /v1/completions, KServe v2: /v2/models/[model]/infer +REPORT_EOF diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/task.toml b/evaluation/without_skills/rh-ai-engineer__model-deploy/task.toml new file mode 100644 index 00000000..90674851 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__model-deploy/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-ai-engineer__model-deploy" +name = "rh-ai-engineer Model Deployment Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-ai-engineer", "model-deploy", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/tests/llm_judge.py b/evaluation/without_skills/rh-ai-engineer__model-deploy/tests/llm_judge.py new file mode 100644 index 00000000..5cd7c20e --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__model-deploy/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "vram_budget_methodology", + "file": "/root/report.md", + "question": "Does the report present a specific GPU VRAM budget calculation for the vLLM OOM issue — showing the model weight size (~13.5 GiB), the KV cache allocation requirement (~28.5 GiB), and the available VRAM after model load (~10.1 GiB) — and explicitly state that this is a GPU VRAM constraint, NOT a pod system memory issue?", + "reference": "A skilled report shows a VRAM budget breakdown: model weights (~13.5 GiB) loaded into GPU, KV cache requiring ~28.5 GiB, but only ~10.1 GiB available on the 24 GB A10G after model load. It explicitly distinguishes GPU VRAM from pod memory (system RAM). A report that says 'OOMKilled' and recommends increasing pod memory from 16Gi to 32Gi WITHOUT this GPU VRAM analysis is insufficient." + }, + { + "id": "rhoai_deployment_conventions", + "file": "/root/report.md", + "question": "Does the report use RHOAI-specific deployment conventions such as the RawDeployment annotation and GPU_MEMORY_UTILIZATION environment variable configuration, rather than generic Kubernetes deployment patterns?", + "reference": "A skilled report uses serving.kserve.io/deploymentMode: RawDeployment annotation and configures vLLM tuning parameters (GPU_MEMORY_UTILIZATION, MAX_MODEL_LEN) as environment variables in the InferenceService spec. It also identifies NIMAccount CR dependencies for NIM deployments. A report that uses generic Kubernetes deployments or command-line args without RHOAI-specific annotations is insufficient." + }, + { + "id": "kserve_yaml_manifest", + "file": "/root/report.md", + "question": "Does the report include a complete KServe InferenceService YAML manifest with the serving.kserve.io/v1beta1 apiVersion, including metadata (name, namespace) and spec.predictor with model format, storage URI, resource requests, and GPU count?", + "reference": "A skilled report provides a deployable InferenceService YAML with apiVersion: serving.kserve.io/v1beta1, kind: InferenceService, and a complete spec including predictor with model format, runtime reference, storage URI, resource requests (CPU, memory, GPU), and environment variables (VLLM_MAX_MODEL_LEN). A report that only describes fixes in narrative or MCP tool call format without a formal YAML manifest is insufficient." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/tests/test.sh b/evaluation/without_skills/rh-ai-engineer__model-deploy/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__model-deploy/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/tests/test_outputs.py b/evaluation/without_skills/rh-ai-engineer__model-deploy/tests/test_outputs.py new file mode 100644 index 00000000..0669d687 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__model-deploy/tests/test_outputs.py @@ -0,0 +1,94 @@ +""" +Tests for rh-ai-engineer__model-deploy per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ["model", "deploy", "inference", "serving"]), ( + "report should mention model deployment" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 200, "report should have substantial content" + + +class TestSkillDependent: + def test_vram_budget_analysis(self): + """Skill teaches GPU VRAM budget: model weights (13.5 GiB) + KV cache (28.5 GiB) + exceeds A10G capacity (24 GB). Without skill, agents report OOM with approximate + numbers (~14GB) without KV cache sizing or available VRAM calculation.""" + c = read_report() + assert any(t in c for t in [ + "28.5", "10.1 GiB", "10.1 GB", "24576", + ]), ( + "should include specific VRAM budget numbers " + "(KV cache size ~28.5 GiB, available VRAM ~10.1 GiB, or total GPU VRAM 24576 MiB)" + ) + + def test_default_context_window_32768(self): + """Skill teaches that vLLM default max_model_len=32768 causes KV cache to exhaust + GPU VRAM on A10G. Without skill, agents report OOM without identifying the specific + default value that triggers the oversized KV cache allocation.""" + c = read_report() + assert "32768" in c or "32,768" in c, ( + "should identify max_model_len=32768 as the specific vLLM default causing GPU OOM" + ) + + def test_kserve_yaml_apiversion(self): + """Skill teaches creating InferenceService YAML with serving.kserve.io/v1beta1. + Without skill, agents describe fixes via MCP tool calls or narrative without + providing a formal KServe YAML manifest with the correct apiVersion.""" + c = read_report() + assert "serving.kserve.io/v1beta1" in c, ( + "should include InferenceService YAML manifest with serving.kserve.io/v1beta1 apiVersion" + ) + + def test_raw_deployment_mode(self): + """Skill teaches using serving.kserve.io/deploymentMode: RawDeployment annotation + for RHOAI model deployments. Without skill, agents omit this RHOAI-specific + annotation, which controls how KServe deploys the predictor.""" + c = read_report() + assert "RawDeployment" in c or "deploymentMode" in c, ( + "should include RawDeployment annotation (RHOAI deployment mode)" + ) + + def test_known_model_profile(self): + """Docs teach known model profiles: e.g., Llama 3.1 8B needs 1 GPU with 16GB VRAM, + --max-model-len=4096; 70B needs 4xA100 80GB with --tensor-parallel-size=4. + Without docs, agents can't size GPU allocation per model.""" + c = read_report().lower() + assert any(t in c for t in [ + "max-model-len", "max_model_len", "tensor-parallel-size", + "tensor_parallel_size", "16gb", "a100", "a10g", + ]) or ("gpu" in c and ("vram" in c or "model" in c and "profile" in c)), ( + "should reference known model GPU profiles for deployment sizing" + ) + + def test_nim_account_cr(self): + """Skill teaches that NIM deployments require a NIMAccount CR to be ready + before the ServingRuntime can pull images. Without skill, agents diagnose + ImagePullBackOff generically without identifying the NIMAccount dependency.""" + c = read_report() + assert any(t in c for t in [ + "NIMAccount", "NimAccount", "nim-account", "NIM Account", + "NIMAccountNotReady", + ]), "should identify NIMAccount CR as prerequisite for NIM deployment" diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/Dockerfile b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/Dockerfile new file mode 100644 index 00000000..aac4c84e --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/Dockerfile @@ -0,0 +1,67 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "rhoai": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-rhoai-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/common-issues.md b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/examples/model-deploy.md b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/examples/model-deploy.md new file mode 100644 index 00000000..dcd79e44 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/examples/model-deploy.md @@ -0,0 +1,160 @@ +# /model-deploy Examples + +Example interactions demonstrating the model-deploy skill workflow. + +## Example 1: Deploy Llama 3.1 8B with vLLM + +``` +User: "Deploy Llama 3.1 8B on my cluster" + +Agent: [Invokes /model-deploy skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand runtime capabilities. + +Step 1 of 10: Gathering deployment information. + +| Setting | Value | Source | +|---------|-------|--------| +| Model | Llama 3.1 8B Instruct | user input | +| Runtime | TBD | auto-detect | +| Namespace | ? | needs input | +| Model Source | ? | needs input | +| Deployment Mode | Serverless | default | + +What namespace should I deploy to, and where are the model weights stored? +(e.g., S3 bucket, OCI registry, PVC) + +User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" + +Agent: Step 2 of 10: Determining runtime. + +I consulted supported-runtimes.md to understand runtime capabilities. + +Based on Llama 3.1 8B (open-source HuggingFace model): + +Recommended runtime: vLLM +Reason: Default runtime for open-source LLMs. Broad compatibility, + OpenAI-compatible API, no additional setup required. + +Proceed with vLLM? (yes / no / suggest alternative) + +User: "yes" + +Agent: Step 3 of 10: Looking up hardware profile. + +I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. + +| Requirement | Value | +|-------------|-------| +| GPUs | 1x A10G/L4/A100/H100 | +| VRAM | 16GB | +| Key Args | --max-model-len=4096 --tool-call-parser hermes | + +Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). +Ensure your cluster has at least 1 GPU with 16GB VRAM. + +Step 5 of 10: Skipped (NIM platform check not needed for vLLM). + +Step 6-7 of 10: Generating InferenceService YAML. + +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: llama-3-1-8b + namespace: ai-models + annotations: + serving.kserve.io/deploymentMode: Serverless +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: vllm-runtime + storageUri: s3://my-bucket/llama-3.1-8b + resources: + limits: + nvidia.com/gpu: "1" + requests: + cpu: "4" + memory: "16Gi" + args: + - --max-model-len=4096 + - --tool-call-parser=hermes + +| Setting | Value | +|---------|-------| +| Model | llama-3-1-8b | +| Runtime | vllm-runtime | +| GPUs | 1 | +| Namespace | ai-models | +| Mode | Serverless | + +Proceed with creating this InferenceService? (yes/no/modify) + +User: "yes" + +Agent: Step 8 of 10: Creating InferenceService... + [Creates via resources_create_or_update] + + InferenceService created. Monitoring rollout... + +Step 9 of 10: Monitoring rollout. + +| Stage | Status | +|-------|--------| +| Pod Scheduled | Complete | +| Image Pulled | Complete | +| Container Started | Complete | +| Model Loaded | Complete | +| Ready | Complete | + +Step 10 of 10: Deployment complete! + +Model Deployment Successful + +Model: llama-3-1-8b +Runtime: vLLM +Namespace: ai-models +GPUs: 1 + +Inference Endpoint: + URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com + API: OpenAI-compatible REST + +Quick Test: + curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' + +Next Steps: + Monitor performance: "/ai-observability llama-3-1-8b" +``` + +## Example 2: Deploy a Model with NIM (NIM Not Set Up) + +``` +User: "Deploy Llama 3.1 70B with NIM" + +Agent: [Invokes /model-deploy skill] + ... + Step 2: User requested NIM runtime. + + Step 5: Checking NIM platform... + +NIM platform is not set up in namespace ai-models. + +The NIM runtime requires a one-time platform setup. Would you like to: +1. Run /nim-setup now to configure the NIM platform +2. Switch to vLLM runtime instead (no setup required) +3. Abort deployment + +User: "1" + +Agent: [Invokes /nim-setup skill — walks through NGC credential setup] + ... + NIM Platform Setup Complete. + + Resuming model deployment... + [Continues from Step 6 with NIM runtime] +``` diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/examples/nim-setup.md b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/examples/nim-setup.md new file mode 100644 index 00000000..ccfc1a5d --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/examples/nim-setup.md @@ -0,0 +1,115 @@ +# /nim-setup Examples + +Example interactions demonstrating the nim-setup skill workflow. + +## Example 1: First-Time NIM Setup + +``` +User: "Set up NIM on my cluster" + +Agent: [Invokes /nim-setup skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand NIM platform requirements. + +Let me verify the prerequisites for NIM setup. + +Step 1 of 7: Checking GPU Operator and NFD Operator... + +GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) +NFD Operator: nfd.v4.19.0 (Succeeded) + +Both operators are installed and healthy. + +Step 2 of 7: I need your NGC credentials. + +To set up NIM, I need your NVIDIA NGC API key. +You can generate one at: https://ngc.nvidia.com/setup/api-key + +Please provide: +1. Your NGC API key +2. The target namespace for NIM resources + +User: "Here's my key: [user provides NGC API key], namespace: ai-models" + +Agent: Step 3 of 7: Creating NGC image pull secret... + +I'll create a docker-registry secret for pulling NIM images from nvcr.io: + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-image-pull-secret + namespace: ai-models + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] + +Should I create this image pull secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + Image pull secret created successfully. + +Step 4 of 7: Creating NGC API key secret... + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-api-key + namespace: ai-models + type: Opaque + stringData: + NGC_API_KEY: "[REDACTED]" + +Should I create this API key secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + API key secret created successfully. + +Step 5 of 7: Creating NIM Account CR... + + apiVersion: nim.opendatahub.io/v1 + kind: Account + metadata: + name: nim-account + namespace: ai-models + spec: + apiKeySecret: + name: ngc-api-key + imagePullSecret: + name: ngc-image-pull-secret + +Should I create this NIM Account CR in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates Account CR via resources_create_or_update] + Account CR created successfully. + +Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) + +User: "no" + +Agent: Step 7 of 7: Validating NIM platform readiness... + +NIM Platform Setup Complete + +Account CR: `nim-account` (namespace: ai-models) + Status: Ready + +NGC Credentials: + Image Pull Secret: ngc-image-pull-secret [created] + API Key Secret: ngc-api-key [created] + +NIM ServingRuntimes Available: + - nim-llama-3-1-8b-instruct + - nim-llama-3-1-70b-instruct + - nim-mistral-7b-instruct + +Next Steps: + To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" +``` diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/live-doc-lookup.md b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/references/known-model-profiles.md b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/references/known-model-profiles.md new file mode 100644 index 00000000..40148981 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/references/known-model-profiles.md @@ -0,0 +1,83 @@ +--- +title: Known Model Hardware Profiles +category: references +tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] +semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] +use_cases: [model-deploy, debug-inference, ai-observability] +last_updated: 2026-03-10 +--- + +# Known Model Hardware Profiles + +Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). + +**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. + +## Model Source Conventions + +Each model lists a recommended `storageUri` with its authentication requirements: +- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** +- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). +- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. + +When the user does not specify a model source, use the `hf://` URI listed in the profile below. + +## Llama 3.x (Meta) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | +| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | +| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | +| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | + +- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) +- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly + +## Granite 3.x (IBM/Red Hat) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | +| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | + +- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) +- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser granite --chat-template` +- Red Hat-supported model family on RHOAI + +## Mixtral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | +| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) +- Mixture-of-Experts architecture: only ~13B/45B parameters active per token + +## Mistral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | +| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) + +## When a Model Is Not Listed + +If the requested model is not in this file, the agent MUST use the live doc lookup protocol: + +1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure +2. Fetch hardware specs from the appropriate source +3. Report findings to the user before proceeding with deployment + +Common cases requiring live lookup: +- Newly released models (after this file's last update) +- Domain-specific fine-tuned models +- Models with custom quantization +- NIM-specific optimized profiles diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/references/supported-runtimes.md b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/references/supported-runtimes.md new file mode 100644 index 00000000..9d018651 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/references/supported-runtimes.md @@ -0,0 +1,104 @@ +--- +title: Supported Serving Runtimes +category: references +tags: [runtimes, vllm, nim, caikit, tgis, serving] +semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] +use_cases: [model-deploy, serving-runtime-config, nim-setup] +last_updated: 2026-02-26 +--- + +# Supported Serving Runtimes + +This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. + +## Runtime Comparison + +| Runtime | API Type | Model Format | GPU Support | License | Setup Required | +|---------|----------|-------------|-------------|---------|----------------| +| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | +| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | +| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | +| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | + +## vLLM + +**Default runtime for most open-source models.** + +- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) +- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized +- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) +- **Model source**: S3-compatible storage, OCI registry, PVC, URI +- **Key features**: + - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) + - PagedAttention for efficient memory management + - Tensor parallelism for multi-GPU inference + - Tool/function calling support (`--tool-call-parser`) + - Continuous batching for high throughput +- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name + +## NVIDIA NIM + +**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** + +- **API**: OpenAI-compatible REST +- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog +- **GPU support**: NVIDIA only (requires specific GPU models per model profile) +- **Model source**: NGC model catalog (pulled automatically via NGC credentials) +- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) +- **Key features**: + - TensorRT-LLM optimization for lower latency + - Pre-compiled model engines (no compilation on first load) + - Optimized and generic profiles per GPU type + - Automatic model download from NGC +- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. +- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM +- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. +- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state + +## Caikit+TGIS + +**Red Hat's Caikit format with Text Generation Inference Server.** + +- **API**: gRPC (not REST) +- **Model formats**: Caikit format (requires conversion from HuggingFace) +- **GPU support**: NVIDIA +- **Model source**: S3-compatible storage +- **Key features**: + - Red Hat-supported runtime + - gRPC API for streaming inference + - Integrated with RHOAI model serving platform +- **When to choose**: When the model is already in Caikit format or when gRPC API is required. +- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name + +## Custom Runtimes + +**User-provided ServingRuntime CRs for unsupported frameworks.** + +- **API**: Defined by the custom runtime +- **Model formats**: Defined by the custom runtime +- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. +- **How to create**: Use `/serving-runtime-config` skill +- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility + +## Runtime Selection Decision Tree + +``` +Is the user's preferred runtime explicitly stated? +├── Yes → Use that runtime +└── No → Continue + +Is the model available in the NGC NIM catalog? +├── Yes → Suggest NIM (with vLLM as fallback) +│ Note: Requires /nim-setup and NVIDIA GPUs +└── No → Continue + +Is the model in Caikit format? +├── Yes → Caikit+TGIS +└── No → Continue + +Is the model a standard open-source LLM (HuggingFace-compatible)? +├── Yes → vLLM (default) +└── No → Custom runtime via /serving-runtime-config +``` diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/skill-conventions.md b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..d43c891d --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,540 @@ +#!/usr/bin/env python3 +"""Mock OpenShift MCP server for SkillsBench rh-ai-engineer task. + +Simulates Kubernetes resource CRUD, pod management, logs, and events. + +Key scenario elements: +- LimitRange in namespaces: min CPU=100m, min memory=128Mi + (conflicts with KServe sidecar containers hardcoded at 10m CPU/15Mi memory) +- GPU node with custom taint ai-workload=true:NoSchedule +- NIM Account CR in ml-production: not ready (NGC credentials invalid) +- text-gen-legacy pods: OOMKilled (max-model-len=32768 on A10G) +- nim-llama-prod: no pods created (Account CR not ready) +""" + +import base64 +import json +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + +# ── Cluster state ──────────────────────────────────────────────────────── + +GPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "gpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + "nvidia.com/gpu.present": "true", + "nvidia.com/gpu.product": "NVIDIA-A10G", + }, + }, + "spec": { + "taints": [ + { + "key": "ai-workload", + "value": "true", + "effect": "NoSchedule", + }, + ], + }, + "status": { + "allocatable": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "capacity": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "conditions": [ + {"type": "Ready", "status": "True"}, + ], + }, +} + +CPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "cpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + }, + }, + "spec": {"taints": []}, + "status": { + "allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "capacity": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +MASTER_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "master-1", + "labels": { + "node-role.kubernetes.io/master": "", + "node-role.kubernetes.io/control-plane": "", + }, + }, + "spec": { + "taints": [ + {"key": "node-role.kubernetes.io/master", "effect": "NoSchedule"}, + ], + }, + "status": { + "allocatable": {"cpu": "8", "memory": "32Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +ALL_NODES = [GPU_NODE, CPU_NODE, MASTER_NODE] + +# LimitRange applied by cluster policy to all DS project namespaces +NAMESPACE_LIMITRANGE = { + "apiVersion": "v1", + "kind": "LimitRange", + "metadata": { + "name": "default-limits", + }, + "spec": { + "limits": [ + { + "type": "Container", + "default": { + "cpu": "2", + "memory": "4Gi", + }, + "defaultRequest": { + "cpu": "500m", + "memory": "256Mi", + }, + "min": { + "cpu": "100m", + "memory": "128Mi", + }, + "max": { + "cpu": "32", + "memory": "128Gi", + }, + }, + ], + }, +} + +NIM_ACCOUNT_CR = { + "apiVersion": "nim.opendatahub.io/v1", + "kind": "Account", + "metadata": { + "name": "nim-account", + "namespace": "ml-production", + }, + "spec": { + "apiKeySecret": { + "name": "ngc-api-key", + }, + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "NGCCredentialsInvalid", + "message": "NGC API key validation failed: 401 Unauthorized. " + "The API key in secret 'ngc-api-key' is expired or invalid. " + "Re-create the secret with a valid NGC API key from " + "https://ngc.nvidia.com/setup/api-key and restart the " + "Account reconciliation.", + "lastTransitionTime": "2026-03-14T12:00:00Z", + }, + ], + "nimPullSecretStatus": "Failed", + "nimConfigStatus": "Pending", + }, +} + +SERVING_RUNTIME_VLLM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "vllm-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "vLLM", "version": "1", "autoSelect": True}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "quay.io/modh/vllm:rhoai-2.16", + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + }, + ], + }, +} + +SERVING_RUNTIME_NIM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "nim-serving-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "NIM", "version": "1"}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "nvcr.io/nim/meta/llama-3.1-8b-instruct:latest", + "ports": [{"containerPort": 8000, "protocol": "TCP"}], + "env": [ + {"name": "NGC_API_KEY", "valueFrom": { + "secretKeyRef": {"name": "ngc-api-key", "key": "api_key"}, + }}, + ], + }, + ], + }, +} + +PODS_BY_NAMESPACE = { + "ml-production": [ + { + "name": "text-gen-legacy-predictor-00001-abc12", + "namespace": "ml-production", + "status": "CrashLoopBackOff", + "restarts": 5, + "node": "gpu-worker-1", + "containers": [ + { + "name": "kserve-container", + "state": "waiting", + "reason": "CrashLoopBackOff", + "last_termination_reason": "OOMKilled", + "last_termination_exit_code": 137, + }, + ], + "labels": { + "serving.kserve.io/inferenceservice": "text-gen-legacy", + }, + "gpu": "1", + }, + # nim-llama-prod: NO pods created (Account CR not ready) + ], +} + +POD_LOGS = { + "text-gen-legacy-predictor-00001-abc12": ( + "INFO 2026-03-01 10:00:00 vllm_engine.py:125] vLLM engine starting...\n" + "INFO 2026-03-01 10:00:01 config.py:89] Model: mistralai/Mistral-7B-Instruct-v0.3\n" + "INFO 2026-03-01 10:00:01 config.py:92] max_model_len = 32768\n" + "INFO 2026-03-01 10:00:02 gpu_executor.py:45] GPU 0: NVIDIA A10G (24576 MiB)\n" + "INFO 2026-03-01 10:00:03 model_runner.py:88] Loading model weights...\n" + "INFO 2026-03-01 10:00:15 model_runner.py:112] Model weights loaded: 13.5 GiB\n" + "INFO 2026-03-01 10:00:15 worker.py:201] Allocating KV cache...\n" + "ERROR 2026-03-01 10:00:16 worker.py:215] torch.cuda.OutOfMemoryError: " + "CUDA out of memory. Tried to allocate 28.5 GiB for KV cache but only " + "10.1 GiB available after loading model weights (13.5 GiB).\n" + "ERROR 2026-03-01 10:00:16 vllm_engine.py:178] Engine failed to start\n" + "Traceback (most recent call last):\n" + " File \"/opt/vllm/vllm/engine/engine.py\", line 175, in start\n" + " self._init_kv_cache()\n" + " File \"/opt/vllm/vllm/worker/worker.py\", line 215, in _init_kv_cache\n" + " raise torch.cuda.OutOfMemoryError(msg)\n" + "torch.cuda.OutOfMemoryError: CUDA out of memory\n" + ), +} + +EVENTS_BY_NAMESPACE = { + "ml-production": [ + { + "type": "Warning", + "reason": "BackOff", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Back-off restarting failed container kserve-container in pod " + "text-gen-legacy-predictor-00001-abc12", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Warning", + "reason": "OOMKilled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Container kserve-container was OOMKilled (exit code 137). " + "GPU memory exhausted during KV cache allocation.", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Normal", + "reason": "Scheduled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Successfully assigned ml-production/" + "text-gen-legacy-predictor-00001-abc12 to gpu-worker-1", + "count": 1, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-02-28T08:00:00Z", + }, + { + "type": "Warning", + "reason": "NIMAccountNotReady", + "object": "InferenceService/nim-llama-prod", + "message": "NIM Account 'nim-account' in namespace 'ml-production' " + "is not ready", + "count": 12, + "first_timestamp": "2026-03-14T12:00:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + { + "type": "Warning", + "reason": "ImagePullBackOff", + "object": "InferenceService/nim-llama-prod", + "message": "Failed to pull image 'nvcr.io/nim/meta/llama-3.1-8b-instruct:" + "latest': unauthorized: authentication required", + "count": 8, + "first_timestamp": "2026-03-14T12:05:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + ], +} + + +# ── Resource tools ─────────────────────────────────────────────────────── + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: str = "", +) -> str: + """Get a single Kubernetes resource by apiVersion, kind, and name.""" + if kind == "Node": + for node in ALL_NODES: + if node["metadata"]["name"] == name: + return json.dumps(node, indent=2) + raise ValueError(f"Node '{name}' not found") + + if kind == "ServingRuntime": + if name == "vllm-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_VLLM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + if name == "nim-serving-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_NIM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + raise ValueError(f"ServingRuntime '{name}' not found in namespace '{namespace}'") + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps(lr, indent=2) + + if kind == "Account" and "nim" in apiVersion.lower(): + if namespace == "ml-production" and name == "nim-account": + return json.dumps(NIM_ACCOUNT_CR, indent=2) + raise ValueError( + f"Account '{name}' not found in namespace '{namespace}'" + ) + + if kind == "ClusterVersion" and apiVersion == "config.openshift.io/v1": + return json.dumps({ + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterVersion", + "metadata": {"name": "version"}, + "status": {"desired": {"version": "4.16.3"}}, + }) + + raise ValueError(f"Resource {apiVersion}/{kind}/{name} not found") + + +@mcp.tool() +def resources_create_or_update( + api_version: str, + kind: str, + namespace: str, + name: str, + body: str, +) -> str: + """Create or update a Kubernetes resource. Accepts apiVersion, kind, namespace, name, and body (JSON).""" + try: + resource = json.loads(body) + except json.JSONDecodeError as e: + raise ValueError(f"Invalid JSON body: {e}") from e + + resource.setdefault("metadata", {}) + resource["metadata"]["name"] = name + resource["metadata"]["namespace"] = namespace + resource["apiVersion"] = api_version + resource["kind"] = kind + + if kind == "Secret": + resource.setdefault("type", "Opaque") + return json.dumps({ + "status": "created", + "resource": resource, + "message": f"Secret '{name}' created/updated in namespace '{namespace}'", + }, indent=2) + + if kind in ("NIMAccount", "Account") and "nim" in api_version.lower(): + resource.setdefault("status", {}) + resource["status"]["conditions"] = [ + { + "type": "Ready", + "status": "True", + "reason": "NGCCredentialsValid", + "message": "NGC API key validated successfully", + "lastTransitionTime": "2026-03-17T12:00:00Z", + }, + ] + resource["status"]["nimPullSecretStatus"] = "Ready" + resource["status"]["nimConfigStatus"] = "Ready" + return json.dumps({ + "status": "created", + "resource": resource, + "message": f"NIM Account '{name}' created/updated in namespace '{namespace}'", + }, indent=2) + + if kind == "ConfigMap": + return json.dumps({ + "status": "created", + "resource": resource, + "message": f"ConfigMap '{name}' created/updated in namespace '{namespace}'", + }, indent=2) + + raise ValueError(f"Unsupported kind for create/update: {kind}") + + +@mcp.tool() +def create_secret( + namespace: str, + name: str, + data: dict, + type: str = "Opaque", +) -> str: + """Create a Secret in a namespace. data is a dict of key-value pairs (values will be base64-encoded).""" + if isinstance(data, str): + data = json.loads(data) + encoded_data = {k: base64.b64encode(str(v).encode()).decode() for k, v in data.items()} + resource = { + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": name, "namespace": namespace}, + "type": type, + "data": encoded_data, + } + return json.dumps({ + "status": "created", + "resource": resource, + "message": f"Secret '{name}' created in namespace '{namespace}'", + }, indent=2) + + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: str = "", + labelSelector: str = "", +) -> str: + """List Kubernetes resources by apiVersion and kind.""" + if kind == "Node": + nodes = ALL_NODES + if labelSelector: + parts = labelSelector.split("=", 1) + key = parts[0] + value = parts[1] if len(parts) > 1 else "" + nodes = [ + n for n in nodes + if n["metadata"]["labels"].get(key) == value + ] + return json.dumps(nodes, indent=2) + + if kind == "Service" and apiVersion == "serving.knative.dev/v1": + return json.dumps({ + "kind": "ServiceList", + "apiVersion": "serving.knative.dev/v1", + "items": [], + "metadata": {}, + }) + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps({ + "kind": "LimitRangeList", + "items": [lr], + }) + + if kind == "InferenceService": + return json.dumps({ + "kind": "InferenceServiceList", + "items": [], + }) + + raise ValueError(f"Unsupported list: {apiVersion}/{kind}") + + +@mcp.tool() +def pods_list( + namespace: str, + labelSelector: str = "", +) -> str: + """List pods in a namespace with optional label selector.""" + pods = PODS_BY_NAMESPACE.get(namespace, []) + + if labelSelector: + key, _, value = labelSelector.partition("=") + pods = [p for p in pods if p.get("labels", {}).get(key) == value] + + results = [] + for pod in pods: + results.append({ + "name": pod["name"], + "namespace": pod["namespace"], + "status": pod["status"], + "restarts": pod.get("restarts", 0), + "node": pod.get("node", ""), + "containers": pod.get("containers", []), + "gpu": pod.get("gpu", "0"), + }) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def pods_log( + namespace: str, + name: str, + container: str = "", +) -> str: + """Get logs from a pod container.""" + logs = POD_LOGS.get(name) + if logs is None: + raise ValueError(f"Pod '{name}' not found in namespace '{namespace}'") + return logs + + +@mcp.tool() +def events_list(namespace: str) -> str: + """List events in a namespace.""" + events = EVENTS_BY_NAMESPACE.get(namespace, []) + return json.dumps(events, indent=2) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/mcp-servers/mock-rhoai-mcp.py b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/mcp-servers/mock-rhoai-mcp.py new file mode 100644 index 00000000..0ae9e4cb --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/mcp-servers/mock-rhoai-mcp.py @@ -0,0 +1,780 @@ +#!/usr/bin/env python3 +"""Mock RHOAI MCP server for SkillsBench rh-ai-engineer task. + +Simulates Red Hat OpenShift AI operations: Data Science Projects, +model serving, data connections, serving runtimes, inference services. + +Scenario: +- ml-production: existing project with two broken deployments + - text-gen-legacy: vLLM OOMKilled (max-model-len=32768 on A10G) + - nim-llama-prod: NIM failing (Account CR not ready, NGC creds invalid) +- fraud-detection: does not exist yet (agent creates it) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("rhoai") + +# ── In-memory state ────────────────────────────────────────────────────── + +PROJECTS = { + "ml-production": { + "name": "ml-production", + "display_name": "ML Production", + "description": "Production ML workloads", + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": "single", + "pipeline_server": True, + }, +} + +DATA_CONNECTIONS = { + "ml-production": [ + { + "name": "prod-model-store", + "type": "S3", + "bucket": "ml-models-prod", + "endpoint": "https://s3.us-east-1.amazonaws.com", + "region": "us-east-1", + }, + ], +} + +SERVING_RUNTIMES = { + "__platform_templates__": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "REST", + "supported_model_formats": [ + {"name": "vLLM", "version": "1", "autoSelect": True} + ], + }, + { + "name": "caikit-tgis-runtime", + "display_name": "Caikit+TGIS ServingRuntime", + "model_formats": ["caikit"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "gRPC", + }, + ], + "ml-production": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + { + "name": "nim-serving-runtime", + "display_name": "NVIDIA NIM ServingRuntime", + "model_formats": ["NIM"], + "requires_instantiation": False, + "source": "nim-account", + "api_protocol": "REST", + }, + { + "name": "ovms-1", + "display_name": "OpenVINO Model Server", + "model_formats": ["openvino_ir", "onnx"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + ], +} + +INFERENCE_SERVICES = { + "ml-production": { + "text-gen-legacy": { + "name": "text-gen-legacy", + "namespace": "ml-production", + "runtime": "vllm-runtime", + "model_format": "vLLM", + "storage_uri": "hf://mistralai/Mistral-7B-Instruct-v0.3", + "display_name": "Mistral 7B Legacy", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "16Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "PredictorFailed", + "message": "Predictor pod is not ready", + }, + { + "type": "PredictorReady", + "status": "False", + "reason": "ContainerCrashLoop", + "message": "Container kserve-container terminated: " + "OOMKilled (exit code 137). 5 restarts.", + }, + { + "type": "IngressReady", + "status": "True", + "reason": "IngressReady", + "message": "Ingress is ready", + }, + ], + "age": "3d", + }, + "nim-llama-prod": { + "name": "nim-llama-prod", + "namespace": "ml-production", + "runtime": "nim-serving-runtime", + "model_format": "NIM", + "storage_uri": "nim://meta/llama-3.1-8b-instruct", + "display_name": "Llama 3.1 8B (NIM)", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "32Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "RuntimeNotReady", + "message": "ServingRuntime 'nim-serving-runtime' " + "is not in ready state", + }, + { + "type": "PredictorReady", + "status": "Unknown", + "reason": "PodNotCreated", + "message": "Predictor pod has not been created. " + "Waiting for ServingRuntime to become ready.", + }, + { + "type": "IngressReady", + "status": "Unknown", + "reason": "PredictorNotReady", + "message": "Waiting for predictor to become ready", + }, + ], + "age": "1d", + }, + }, +} + +DEPLOYED_MODELS = {} + +WORKBENCHES = { + "ml-production": [ + { + "name": "data-exploration-nb", + "display_name": "Data Exploration", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Running", + "cpu_request": "1", + "memory_request": "8Gi", + "gpu_count": 0, + "pvc_name": "data-exploration-nb-pvc", + "pvc_size": "20Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-10T09:00:00Z", + }, + { + "name": "model-training-nb", + "display_name": "Model Training", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Stopped", + "cpu_request": "4", + "memory_request": "16Gi", + "gpu_count": 1, + "pvc_name": "model-training-nb-pvc", + "pvc_size": "50Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-15T14:00:00Z", + }, + ], +} + +PIPELINE_SERVERS = { + "ml-production": { + "configured": True, + "data_connection": "prod-model-store", + "status": "Ready", + "database": "MariaDB", + }, +} + +NOTEBOOK_IMAGES = [ + {"name": "jupyter-pytorch-ubi9-python-3.9-2024.1", "display_name": "PyTorch 2024.1", "packages": ["torch", "transformers"]}, + {"name": "jupyter-tensorflow-ubi9-python-3.9-2024.1", "display_name": "TensorFlow 2024.1", "packages": ["tensorflow"]}, + {"name": "jupyter-datascience-ubi9-python-3.9-2024.1", "display_name": "Standard Data Science", "packages": ["pandas", "scikit-learn"]}, + {"name": "jupyter-minimal-ubi9-python-3.9-2024.1", "display_name": "Minimal Python", "packages": []}, +] + + +# ── Project tools ──────────────────────────────────────────────────────── + + +@mcp.tool() +def list_data_science_projects() -> str: + """List all RHOAI Data Science Projects on the cluster.""" + projects = [] + for name, proj in PROJECTS.items(): + projects.append({ + "name": name, + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + }) + return json.dumps(projects, indent=2) + + +@mcp.tool() +def create_data_science_project( + name: str, + display_name: str, + description: str = "", +) -> str: + """Create a new RHOAI Data Science Project (namespace with dashboard labels).""" + if name in PROJECTS: + raise ValueError( + f"Project '{name}' already exists. Choose a different name " + "or configure the existing project." + ) + if not name.replace("-", "").replace("_", "").isalnum() or len(name) > 63: + raise ValueError( + f"Invalid project name '{name}'. Must be DNS-compatible: " + "lowercase alphanumeric and hyphens, max 63 chars." + ) + + PROJECTS[name] = { + "name": name, + "display_name": display_name, + "description": description, + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": None, + "pipeline_server": False, + } + DATA_CONNECTIONS[name] = [] + SERVING_RUNTIMES[name] = [] + INFERENCE_SERVICES[name] = {} + + return json.dumps({ + "status": "created", + "name": name, + "display_name": display_name, + "namespace": name, + "labels": {"opendatahub.io/dashboard": "true"}, + }) + + +@mcp.tool() +def get_project_details(name: str) -> str: + """Get detailed information about an RHOAI Data Science Project.""" + if name not in PROJECTS: + raise ValueError(f"Project '{name}' not found") + proj = PROJECTS[name] + dc_count = len(DATA_CONNECTIONS.get(name, [])) + isvc_count = len(INFERENCE_SERVICES.get(name, {})) + return json.dumps({ + "name": proj["name"], + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "labels": proj["labels"], + "data_connections": dc_count, + "inference_services": isvc_count, + "model_serving_mode": proj.get("model_serving_mode"), + "pipeline_server": proj.get("pipeline_server", False), + }) + + +@mcp.tool() +def get_project_status(namespace: str) -> str: + """Get comprehensive status of an RHOAI Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Project '{namespace}' not found") + proj = PROJECTS[namespace] + dcs = DATA_CONNECTIONS.get(namespace, []) + isvcs = INFERENCE_SERVICES.get(namespace, {}) + return json.dumps({ + "namespace": namespace, + "display_name": proj["display_name"], + "status": "Active", + "components": { + "data_connections": len(dcs), + "inference_services": len(isvcs), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + "pipeline_server": "configured" if proj.get("pipeline_server") else "not configured", + }, + }) + + +# ── Data connection tools ──────────────────────────────────────────────── + + +@mcp.tool() +def create_s3_data_connection( + namespace: str, + name: str, + bucket: str, + endpoint: str, + access_key: str, + secret_key: str, + region: str = "", +) -> str: + """Create an S3-compatible data connection in an RHOAI project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + existing = DATA_CONNECTIONS.get(namespace, []) + if any(dc["name"] == name for dc in existing): + raise ValueError( + f"Data connection '{name}' already exists in namespace '{namespace}'" + ) + + dc = { + "name": name, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + "region": region, + } + DATA_CONNECTIONS.setdefault(namespace, []).append(dc) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + }) + + +@mcp.tool() +def list_data_connections(namespace: str) -> str: + """List data connections in an RHOAI project namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + return json.dumps(dcs, indent=2) + + +# ── Model serving tools ───────────────────────────────────────────────── + + +@mcp.tool() +def set_model_serving_mode(namespace: str, mode: str) -> str: + """Enable model serving on a Data Science Project (single or multi mode).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + if mode not in ("single", "multi"): + raise ValueError(f"Invalid mode '{mode}'. Must be 'single' or 'multi'.") + + PROJECTS[namespace]["model_serving_mode"] = mode + + if not SERVING_RUNTIMES.get(namespace): + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + SERVING_RUNTIMES[namespace] = [ + {**t, "requires_instantiation": False, "source": "existing"} + for t in templates + ] + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "mode": mode, + }) + + +@mcp.tool() +def list_serving_runtimes( + namespace: str, + include_templates: bool = False, +) -> str: + """List available ServingRuntimes in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + runtimes = list(SERVING_RUNTIMES.get(namespace, [])) + if include_templates: + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + existing_names = {r["name"] for r in runtimes} + for t in templates: + if t["name"] not in existing_names: + runtimes.append(t) + + return json.dumps(runtimes, indent=2) + + +# ── Inference service tools ────────────────────────────────────────────── + + +@mcp.tool() +def deploy_model( + name: str, + namespace: str, + runtime: str, + model_format: str, + storage_uri: str, + display_name: str = "", + min_replicas: int = 1, + max_replicas: int = 1, + cpu_request: str = "1", + cpu_limit: str = "2", + memory_request: str = "4Gi", + memory_limit: str = "8Gi", + gpu_count: int = 0, +) -> str: + """Deploy an AI/ML model as a KServe InferenceService.""" + if namespace not in PROJECTS: + raise ValueError( + f"Namespace '{namespace}' is not a Data Science Project. " + "Create one via create_data_science_project first." + ) + + ns_runtimes = SERVING_RUNTIMES.get(namespace, []) + runtime_names = [r["name"] for r in ns_runtimes] + if runtime not in runtime_names: + available = ", ".join(runtime_names) or "none" + raise ValueError( + f"ServingRuntime '{runtime}' not found in namespace '{namespace}'. " + f"Available runtimes: {available}" + ) + + endpoint = f"https://{name}-{namespace}.apps.ocp-cluster.example.com" + isvc = { + "name": name, + "namespace": namespace, + "runtime": runtime, + "model_format": model_format, + "storage_uri": storage_uri, + "display_name": display_name or name, + "gpu_count": gpu_count, + "cpu_request": cpu_request, + "memory_request": memory_request, + "min_replicas": min_replicas, + "max_replicas": max_replicas, + "ready": True, + "url": endpoint, + "conditions": [ + {"type": "Ready", "status": "True", "reason": "Ready", "message": ""}, + {"type": "PredictorReady", "status": "True", "reason": "PodReady", "message": ""}, + {"type": "IngressReady", "status": "True", "reason": "IngressReady", "message": ""}, + ], + "age": "0s", + } + + INFERENCE_SERVICES.setdefault(namespace, {})[name] = isvc + DEPLOYED_MODELS[f"{namespace}/{name}"] = isvc + + return json.dumps({ + "status": "deployed", + "name": name, + "namespace": namespace, + "runtime": runtime, + "endpoint": endpoint, + "ready": True, + }) + + +@mcp.tool() +def list_inference_services( + namespace: str, + verbosity: str = "standard", +) -> str: + """List deployed InferenceServices in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + isvcs = INFERENCE_SERVICES.get(namespace, {}) + results = [] + for isvc_name, isvc in isvcs.items(): + entry = { + "name": isvc["name"], + "runtime": isvc["runtime"], + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "age": isvc.get("age", ""), + } + if verbosity == "full": + entry["conditions"] = isvc.get("conditions", []) + entry["storage_uri"] = isvc.get("storage_uri", "") + entry["gpu_count"] = isvc.get("gpu_count", 0) + results.append(entry) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def get_inference_service( + name: str, + namespace: str, + verbosity: str = "standard", +) -> str: + """Get detailed status of a specific InferenceService.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + + isvc = isvcs[name] + result = { + "name": isvc["name"], + "namespace": isvc["namespace"], + "runtime": isvc["runtime"], + "model_format": isvc.get("model_format", ""), + "storage_uri": isvc.get("storage_uri", ""), + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "conditions": isvc.get("conditions", []), + "gpu_count": isvc.get("gpu_count", 0), + "replicas": {"min": isvc.get("min_replicas", 1), "max": isvc.get("max_replicas", 1)}, + "resources": { + "cpu_request": isvc.get("cpu_request", "1"), + "memory_request": isvc.get("memory_request", "4Gi"), + "memory_limit": isvc.get("memory_limit", "8Gi"), + }, + "age": isvc.get("age", ""), + } + return json.dumps(result, indent=2) + + +@mcp.tool() +def get_model_endpoint(name: str, namespace: str) -> str: + """Get the inference endpoint URL for a deployed model.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + isvc = isvcs[name] + if not isvc["ready"]: + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": "", + "error": "InferenceService is not ready. Check conditions for details.", + }) + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": isvc["url"], + }) + + +# ── Workbench tools ────────────────────────────────────────────────────── + + +@mcp.tool() +def list_workbenches(namespace: str) -> str: + """List workbenches (Jupyter notebooks) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + return json.dumps(wbs, indent=2) + + +@mcp.tool() +def create_workbench( + namespace: str, + name: str, + display_name: str = "", + image: str = "jupyter-datascience-ubi9-python-3.9-2024.1", + cpu_request: str = "1", + memory_request: str = "4Gi", + gpu_count: int = 0, + pvc_size: str = "20Gi", +) -> str: + """Create a new workbench (Jupyter notebook) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + valid_images = [img["name"] for img in NOTEBOOK_IMAGES] + if image not in valid_images: + raise ValueError( + f"Image '{image}' not found. Available: {', '.join(valid_images)}" + ) + + wb = { + "name": name, + "display_name": display_name or name, + "image": image, + "status": "Running", + "cpu_request": cpu_request, + "memory_request": memory_request, + "gpu_count": gpu_count, + "pvc_name": f"{name}-pvc", + "pvc_size": pvc_size, + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-03-02T12:00:00Z", + } + WORKBENCHES.setdefault(namespace, []).append(wb) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "image": image, + "pvc": f"{name}-pvc", + }) + + +@mcp.tool() +def stop_workbench(namespace: str, name: str) -> str: + """Stop a running workbench (preserves data).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Stopped" + return json.dumps({"status": "stopped", "name": name, "namespace": namespace}) + + +@mcp.tool() +def start_workbench(namespace: str, name: str) -> str: + """Start a stopped workbench.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Running" + return json.dumps({"status": "running", "name": name, "namespace": namespace}) + + +@mcp.tool() +def delete_workbench(namespace: str, name: str) -> str: + """Delete a workbench. WARNING: PVC data may be lost if not backed up.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wbs.remove(wb) + return json.dumps({ + "status": "deleted", + "name": name, + "namespace": namespace, + "warning": "Associated PVC data has been deleted", + }) + + +@mcp.tool() +def list_notebook_images() -> str: + """List available notebook images for workbench creation.""" + return json.dumps(NOTEBOOK_IMAGES, indent=2) + + +# ── Pipeline server tools ─────────────────────────────────────────────── + + +@mcp.tool() +def configure_pipeline_server( + namespace: str, + data_connection: str, + database: str = "MariaDB", +) -> str: + """Configure a pipeline server for a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + if not any(dc["name"] == data_connection for dc in dcs): + available = [dc["name"] for dc in dcs] + raise ValueError( + f"Data connection '{data_connection}' not found. Available: {available}" + ) + + PIPELINE_SERVERS[namespace] = { + "configured": True, + "data_connection": data_connection, + "status": "Ready", + "database": database, + } + PROJECTS[namespace]["pipeline_server"] = True + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "data_connection": data_connection, + "database": database, + }) + + +@mcp.tool() +def get_pipeline_server_status(namespace: str) -> str: + """Get the status of the pipeline server in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + ps = PIPELINE_SERVERS.get(namespace) + if not ps: + return json.dumps({"namespace": namespace, "configured": False}) + return json.dumps({ + "namespace": namespace, + "configured": ps["configured"], + "data_connection": ps["data_connection"], + "status": ps["status"], + "database": ps["database"], + }) + + +# ── Serving runtime creation ──────────────────────────────────────────── + + +@mcp.tool() +def create_serving_runtime( + namespace: str, + name: str, + display_name: str = "", + model_formats: list = None, + container_image: str = "", + container_port: int = 8080, + multi_model: bool = False, + api_protocol: str = "REST", +) -> str: + """Create a custom ServingRuntime in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + if not model_formats: + raise ValueError("model_formats must specify at least one model format") + + runtime = { + "name": name, + "display_name": display_name or name, + "model_formats": model_formats, + "requires_instantiation": False, + "source": "custom", + "api_protocol": api_protocol, + "container_image": container_image, + "container_port": container_port, + "multi_model": multi_model, + } + SERVING_RUNTIMES.setdefault(namespace, []).append(runtime) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "model_formats": model_formats, + }) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/instruction.md b/evaluation/without_skills/rh-ai-engineer__nim-setup/instruction.md new file mode 100644 index 00000000..f0b5fa2c --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__nim-setup/instruction.md @@ -0,0 +1,17 @@ +# NVIDIA NIM Setup Task + +You are an AI engineer on Red Hat OpenShift AI. Your team wants to deploy NVIDIA NIM for GPU-accelerated inference, but the cluster has not been set up for it yet. + +## Scenario +The ML team needs to serve models using NVIDIA's inference microservices. The cluster has GPUs available, but the necessary platform components and credentials have not been configured. You need to assess readiness and produce a complete setup plan. + +## Requirements +- Verify operator prerequisites (GPU Operator and NFD Operator) by checking their ClusterServiceVersion status +- Assess the current cluster state to determine what NIM infrastructure is already in place and what is missing +- Document the complete setup procedure including: the exact Kubernetes Secret manifests (with types, data key names, and structure) needed for NGC authentication, and the NIM Account custom resource with its correct API group and spec fields +- Provide the YAML manifests for each resource that needs to be created, using the correct RHOAI-specific API versions and resource naming conventions +- Flag any potential issues or blockers discovered during your assessment + +Document your assessment and setup plan in `/root/report.md`. + +Use MCP tools to interact with the platform. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/solution/solve.sh b/evaluation/without_skills/rh-ai-engineer__nim-setup/solution/solve.sh new file mode 100644 index 00000000..accbf7fe --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__nim-setup/solution/solve.sh @@ -0,0 +1,28 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# NIM Setup Plan + +## Prerequisites +- GPU Operator CSV in nvidia-gpu-operator namespace (gpu-operator-certified) +- NFD (Node Feature Discovery) in openshift-nfd + +## NGC Secrets +- API key secret: ngc-api-key (NGC_API_KEY) +- Image pull secret: ngc-image-pull-secret + - Registry: nvcr.io + - Username: $oauthtoken + - Password: NGC API key + +## NIM Account CR (nim.opendatahub.io/v1) +```yaml +apiVersion: nim.opendatahub.io/v1 +kind: Account +metadata: + name: nim-account +spec: + apiKeySecret: + name: ngc-api-key + imagePullSecret: + name: ngc-image-pull-secret +``` +REPORT_EOF diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/task.toml b/evaluation/without_skills/rh-ai-engineer__nim-setup/task.toml new file mode 100644 index 00000000..7b53288a --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__nim-setup/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-ai-engineer__nim-setup" +name = "rh-ai-engineer NVIDIA NIM Setup Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-ai-engineer", "nim-setup", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/tests/llm_judge.py b/evaluation/without_skills/rh-ai-engineer__nim-setup/tests/llm_judge.py new file mode 100644 index 00000000..a3c29b06 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__nim-setup/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "opendatahub_api_group", + "file": "/root/report.md", + "question": "Does the report use nim.opendatahub.io as the API group for the NIM Account custom resource, rather than the upstream nim.nvidia.com?", + "reference": "A skilled report specifies apiVersion: nim.opendatahub.io/v1 for the Account CR, which is the RHOAI-specific API group. An unskilled report uses nim.nvidia.com/v1alpha1 (the upstream NVIDIA API group) which is incorrect for Red Hat OpenShift AI." + }, + { + "id": "secret_naming_and_types", + "file": "/root/report.md", + "question": "Does the report create an image pull secret named ngc-image-pull-secret with type kubernetes.io/dockerconfigjson, and an API key secret with stringData containing the NGC_API_KEY field?", + "reference": "A skilled report creates ngc-image-pull-secret (type: kubernetes.io/dockerconfigjson) for nvcr.io registry access, and ngc-api-key (type: Opaque, stringData: NGC_API_KEY) for runtime auth. An unskilled report uses generic names like nvcr-credentials, kubectl shorthands without explicit types, or data.api_key instead of stringData.NGC_API_KEY." + }, + { + "id": "operator_csv_verification", + "file": "/root/report.md", + "question": "Does the report verify gpu-operator-certified and NFD (Node Feature Discovery) Operator as prerequisites, checking their ClusterServiceVersion status?", + "reference": "A skilled report checks for gpu-operator-certified (the specific CSV name, not just 'gpu-operator') and the NFD Operator in openshift-nfd namespace. An unskilled report either skips NFD entirely or uses generic gpu-operator references without the certified CSV name." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/tests/test.sh b/evaluation/without_skills/rh-ai-engineer__nim-setup/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__nim-setup/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/tests/test_outputs.py b/evaluation/without_skills/rh-ai-engineer__nim-setup/tests/test_outputs.py new file mode 100644 index 00000000..ad1f22ef --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__nim-setup/tests/test_outputs.py @@ -0,0 +1,89 @@ +""" +Tests for rh-ai-engineer__nim-setup per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert "nim" in content, "report should mention NIM" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 200, "report should have substantial content" + + +class TestSkillDependent: + def test_opendatahub_nim_api(self): + """Skill teaches nim.opendatahub.io as the RHOAI API group for NIM Account CR. + Without skill, agents use upstream nim.nvidia.com API group.""" + c = read_report() + assert "nim.opendatahub.io" in c, ( + "should use nim.opendatahub.io as the NIM Account CR API group (not nim.nvidia.com)" + ) + + def test_ngc_image_pull_secret_name(self): + """Skill teaches ngc-image-pull-secret as the specific secret name for nvcr.io. + Without skill, agents use generic names like nvcr-credentials.""" + c = read_report() + assert "ngc-image-pull-secret" in c, ( + "should use ngc-image-pull-secret as the image pull secret name" + ) + + def test_dockerconfigjson_secret_type(self): + """Skill teaches kubernetes.io/dockerconfigjson as the secret type for image pull. + Without skill, agents use kubectl docker-registry shorthand without explicit type.""" + c = read_report().lower() + assert "dockerconfigjson" in c, ( + "should specify dockerconfigjson as the image pull secret type" + ) + + def test_gpu_operator_certified_csv(self): + """Skill teaches checking gpu-operator-certified CSV by name. + Without skill, agents check generically for gpu-operator.""" + c = read_report().lower() + assert "gpu-operator-certified" in c, ( + "should verify gpu-operator-certified ClusterServiceVersion by name" + ) + + def test_nfd_operator_reference(self): + """Skill teaches verifying NFD (Node Feature Discovery) Operator as a prerequisite. + Without skill, agents skip NFD verification entirely.""" + c = read_report().lower() + assert "nfd" in c, ( + "should verify NFD (Node Feature Discovery) Operator as a prerequisite" + ) + + def test_stringdata_secret_field(self): + """Skill teaches using stringData in Secret YAML for NGC API key (no base64 needed). + Without skill, agents use kubectl --from-literal or data with base64.""" + c = read_report() + assert "stringData" in c or "stringdata" in c.lower(), ( + "should use stringData field in Secret YAML manifest for API key" + ) + + def test_nvidia_gpu_only(self): + """Docs emphasize NIM requires NVIDIA GPUs only; fallback to vLLM when + NVIDIA GPUs unavailable. Without docs, agents don't mention this constraint.""" + c = read_report().lower() + assert any(t in c for t in [ + "nvidia gpu", "nvidia only", "fallback", "vllm", + ]) and ("nim" in c or "gpu" in c), ( + "should note NIM requires NVIDIA GPUs with vLLM fallback" + ) diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/Dockerfile b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/Dockerfile new file mode 100644 index 00000000..aac4c84e --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/Dockerfile @@ -0,0 +1,67 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "rhoai": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-rhoai-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/common-issues.md b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/model-deploy.md b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/model-deploy.md new file mode 100644 index 00000000..dcd79e44 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/model-deploy.md @@ -0,0 +1,160 @@ +# /model-deploy Examples + +Example interactions demonstrating the model-deploy skill workflow. + +## Example 1: Deploy Llama 3.1 8B with vLLM + +``` +User: "Deploy Llama 3.1 8B on my cluster" + +Agent: [Invokes /model-deploy skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand runtime capabilities. + +Step 1 of 10: Gathering deployment information. + +| Setting | Value | Source | +|---------|-------|--------| +| Model | Llama 3.1 8B Instruct | user input | +| Runtime | TBD | auto-detect | +| Namespace | ? | needs input | +| Model Source | ? | needs input | +| Deployment Mode | Serverless | default | + +What namespace should I deploy to, and where are the model weights stored? +(e.g., S3 bucket, OCI registry, PVC) + +User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" + +Agent: Step 2 of 10: Determining runtime. + +I consulted supported-runtimes.md to understand runtime capabilities. + +Based on Llama 3.1 8B (open-source HuggingFace model): + +Recommended runtime: vLLM +Reason: Default runtime for open-source LLMs. Broad compatibility, + OpenAI-compatible API, no additional setup required. + +Proceed with vLLM? (yes / no / suggest alternative) + +User: "yes" + +Agent: Step 3 of 10: Looking up hardware profile. + +I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. + +| Requirement | Value | +|-------------|-------| +| GPUs | 1x A10G/L4/A100/H100 | +| VRAM | 16GB | +| Key Args | --max-model-len=4096 --tool-call-parser hermes | + +Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). +Ensure your cluster has at least 1 GPU with 16GB VRAM. + +Step 5 of 10: Skipped (NIM platform check not needed for vLLM). + +Step 6-7 of 10: Generating InferenceService YAML. + +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: llama-3-1-8b + namespace: ai-models + annotations: + serving.kserve.io/deploymentMode: Serverless +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: vllm-runtime + storageUri: s3://my-bucket/llama-3.1-8b + resources: + limits: + nvidia.com/gpu: "1" + requests: + cpu: "4" + memory: "16Gi" + args: + - --max-model-len=4096 + - --tool-call-parser=hermes + +| Setting | Value | +|---------|-------| +| Model | llama-3-1-8b | +| Runtime | vllm-runtime | +| GPUs | 1 | +| Namespace | ai-models | +| Mode | Serverless | + +Proceed with creating this InferenceService? (yes/no/modify) + +User: "yes" + +Agent: Step 8 of 10: Creating InferenceService... + [Creates via resources_create_or_update] + + InferenceService created. Monitoring rollout... + +Step 9 of 10: Monitoring rollout. + +| Stage | Status | +|-------|--------| +| Pod Scheduled | Complete | +| Image Pulled | Complete | +| Container Started | Complete | +| Model Loaded | Complete | +| Ready | Complete | + +Step 10 of 10: Deployment complete! + +Model Deployment Successful + +Model: llama-3-1-8b +Runtime: vLLM +Namespace: ai-models +GPUs: 1 + +Inference Endpoint: + URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com + API: OpenAI-compatible REST + +Quick Test: + curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' + +Next Steps: + Monitor performance: "/ai-observability llama-3-1-8b" +``` + +## Example 2: Deploy a Model with NIM (NIM Not Set Up) + +``` +User: "Deploy Llama 3.1 70B with NIM" + +Agent: [Invokes /model-deploy skill] + ... + Step 2: User requested NIM runtime. + + Step 5: Checking NIM platform... + +NIM platform is not set up in namespace ai-models. + +The NIM runtime requires a one-time platform setup. Would you like to: +1. Run /nim-setup now to configure the NIM platform +2. Switch to vLLM runtime instead (no setup required) +3. Abort deployment + +User: "1" + +Agent: [Invokes /nim-setup skill — walks through NGC credential setup] + ... + NIM Platform Setup Complete. + + Resuming model deployment... + [Continues from Step 6 with NIM runtime] +``` diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/nim-setup.md b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/nim-setup.md new file mode 100644 index 00000000..ccfc1a5d --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/nim-setup.md @@ -0,0 +1,115 @@ +# /nim-setup Examples + +Example interactions demonstrating the nim-setup skill workflow. + +## Example 1: First-Time NIM Setup + +``` +User: "Set up NIM on my cluster" + +Agent: [Invokes /nim-setup skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand NIM platform requirements. + +Let me verify the prerequisites for NIM setup. + +Step 1 of 7: Checking GPU Operator and NFD Operator... + +GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) +NFD Operator: nfd.v4.19.0 (Succeeded) + +Both operators are installed and healthy. + +Step 2 of 7: I need your NGC credentials. + +To set up NIM, I need your NVIDIA NGC API key. +You can generate one at: https://ngc.nvidia.com/setup/api-key + +Please provide: +1. Your NGC API key +2. The target namespace for NIM resources + +User: "Here's my key: [user provides NGC API key], namespace: ai-models" + +Agent: Step 3 of 7: Creating NGC image pull secret... + +I'll create a docker-registry secret for pulling NIM images from nvcr.io: + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-image-pull-secret + namespace: ai-models + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] + +Should I create this image pull secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + Image pull secret created successfully. + +Step 4 of 7: Creating NGC API key secret... + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-api-key + namespace: ai-models + type: Opaque + stringData: + NGC_API_KEY: "[REDACTED]" + +Should I create this API key secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + API key secret created successfully. + +Step 5 of 7: Creating NIM Account CR... + + apiVersion: nim.opendatahub.io/v1 + kind: Account + metadata: + name: nim-account + namespace: ai-models + spec: + apiKeySecret: + name: ngc-api-key + imagePullSecret: + name: ngc-image-pull-secret + +Should I create this NIM Account CR in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates Account CR via resources_create_or_update] + Account CR created successfully. + +Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) + +User: "no" + +Agent: Step 7 of 7: Validating NIM platform readiness... + +NIM Platform Setup Complete + +Account CR: `nim-account` (namespace: ai-models) + Status: Ready + +NGC Credentials: + Image Pull Secret: ngc-image-pull-secret [created] + API Key Secret: ngc-api-key [created] + +NIM ServingRuntimes Available: + - nim-llama-3-1-8b-instruct + - nim-llama-3-1-70b-instruct + - nim-mistral-7b-instruct + +Next Steps: + To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" +``` diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/live-doc-lookup.md b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/known-model-profiles.md b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/known-model-profiles.md new file mode 100644 index 00000000..40148981 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/known-model-profiles.md @@ -0,0 +1,83 @@ +--- +title: Known Model Hardware Profiles +category: references +tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] +semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] +use_cases: [model-deploy, debug-inference, ai-observability] +last_updated: 2026-03-10 +--- + +# Known Model Hardware Profiles + +Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). + +**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. + +## Model Source Conventions + +Each model lists a recommended `storageUri` with its authentication requirements: +- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** +- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). +- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. + +When the user does not specify a model source, use the `hf://` URI listed in the profile below. + +## Llama 3.x (Meta) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | +| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | +| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | +| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | + +- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) +- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly + +## Granite 3.x (IBM/Red Hat) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | +| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | + +- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) +- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser granite --chat-template` +- Red Hat-supported model family on RHOAI + +## Mixtral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | +| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) +- Mixture-of-Experts architecture: only ~13B/45B parameters active per token + +## Mistral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | +| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) + +## When a Model Is Not Listed + +If the requested model is not in this file, the agent MUST use the live doc lookup protocol: + +1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure +2. Fetch hardware specs from the appropriate source +3. Report findings to the user before proceeding with deployment + +Common cases requiring live lookup: +- Newly released models (after this file's last update) +- Domain-specific fine-tuned models +- Models with custom quantization +- NIM-specific optimized profiles diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/supported-runtimes.md b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/supported-runtimes.md new file mode 100644 index 00000000..9d018651 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/supported-runtimes.md @@ -0,0 +1,104 @@ +--- +title: Supported Serving Runtimes +category: references +tags: [runtimes, vllm, nim, caikit, tgis, serving] +semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] +use_cases: [model-deploy, serving-runtime-config, nim-setup] +last_updated: 2026-02-26 +--- + +# Supported Serving Runtimes + +This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. + +## Runtime Comparison + +| Runtime | API Type | Model Format | GPU Support | License | Setup Required | +|---------|----------|-------------|-------------|---------|----------------| +| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | +| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | +| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | +| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | + +## vLLM + +**Default runtime for most open-source models.** + +- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) +- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized +- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) +- **Model source**: S3-compatible storage, OCI registry, PVC, URI +- **Key features**: + - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) + - PagedAttention for efficient memory management + - Tensor parallelism for multi-GPU inference + - Tool/function calling support (`--tool-call-parser`) + - Continuous batching for high throughput +- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name + +## NVIDIA NIM + +**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** + +- **API**: OpenAI-compatible REST +- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog +- **GPU support**: NVIDIA only (requires specific GPU models per model profile) +- **Model source**: NGC model catalog (pulled automatically via NGC credentials) +- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) +- **Key features**: + - TensorRT-LLM optimization for lower latency + - Pre-compiled model engines (no compilation on first load) + - Optimized and generic profiles per GPU type + - Automatic model download from NGC +- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. +- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM +- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. +- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state + +## Caikit+TGIS + +**Red Hat's Caikit format with Text Generation Inference Server.** + +- **API**: gRPC (not REST) +- **Model formats**: Caikit format (requires conversion from HuggingFace) +- **GPU support**: NVIDIA +- **Model source**: S3-compatible storage +- **Key features**: + - Red Hat-supported runtime + - gRPC API for streaming inference + - Integrated with RHOAI model serving platform +- **When to choose**: When the model is already in Caikit format or when gRPC API is required. +- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name + +## Custom Runtimes + +**User-provided ServingRuntime CRs for unsupported frameworks.** + +- **API**: Defined by the custom runtime +- **Model formats**: Defined by the custom runtime +- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. +- **How to create**: Use `/serving-runtime-config` skill +- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility + +## Runtime Selection Decision Tree + +``` +Is the user's preferred runtime explicitly stated? +├── Yes → Use that runtime +└── No → Continue + +Is the model available in the NGC NIM catalog? +├── Yes → Suggest NIM (with vLLM as fallback) +│ Note: Requires /nim-setup and NVIDIA GPUs +└── No → Continue + +Is the model in Caikit format? +├── Yes → Caikit+TGIS +└── No → Continue + +Is the model a standard open-source LLM (HuggingFace-compatible)? +├── Yes → vLLM (default) +└── No → Custom runtime via /serving-runtime-config +``` diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/skill-conventions.md b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..cad5f77b --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,529 @@ +#!/usr/bin/env python3 +"""Mock OpenShift MCP server for SkillsBench rh-ai-engineer task. + +Simulates Kubernetes resource CRUD, pod management, logs, and events. + +Key scenario elements: +- LimitRange in namespaces: min CPU=100m, min memory=128Mi + (conflicts with KServe sidecar containers hardcoded at 10m CPU/15Mi memory) +- GPU node with custom taint ai-workload=true:NoSchedule +- NIM Account CR in ml-production: not ready (NGC credentials invalid) +- text-gen-legacy pods: OOMKilled (max-model-len=32768 on A10G) +- nim-llama-prod: no pods created (Account CR not ready) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + +# ── Cluster state ──────────────────────────────────────────────────────── + +GPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "gpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + "nvidia.com/gpu.present": "true", + "nvidia.com/gpu.product": "NVIDIA-A10G", + }, + }, + "spec": { + "taints": [ + { + "key": "ai-workload", + "value": "true", + "effect": "NoSchedule", + }, + ], + }, + "status": { + "allocatable": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "capacity": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "conditions": [ + {"type": "Ready", "status": "True"}, + ], + }, +} + +CPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "cpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + }, + }, + "spec": {"taints": []}, + "status": { + "allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "capacity": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +MASTER_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "master-1", + "labels": { + "node-role.kubernetes.io/master": "", + "node-role.kubernetes.io/control-plane": "", + }, + }, + "spec": { + "taints": [ + {"key": "node-role.kubernetes.io/master", "effect": "NoSchedule"}, + ], + }, + "status": { + "allocatable": {"cpu": "8", "memory": "32Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +ALL_NODES = [GPU_NODE, CPU_NODE, MASTER_NODE] + +# LimitRange applied by cluster policy to all DS project namespaces +NAMESPACE_LIMITRANGE = { + "apiVersion": "v1", + "kind": "LimitRange", + "metadata": { + "name": "default-limits", + }, + "spec": { + "limits": [ + { + "type": "Container", + "default": { + "cpu": "2", + "memory": "4Gi", + }, + "defaultRequest": { + "cpu": "500m", + "memory": "256Mi", + }, + "min": { + "cpu": "100m", + "memory": "128Mi", + }, + "max": { + "cpu": "32", + "memory": "128Gi", + }, + }, + ], + }, +} + +NIM_ACCOUNT_CR = { + "apiVersion": "nim.opendatahub.io/v1", + "kind": "Account", + "metadata": { + "name": "nim-account", + "namespace": "ml-production", + }, + "spec": { + "apiKeySecret": { + "name": "ngc-api-key", + }, + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "NGCCredentialsInvalid", + "message": "NGC API key validation failed: 401 Unauthorized. " + "The API key in secret 'ngc-api-key' is expired or invalid. " + "Re-create the secret with a valid NGC API key from " + "https://ngc.nvidia.com/setup/api-key and restart the " + "Account reconciliation.", + "lastTransitionTime": "2026-03-14T12:00:00Z", + }, + ], + "nimPullSecretStatus": "Failed", + "nimConfigStatus": "Pending", + }, +} + +SERVING_RUNTIME_VLLM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "vllm-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "vLLM", "version": "1", "autoSelect": True}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "quay.io/modh/vllm:rhoai-2.16", + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + }, + ], + }, +} + +SERVING_RUNTIME_NIM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "nim-serving-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "NIM", "version": "1"}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "nvcr.io/nim/meta/llama-3.1-8b-instruct:latest", + "ports": [{"containerPort": 8000, "protocol": "TCP"}], + "env": [ + {"name": "NGC_API_KEY", "valueFrom": { + "secretKeyRef": {"name": "ngc-api-key", "key": "api_key"}, + }}, + ], + }, + ], + }, +} + +PODS_BY_NAMESPACE = { + "ml-production": [ + { + "name": "text-gen-legacy-predictor-00001-abc12", + "namespace": "ml-production", + "status": "CrashLoopBackOff", + "restarts": 5, + "node": "gpu-worker-1", + "containers": [ + { + "name": "kserve-container", + "state": "waiting", + "reason": "CrashLoopBackOff", + "last_termination_reason": "OOMKilled", + "last_termination_exit_code": 137, + }, + ], + "labels": { + "serving.kserve.io/inferenceservice": "text-gen-legacy", + }, + "gpu": "1", + }, + # nim-llama-prod: NO pods created (Account CR not ready) + ], +} + +POD_LOGS = { + "text-gen-legacy-predictor-00001-abc12": ( + "INFO 2026-03-01 10:00:00 vllm_engine.py:125] vLLM engine starting...\n" + "INFO 2026-03-01 10:00:01 config.py:89] Model: mistralai/Mistral-7B-Instruct-v0.3\n" + "INFO 2026-03-01 10:00:01 config.py:92] max_model_len = 32768\n" + "INFO 2026-03-01 10:00:02 gpu_executor.py:45] GPU 0: NVIDIA A10G (24576 MiB)\n" + "INFO 2026-03-01 10:00:03 model_runner.py:88] Loading model weights...\n" + "INFO 2026-03-01 10:00:15 model_runner.py:112] Model weights loaded: 13.5 GiB\n" + "INFO 2026-03-01 10:00:15 worker.py:201] Allocating KV cache...\n" + "ERROR 2026-03-01 10:00:16 worker.py:215] torch.cuda.OutOfMemoryError: " + "CUDA out of memory. Tried to allocate 28.5 GiB for KV cache but only " + "10.1 GiB available after loading model weights (13.5 GiB).\n" + "ERROR 2026-03-01 10:00:16 vllm_engine.py:178] Engine failed to start\n" + "Traceback (most recent call last):\n" + " File \"/opt/vllm/vllm/engine/engine.py\", line 175, in start\n" + " self._init_kv_cache()\n" + " File \"/opt/vllm/vllm/worker/worker.py\", line 215, in _init_kv_cache\n" + " raise torch.cuda.OutOfMemoryError(msg)\n" + "torch.cuda.OutOfMemoryError: CUDA out of memory\n" + ), +} + +EVENTS_BY_NAMESPACE = { + "ml-production": [ + { + "type": "Warning", + "reason": "BackOff", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Back-off restarting failed container kserve-container in pod " + "text-gen-legacy-predictor-00001-abc12", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Warning", + "reason": "OOMKilled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Container kserve-container was OOMKilled (exit code 137). " + "GPU memory exhausted during KV cache allocation.", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Normal", + "reason": "Scheduled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Successfully assigned ml-production/" + "text-gen-legacy-predictor-00001-abc12 to gpu-worker-1", + "count": 1, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-02-28T08:00:00Z", + }, + { + "type": "Warning", + "reason": "NIMAccountNotReady", + "object": "InferenceService/nim-llama-prod", + "message": "NIM Account 'nim-account' in namespace 'ml-production' " + "is not ready", + "count": 12, + "first_timestamp": "2026-03-14T12:00:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + { + "type": "Warning", + "reason": "ImagePullBackOff", + "object": "InferenceService/nim-llama-prod", + "message": "Failed to pull image 'nvcr.io/nim/meta/llama-3.1-8b-instruct:" + "latest': unauthorized: authentication required", + "count": 8, + "first_timestamp": "2026-03-14T12:05:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + ], +} + + +# ── Resource tools ─────────────────────────────────────────────────────── + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: str = "", +) -> str: + """Get a single Kubernetes resource by apiVersion, kind, and name.""" + if kind == "Node": + for node in ALL_NODES: + if node["metadata"]["name"] == name: + return json.dumps(node, indent=2) + raise ValueError(f"Node '{name}' not found") + + if kind == "ServingRuntime": + if name == "vllm-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_VLLM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + if name == "nim-serving-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_NIM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + raise ValueError(f"ServingRuntime '{name}' not found in namespace '{namespace}'") + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps(lr, indent=2) + + if kind == "Account" and "nim" in apiVersion.lower(): + if namespace == "ml-production" and name == "nim-account": + return json.dumps(NIM_ACCOUNT_CR, indent=2) + raise ValueError( + f"Account '{name}' not found in namespace '{namespace}'" + ) + + if kind == "ClusterVersion" and apiVersion == "config.openshift.io/v1": + return json.dumps({ + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterVersion", + "metadata": {"name": "version"}, + "status": {"desired": {"version": "4.16.3"}}, + }) + + raise ValueError(f"Resource {apiVersion}/{kind}/{name} not found") + + +@mcp.tool() +def resources_create_or_update( + api_version: str, + kind: str, + namespace: str, + name: str, + body: str, +) -> str: + """Create or update a Kubernetes resource. Accepts apiVersion, kind, namespace, name, and body (JSON).""" + try: + resource = json.loads(body) + except json.JSONDecodeError as e: + raise ValueError(f"Invalid JSON body: {e}") from e + + resource.setdefault("metadata", {}) + resource["metadata"]["name"] = name + resource["metadata"]["namespace"] = namespace + resource["apiVersion"] = api_version + resource["kind"] = kind + + if kind == "ServingRuntime": + resource.setdefault("status", {}) + resource["status"]["conditions"] = [ + { + "type": "Ready", + "status": "True", + "reason": "ServingRuntimeReady", + "message": "ServingRuntime is ready", + "lastTransitionTime": "2026-03-17T12:00:00Z", + }, + ] + return json.dumps({ + "status": "created", + "resource": resource, + "message": f"ServingRuntime '{name}' created/updated in namespace '{namespace}'", + }, indent=2) + + if kind == "Secret": + resource.setdefault("type", "Opaque") + return json.dumps({ + "status": "created", + "resource": resource, + "message": f"Secret '{name}' created/updated in namespace '{namespace}'", + }, indent=2) + + if kind in ("NIMAccount", "Account") and "nim" in api_version.lower(): + resource.setdefault("status", {}) + resource["status"]["conditions"] = [ + { + "type": "Ready", + "status": "True", + "reason": "NGCCredentialsValid", + "message": "NGC API key validated successfully", + "lastTransitionTime": "2026-03-17T12:00:00Z", + }, + ] + return json.dumps({ + "status": "created", + "resource": resource, + "message": f"NIM Account '{name}' created/updated in namespace '{namespace}'", + }, indent=2) + + if kind == "ConfigMap": + return json.dumps({ + "status": "created", + "resource": resource, + "message": f"ConfigMap '{name}' created/updated in namespace '{namespace}'", + }, indent=2) + + raise ValueError(f"Unsupported kind for create/update: {kind}") + + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: str = "", + labelSelector: str = "", +) -> str: + """List Kubernetes resources by apiVersion and kind.""" + if kind == "Node": + nodes = ALL_NODES + if labelSelector: + parts = labelSelector.split("=", 1) + key = parts[0] + value = parts[1] if len(parts) > 1 else "" + nodes = [ + n for n in nodes + if n["metadata"]["labels"].get(key) == value + ] + return json.dumps(nodes, indent=2) + + if kind == "Service" and apiVersion == "serving.knative.dev/v1": + return json.dumps({ + "kind": "ServiceList", + "apiVersion": "serving.knative.dev/v1", + "items": [], + "metadata": {}, + }) + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps({ + "kind": "LimitRangeList", + "items": [lr], + }) + + if kind == "InferenceService": + return json.dumps({ + "kind": "InferenceServiceList", + "items": [], + }) + + raise ValueError(f"Unsupported list: {apiVersion}/{kind}") + + +@mcp.tool() +def pods_list( + namespace: str, + labelSelector: str = "", +) -> str: + """List pods in a namespace with optional label selector.""" + pods = PODS_BY_NAMESPACE.get(namespace, []) + + if labelSelector: + key, _, value = labelSelector.partition("=") + pods = [p for p in pods if p.get("labels", {}).get(key) == value] + + results = [] + for pod in pods: + results.append({ + "name": pod["name"], + "namespace": pod["namespace"], + "status": pod["status"], + "restarts": pod.get("restarts", 0), + "node": pod.get("node", ""), + "containers": pod.get("containers", []), + "gpu": pod.get("gpu", "0"), + }) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def pods_log( + namespace: str, + name: str, + container: str = "", +) -> str: + """Get logs from a pod container.""" + logs = POD_LOGS.get(name) + if logs is None: + raise ValueError(f"Pod '{name}' not found in namespace '{namespace}'") + return logs + + +@mcp.tool() +def events_list(namespace: str) -> str: + """List events in a namespace.""" + events = EVENTS_BY_NAMESPACE.get(namespace, []) + return json.dumps(events, indent=2) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/mcp-servers/mock-rhoai-mcp.py b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/mcp-servers/mock-rhoai-mcp.py new file mode 100644 index 00000000..0ae9e4cb --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/mcp-servers/mock-rhoai-mcp.py @@ -0,0 +1,780 @@ +#!/usr/bin/env python3 +"""Mock RHOAI MCP server for SkillsBench rh-ai-engineer task. + +Simulates Red Hat OpenShift AI operations: Data Science Projects, +model serving, data connections, serving runtimes, inference services. + +Scenario: +- ml-production: existing project with two broken deployments + - text-gen-legacy: vLLM OOMKilled (max-model-len=32768 on A10G) + - nim-llama-prod: NIM failing (Account CR not ready, NGC creds invalid) +- fraud-detection: does not exist yet (agent creates it) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("rhoai") + +# ── In-memory state ────────────────────────────────────────────────────── + +PROJECTS = { + "ml-production": { + "name": "ml-production", + "display_name": "ML Production", + "description": "Production ML workloads", + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": "single", + "pipeline_server": True, + }, +} + +DATA_CONNECTIONS = { + "ml-production": [ + { + "name": "prod-model-store", + "type": "S3", + "bucket": "ml-models-prod", + "endpoint": "https://s3.us-east-1.amazonaws.com", + "region": "us-east-1", + }, + ], +} + +SERVING_RUNTIMES = { + "__platform_templates__": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "REST", + "supported_model_formats": [ + {"name": "vLLM", "version": "1", "autoSelect": True} + ], + }, + { + "name": "caikit-tgis-runtime", + "display_name": "Caikit+TGIS ServingRuntime", + "model_formats": ["caikit"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "gRPC", + }, + ], + "ml-production": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + { + "name": "nim-serving-runtime", + "display_name": "NVIDIA NIM ServingRuntime", + "model_formats": ["NIM"], + "requires_instantiation": False, + "source": "nim-account", + "api_protocol": "REST", + }, + { + "name": "ovms-1", + "display_name": "OpenVINO Model Server", + "model_formats": ["openvino_ir", "onnx"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + ], +} + +INFERENCE_SERVICES = { + "ml-production": { + "text-gen-legacy": { + "name": "text-gen-legacy", + "namespace": "ml-production", + "runtime": "vllm-runtime", + "model_format": "vLLM", + "storage_uri": "hf://mistralai/Mistral-7B-Instruct-v0.3", + "display_name": "Mistral 7B Legacy", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "16Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "PredictorFailed", + "message": "Predictor pod is not ready", + }, + { + "type": "PredictorReady", + "status": "False", + "reason": "ContainerCrashLoop", + "message": "Container kserve-container terminated: " + "OOMKilled (exit code 137). 5 restarts.", + }, + { + "type": "IngressReady", + "status": "True", + "reason": "IngressReady", + "message": "Ingress is ready", + }, + ], + "age": "3d", + }, + "nim-llama-prod": { + "name": "nim-llama-prod", + "namespace": "ml-production", + "runtime": "nim-serving-runtime", + "model_format": "NIM", + "storage_uri": "nim://meta/llama-3.1-8b-instruct", + "display_name": "Llama 3.1 8B (NIM)", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "32Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "RuntimeNotReady", + "message": "ServingRuntime 'nim-serving-runtime' " + "is not in ready state", + }, + { + "type": "PredictorReady", + "status": "Unknown", + "reason": "PodNotCreated", + "message": "Predictor pod has not been created. " + "Waiting for ServingRuntime to become ready.", + }, + { + "type": "IngressReady", + "status": "Unknown", + "reason": "PredictorNotReady", + "message": "Waiting for predictor to become ready", + }, + ], + "age": "1d", + }, + }, +} + +DEPLOYED_MODELS = {} + +WORKBENCHES = { + "ml-production": [ + { + "name": "data-exploration-nb", + "display_name": "Data Exploration", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Running", + "cpu_request": "1", + "memory_request": "8Gi", + "gpu_count": 0, + "pvc_name": "data-exploration-nb-pvc", + "pvc_size": "20Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-10T09:00:00Z", + }, + { + "name": "model-training-nb", + "display_name": "Model Training", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Stopped", + "cpu_request": "4", + "memory_request": "16Gi", + "gpu_count": 1, + "pvc_name": "model-training-nb-pvc", + "pvc_size": "50Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-15T14:00:00Z", + }, + ], +} + +PIPELINE_SERVERS = { + "ml-production": { + "configured": True, + "data_connection": "prod-model-store", + "status": "Ready", + "database": "MariaDB", + }, +} + +NOTEBOOK_IMAGES = [ + {"name": "jupyter-pytorch-ubi9-python-3.9-2024.1", "display_name": "PyTorch 2024.1", "packages": ["torch", "transformers"]}, + {"name": "jupyter-tensorflow-ubi9-python-3.9-2024.1", "display_name": "TensorFlow 2024.1", "packages": ["tensorflow"]}, + {"name": "jupyter-datascience-ubi9-python-3.9-2024.1", "display_name": "Standard Data Science", "packages": ["pandas", "scikit-learn"]}, + {"name": "jupyter-minimal-ubi9-python-3.9-2024.1", "display_name": "Minimal Python", "packages": []}, +] + + +# ── Project tools ──────────────────────────────────────────────────────── + + +@mcp.tool() +def list_data_science_projects() -> str: + """List all RHOAI Data Science Projects on the cluster.""" + projects = [] + for name, proj in PROJECTS.items(): + projects.append({ + "name": name, + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + }) + return json.dumps(projects, indent=2) + + +@mcp.tool() +def create_data_science_project( + name: str, + display_name: str, + description: str = "", +) -> str: + """Create a new RHOAI Data Science Project (namespace with dashboard labels).""" + if name in PROJECTS: + raise ValueError( + f"Project '{name}' already exists. Choose a different name " + "or configure the existing project." + ) + if not name.replace("-", "").replace("_", "").isalnum() or len(name) > 63: + raise ValueError( + f"Invalid project name '{name}'. Must be DNS-compatible: " + "lowercase alphanumeric and hyphens, max 63 chars." + ) + + PROJECTS[name] = { + "name": name, + "display_name": display_name, + "description": description, + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": None, + "pipeline_server": False, + } + DATA_CONNECTIONS[name] = [] + SERVING_RUNTIMES[name] = [] + INFERENCE_SERVICES[name] = {} + + return json.dumps({ + "status": "created", + "name": name, + "display_name": display_name, + "namespace": name, + "labels": {"opendatahub.io/dashboard": "true"}, + }) + + +@mcp.tool() +def get_project_details(name: str) -> str: + """Get detailed information about an RHOAI Data Science Project.""" + if name not in PROJECTS: + raise ValueError(f"Project '{name}' not found") + proj = PROJECTS[name] + dc_count = len(DATA_CONNECTIONS.get(name, [])) + isvc_count = len(INFERENCE_SERVICES.get(name, {})) + return json.dumps({ + "name": proj["name"], + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "labels": proj["labels"], + "data_connections": dc_count, + "inference_services": isvc_count, + "model_serving_mode": proj.get("model_serving_mode"), + "pipeline_server": proj.get("pipeline_server", False), + }) + + +@mcp.tool() +def get_project_status(namespace: str) -> str: + """Get comprehensive status of an RHOAI Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Project '{namespace}' not found") + proj = PROJECTS[namespace] + dcs = DATA_CONNECTIONS.get(namespace, []) + isvcs = INFERENCE_SERVICES.get(namespace, {}) + return json.dumps({ + "namespace": namespace, + "display_name": proj["display_name"], + "status": "Active", + "components": { + "data_connections": len(dcs), + "inference_services": len(isvcs), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + "pipeline_server": "configured" if proj.get("pipeline_server") else "not configured", + }, + }) + + +# ── Data connection tools ──────────────────────────────────────────────── + + +@mcp.tool() +def create_s3_data_connection( + namespace: str, + name: str, + bucket: str, + endpoint: str, + access_key: str, + secret_key: str, + region: str = "", +) -> str: + """Create an S3-compatible data connection in an RHOAI project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + existing = DATA_CONNECTIONS.get(namespace, []) + if any(dc["name"] == name for dc in existing): + raise ValueError( + f"Data connection '{name}' already exists in namespace '{namespace}'" + ) + + dc = { + "name": name, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + "region": region, + } + DATA_CONNECTIONS.setdefault(namespace, []).append(dc) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + }) + + +@mcp.tool() +def list_data_connections(namespace: str) -> str: + """List data connections in an RHOAI project namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + return json.dumps(dcs, indent=2) + + +# ── Model serving tools ───────────────────────────────────────────────── + + +@mcp.tool() +def set_model_serving_mode(namespace: str, mode: str) -> str: + """Enable model serving on a Data Science Project (single or multi mode).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + if mode not in ("single", "multi"): + raise ValueError(f"Invalid mode '{mode}'. Must be 'single' or 'multi'.") + + PROJECTS[namespace]["model_serving_mode"] = mode + + if not SERVING_RUNTIMES.get(namespace): + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + SERVING_RUNTIMES[namespace] = [ + {**t, "requires_instantiation": False, "source": "existing"} + for t in templates + ] + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "mode": mode, + }) + + +@mcp.tool() +def list_serving_runtimes( + namespace: str, + include_templates: bool = False, +) -> str: + """List available ServingRuntimes in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + runtimes = list(SERVING_RUNTIMES.get(namespace, [])) + if include_templates: + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + existing_names = {r["name"] for r in runtimes} + for t in templates: + if t["name"] not in existing_names: + runtimes.append(t) + + return json.dumps(runtimes, indent=2) + + +# ── Inference service tools ────────────────────────────────────────────── + + +@mcp.tool() +def deploy_model( + name: str, + namespace: str, + runtime: str, + model_format: str, + storage_uri: str, + display_name: str = "", + min_replicas: int = 1, + max_replicas: int = 1, + cpu_request: str = "1", + cpu_limit: str = "2", + memory_request: str = "4Gi", + memory_limit: str = "8Gi", + gpu_count: int = 0, +) -> str: + """Deploy an AI/ML model as a KServe InferenceService.""" + if namespace not in PROJECTS: + raise ValueError( + f"Namespace '{namespace}' is not a Data Science Project. " + "Create one via create_data_science_project first." + ) + + ns_runtimes = SERVING_RUNTIMES.get(namespace, []) + runtime_names = [r["name"] for r in ns_runtimes] + if runtime not in runtime_names: + available = ", ".join(runtime_names) or "none" + raise ValueError( + f"ServingRuntime '{runtime}' not found in namespace '{namespace}'. " + f"Available runtimes: {available}" + ) + + endpoint = f"https://{name}-{namespace}.apps.ocp-cluster.example.com" + isvc = { + "name": name, + "namespace": namespace, + "runtime": runtime, + "model_format": model_format, + "storage_uri": storage_uri, + "display_name": display_name or name, + "gpu_count": gpu_count, + "cpu_request": cpu_request, + "memory_request": memory_request, + "min_replicas": min_replicas, + "max_replicas": max_replicas, + "ready": True, + "url": endpoint, + "conditions": [ + {"type": "Ready", "status": "True", "reason": "Ready", "message": ""}, + {"type": "PredictorReady", "status": "True", "reason": "PodReady", "message": ""}, + {"type": "IngressReady", "status": "True", "reason": "IngressReady", "message": ""}, + ], + "age": "0s", + } + + INFERENCE_SERVICES.setdefault(namespace, {})[name] = isvc + DEPLOYED_MODELS[f"{namespace}/{name}"] = isvc + + return json.dumps({ + "status": "deployed", + "name": name, + "namespace": namespace, + "runtime": runtime, + "endpoint": endpoint, + "ready": True, + }) + + +@mcp.tool() +def list_inference_services( + namespace: str, + verbosity: str = "standard", +) -> str: + """List deployed InferenceServices in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + isvcs = INFERENCE_SERVICES.get(namespace, {}) + results = [] + for isvc_name, isvc in isvcs.items(): + entry = { + "name": isvc["name"], + "runtime": isvc["runtime"], + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "age": isvc.get("age", ""), + } + if verbosity == "full": + entry["conditions"] = isvc.get("conditions", []) + entry["storage_uri"] = isvc.get("storage_uri", "") + entry["gpu_count"] = isvc.get("gpu_count", 0) + results.append(entry) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def get_inference_service( + name: str, + namespace: str, + verbosity: str = "standard", +) -> str: + """Get detailed status of a specific InferenceService.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + + isvc = isvcs[name] + result = { + "name": isvc["name"], + "namespace": isvc["namespace"], + "runtime": isvc["runtime"], + "model_format": isvc.get("model_format", ""), + "storage_uri": isvc.get("storage_uri", ""), + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "conditions": isvc.get("conditions", []), + "gpu_count": isvc.get("gpu_count", 0), + "replicas": {"min": isvc.get("min_replicas", 1), "max": isvc.get("max_replicas", 1)}, + "resources": { + "cpu_request": isvc.get("cpu_request", "1"), + "memory_request": isvc.get("memory_request", "4Gi"), + "memory_limit": isvc.get("memory_limit", "8Gi"), + }, + "age": isvc.get("age", ""), + } + return json.dumps(result, indent=2) + + +@mcp.tool() +def get_model_endpoint(name: str, namespace: str) -> str: + """Get the inference endpoint URL for a deployed model.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + isvc = isvcs[name] + if not isvc["ready"]: + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": "", + "error": "InferenceService is not ready. Check conditions for details.", + }) + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": isvc["url"], + }) + + +# ── Workbench tools ────────────────────────────────────────────────────── + + +@mcp.tool() +def list_workbenches(namespace: str) -> str: + """List workbenches (Jupyter notebooks) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + return json.dumps(wbs, indent=2) + + +@mcp.tool() +def create_workbench( + namespace: str, + name: str, + display_name: str = "", + image: str = "jupyter-datascience-ubi9-python-3.9-2024.1", + cpu_request: str = "1", + memory_request: str = "4Gi", + gpu_count: int = 0, + pvc_size: str = "20Gi", +) -> str: + """Create a new workbench (Jupyter notebook) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + valid_images = [img["name"] for img in NOTEBOOK_IMAGES] + if image not in valid_images: + raise ValueError( + f"Image '{image}' not found. Available: {', '.join(valid_images)}" + ) + + wb = { + "name": name, + "display_name": display_name or name, + "image": image, + "status": "Running", + "cpu_request": cpu_request, + "memory_request": memory_request, + "gpu_count": gpu_count, + "pvc_name": f"{name}-pvc", + "pvc_size": pvc_size, + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-03-02T12:00:00Z", + } + WORKBENCHES.setdefault(namespace, []).append(wb) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "image": image, + "pvc": f"{name}-pvc", + }) + + +@mcp.tool() +def stop_workbench(namespace: str, name: str) -> str: + """Stop a running workbench (preserves data).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Stopped" + return json.dumps({"status": "stopped", "name": name, "namespace": namespace}) + + +@mcp.tool() +def start_workbench(namespace: str, name: str) -> str: + """Start a stopped workbench.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Running" + return json.dumps({"status": "running", "name": name, "namespace": namespace}) + + +@mcp.tool() +def delete_workbench(namespace: str, name: str) -> str: + """Delete a workbench. WARNING: PVC data may be lost if not backed up.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wbs.remove(wb) + return json.dumps({ + "status": "deleted", + "name": name, + "namespace": namespace, + "warning": "Associated PVC data has been deleted", + }) + + +@mcp.tool() +def list_notebook_images() -> str: + """List available notebook images for workbench creation.""" + return json.dumps(NOTEBOOK_IMAGES, indent=2) + + +# ── Pipeline server tools ─────────────────────────────────────────────── + + +@mcp.tool() +def configure_pipeline_server( + namespace: str, + data_connection: str, + database: str = "MariaDB", +) -> str: + """Configure a pipeline server for a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + if not any(dc["name"] == data_connection for dc in dcs): + available = [dc["name"] for dc in dcs] + raise ValueError( + f"Data connection '{data_connection}' not found. Available: {available}" + ) + + PIPELINE_SERVERS[namespace] = { + "configured": True, + "data_connection": data_connection, + "status": "Ready", + "database": database, + } + PROJECTS[namespace]["pipeline_server"] = True + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "data_connection": data_connection, + "database": database, + }) + + +@mcp.tool() +def get_pipeline_server_status(namespace: str) -> str: + """Get the status of the pipeline server in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + ps = PIPELINE_SERVERS.get(namespace) + if not ps: + return json.dumps({"namespace": namespace, "configured": False}) + return json.dumps({ + "namespace": namespace, + "configured": ps["configured"], + "data_connection": ps["data_connection"], + "status": ps["status"], + "database": ps["database"], + }) + + +# ── Serving runtime creation ──────────────────────────────────────────── + + +@mcp.tool() +def create_serving_runtime( + namespace: str, + name: str, + display_name: str = "", + model_formats: list = None, + container_image: str = "", + container_port: int = 8080, + multi_model: bool = False, + api_protocol: str = "REST", +) -> str: + """Create a custom ServingRuntime in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + if not model_formats: + raise ValueError("model_formats must specify at least one model format") + + runtime = { + "name": name, + "display_name": display_name or name, + "model_formats": model_formats, + "requires_instantiation": False, + "source": "custom", + "api_protocol": api_protocol, + "container_image": container_image, + "container_port": container_port, + "multi_model": multi_model, + } + SERVING_RUNTIMES.setdefault(namespace, []).append(runtime) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "model_formats": model_formats, + }) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/instruction.md b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/instruction.md new file mode 100644 index 00000000..d89e7c6a --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/instruction.md @@ -0,0 +1,19 @@ +# Serving Runtime Configuration Task + +You are an AI engineer on Red Hat OpenShift AI. Your team needs to serve a model using a custom inference engine that is not available as a default runtime on the platform. + +## Scenario +The existing platform-provided serving runtimes do not support the model format your team needs. You must create a custom runtime configuration that integrates properly with the platform and can be used to deploy models. + +## Requirements +- Examine the currently available serving runtimes and platform templates, distinguishing which are already instantiated versus which require instantiation before use +- Design a custom ServingRuntime CR that specifies the inference container, supported model formats, resource requirements, and API protocol +- Follow KServe container naming conventions so the runtime integrates correctly with the platform's model serving framework +- For runtimes supporting multiple model formats, explain how autoSelect should be configured to avoid format conflicts +- Explain where GPU resource allocation belongs (in the ServingRuntime vs in the InferenceService) and why +- Ensure the runtime will be visible and usable from the platform dashboard +- Document your design decisions and trade-offs + +Document your configuration plan and the complete runtime specification in `/root/report.md`. + +Use MCP tools to interact with the platform. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/solution/solve.sh b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/solution/solve.sh new file mode 100644 index 00000000..043771f9 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/solution/solve.sh @@ -0,0 +1,31 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# ServingRuntime Configuration + +## Custom Runtime: triton-onnx + +Platform templates: list_serving_runtimes with include_templates: true. Templates with requires_instantiation: true use create_serving_runtime. + +```yaml +apiVersion: serving.kserve.io/v1alpha1 +kind: ServingRuntime +metadata: + name: triton-onnx-runtime + labels: + opendatahub.io/dashboard: "true" +spec: + supportedModelFormats: + - name: onnx + version: "1" + autoSelect: true + multiModel: false + containers: + - name: kserve-container + image: nvcr.io/nvidia/tritonserver:latest + ports: + - containerPort: 8080 + protocol: TCP +``` + +### Key: supportedModelFormats.name must match InferenceService modelFormat.name +REPORT_EOF diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/task.toml b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/task.toml new file mode 100644 index 00000000..8ee93afa --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-ai-engineer__serving-runtime-config" +name = "rh-ai-engineer Serving Runtime Configuration Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-ai-engineer", "serving-runtime-config", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/tests/llm_judge.py b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/tests/llm_judge.py new file mode 100644 index 00000000..11fdec60 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "kserve_container_naming", + "file": "/root/report.md", + "question": "Does the ServingRuntime YAML in the report name the main container 'kserve-container' (the required KServe naming convention)?", + "reference": "A skilled report names the container kserve-container in the ServingRuntime spec, which is required by KServe for the model serving framework to function correctly. An unskilled report might use a framework-specific name like 'triton' or 'vllm', which would cause KServe integration issues." + }, + { + "id": "gpu_allocation_strategy", + "file": "/root/report.md", + "question": "Does the report explain that GPU resources should NOT be hardcoded in the ServingRuntime and instead should be allocated at the InferenceService level for flexibility?", + "reference": "A skilled report explains that GPU resources (nvidia.com/gpu) belong at the InferenceService deployment level because different models need 0, 1, or multiple GPUs. The ServingRuntime should remain GPU-agnostic. An unskilled report hardcodes nvidia.com/gpu: 1 directly in the ServingRuntime spec." + }, + { + "id": "autoselect_and_api_conventions", + "file": "/root/report.md", + "question": "Does the report configure autoSelect: false for non-primary model formats and use the correct ServingRuntime API version (v1alpha1)?", + "reference": "A skilled report uses autoSelect: true only for the primary format and false for secondary formats to prevent conflicts, and uses the serving.kserve.io/v1alpha1 API version for ServingRuntime (distinct from v1beta1 used for InferenceService). An unskilled report sets autoSelect: true for all formats or uses the wrong API version." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/tests/test.sh b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/tests/test_outputs.py b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/tests/test_outputs.py new file mode 100644 index 00000000..71257bf2 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/tests/test_outputs.py @@ -0,0 +1,97 @@ +""" +Tests for rh-ai-engineer__serving-runtime-config per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ["servingruntime", "serving runtime", "runtime"]), ( + "report should mention ServingRuntime" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 200, "report should have substantial content" + + +class TestSkillDependent: + def test_kserve_container_name(self): + """Skill teaches the main container MUST be named kserve-container for KServe + compatibility. Without skill, agents use framework-specific names like 'triton'.""" + c = read_report() + assert "kserve-container" in c, ( + "should name the main container 'kserve-container' (required by KServe)" + ) + + def test_serving_runtime_api_version(self): + """Skill teaches ServingRuntime uses serving.kserve.io/v1alpha1 API (alpha, + not beta like InferenceService). Without skill, agents use v1beta1 or omit + the apiVersion distinction between ServingRuntime and InferenceService.""" + c = read_report() + assert "v1alpha1" in c or ( + "alpha" in c.lower() and "serving" in c.lower() + ), "should use v1alpha1 API version for ServingRuntime" + + def test_autoselect_false_for_secondary(self): + """Skill teaches using autoSelect: true only for primary format and false for + secondary formats to avoid conflicts. Without skill, agents set true for all.""" + c = read_report().lower() + assert "autoselect: false" in c or "autoselect\":false" in c or "autoselect\": false" in c, ( + "should use autoSelect: false for non-primary model formats" + ) + + def test_gpu_at_inferenceservice_level(self): + """Skill teaches not hardcoding GPU in ServingRuntime; GPU allocation belongs + at the InferenceService level for flexibility. Without skill, agents hardcode + nvidia.com/gpu in the runtime spec.""" + c = read_report().lower() + assert any(t in c for t in [ + "inferenceservice level", "inferenceservice deployment", + "per inferenceservice", "not specified in the servingruntime", + "gpu allocation happens at", + ]), "should explain GPU allocation belongs at InferenceService level, not in the runtime" + + def test_model_format_matching(self): + """Skill teaches that supportedModelFormats must match InferenceService model + format for runtime selection.""" + c = read_report().lower() + assert any(t in c for t in [ + "model format", "supportedmodelformat", "supported model format", + "inferenceservice", "match", + ]), "should address model format matching for runtime selection" + + def test_dashboard_label(self): + """Skill teaches opendatahub.io/dashboard label for dashboard visibility.""" + c = read_report().lower() + assert any(t in c for t in [ + "opendatahub", "dashboard", "label", "visible", + "platform", "display", + ]), "should address dashboard/platform visibility via labels" + + def test_caikit_tgis_grpc(self): + """Docs teach Caikit+TGIS is gRPC-only (no REST API) and NIM uses + TensorRT-LLM with pre-compiled engines. Without docs, agents assume REST + for all runtimes.""" + c = read_report().lower() + assert any(t in c for t in [ + "grpc", "caikit", "tgis", "tensorrt", + ]) and ("runtime" in c or "serving" in c), ( + "should note Caikit+TGIS gRPC-only or NIM TensorRT-LLM characteristics" + ) diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/Dockerfile b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/Dockerfile new file mode 100644 index 00000000..aac4c84e --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/Dockerfile @@ -0,0 +1,67 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "rhoai": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-rhoai-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/common-issues.md b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/common-issues.md new file mode 100644 index 00000000..dbefbc81 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/common-issues.md @@ -0,0 +1,84 @@ +--- +title: Common Issues Across Skills +category: references +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] +last_updated: 2026-03-12 +--- + +# Common Issues Across Skills + +Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. + +## GPU Scheduling Failure + +**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` + +**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" + +**Cause**: Cluster does not have enough available GPUs of the required type. + +**Solution:** +1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` +2. Reduce GPU request or use a quantized model variant +3. Check if other workloads are consuming GPU resources +4. Verify GPU Operator and NFD Operator are healthy +5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization + +## OOMKilled During Model or Workbench Loading + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading + +**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. + +**Solution:** +1. Increase memory limits in the InferenceService or workbench spec +2. Reduce `--max-model-len` to lower KV cache memory usage +3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint +4. Verify GPU VRAM is sufficient using `get_gpu_info` +5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing + +## Image Pull Error from nvcr.io (NIM) + +**Applies to**: `/model-deploy`, `/nim-setup` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` + +**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. + +**Solution:** +1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret +2. Check that the secret contains valid docker credentials for `nvcr.io` +3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key +4. Ensure the secret is referenced by the ServiceAccount or Account CR + +## Image Pull Error from OCI Registries + +**Applies to**: `/model-deploy`, `/serving-runtime-config` + +**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images + +**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. + +**Solution:** +1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models +2. If OCI source is required: verify entitlements are included in the pull secret +3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) +4. Verify the image URI and tag are correct + +## RBAC / Permission Errors + +**Applies to**: All skills that create or modify Kubernetes resources + +**Error**: API call returns 403 Forbidden or "insufficient permissions" message + +**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. + +**Solution:** +1. Report the specific permission error to the user +2. Identify the required role: which API group, resource, and verb is needed +3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding +4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/model-deploy.md b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/model-deploy.md new file mode 100644 index 00000000..dcd79e44 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/model-deploy.md @@ -0,0 +1,160 @@ +# /model-deploy Examples + +Example interactions demonstrating the model-deploy skill workflow. + +## Example 1: Deploy Llama 3.1 8B with vLLM + +``` +User: "Deploy Llama 3.1 8B on my cluster" + +Agent: [Invokes /model-deploy skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand runtime capabilities. + +Step 1 of 10: Gathering deployment information. + +| Setting | Value | Source | +|---------|-------|--------| +| Model | Llama 3.1 8B Instruct | user input | +| Runtime | TBD | auto-detect | +| Namespace | ? | needs input | +| Model Source | ? | needs input | +| Deployment Mode | Serverless | default | + +What namespace should I deploy to, and where are the model weights stored? +(e.g., S3 bucket, OCI registry, PVC) + +User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" + +Agent: Step 2 of 10: Determining runtime. + +I consulted supported-runtimes.md to understand runtime capabilities. + +Based on Llama 3.1 8B (open-source HuggingFace model): + +Recommended runtime: vLLM +Reason: Default runtime for open-source LLMs. Broad compatibility, + OpenAI-compatible API, no additional setup required. + +Proceed with vLLM? (yes / no / suggest alternative) + +User: "yes" + +Agent: Step 3 of 10: Looking up hardware profile. + +I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. + +| Requirement | Value | +|-------------|-------| +| GPUs | 1x A10G/L4/A100/H100 | +| VRAM | 16GB | +| Key Args | --max-model-len=4096 --tool-call-parser hermes | + +Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). +Ensure your cluster has at least 1 GPU with 16GB VRAM. + +Step 5 of 10: Skipped (NIM platform check not needed for vLLM). + +Step 6-7 of 10: Generating InferenceService YAML. + +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: llama-3-1-8b + namespace: ai-models + annotations: + serving.kserve.io/deploymentMode: Serverless +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: vllm-runtime + storageUri: s3://my-bucket/llama-3.1-8b + resources: + limits: + nvidia.com/gpu: "1" + requests: + cpu: "4" + memory: "16Gi" + args: + - --max-model-len=4096 + - --tool-call-parser=hermes + +| Setting | Value | +|---------|-------| +| Model | llama-3-1-8b | +| Runtime | vllm-runtime | +| GPUs | 1 | +| Namespace | ai-models | +| Mode | Serverless | + +Proceed with creating this InferenceService? (yes/no/modify) + +User: "yes" + +Agent: Step 8 of 10: Creating InferenceService... + [Creates via resources_create_or_update] + + InferenceService created. Monitoring rollout... + +Step 9 of 10: Monitoring rollout. + +| Stage | Status | +|-------|--------| +| Pod Scheduled | Complete | +| Image Pulled | Complete | +| Container Started | Complete | +| Model Loaded | Complete | +| Ready | Complete | + +Step 10 of 10: Deployment complete! + +Model Deployment Successful + +Model: llama-3-1-8b +Runtime: vLLM +Namespace: ai-models +GPUs: 1 + +Inference Endpoint: + URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com + API: OpenAI-compatible REST + +Quick Test: + curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' + +Next Steps: + Monitor performance: "/ai-observability llama-3-1-8b" +``` + +## Example 2: Deploy a Model with NIM (NIM Not Set Up) + +``` +User: "Deploy Llama 3.1 70B with NIM" + +Agent: [Invokes /model-deploy skill] + ... + Step 2: User requested NIM runtime. + + Step 5: Checking NIM platform... + +NIM platform is not set up in namespace ai-models. + +The NIM runtime requires a one-time platform setup. Would you like to: +1. Run /nim-setup now to configure the NIM platform +2. Switch to vLLM runtime instead (no setup required) +3. Abort deployment + +User: "1" + +Agent: [Invokes /nim-setup skill — walks through NGC credential setup] + ... + NIM Platform Setup Complete. + + Resuming model deployment... + [Continues from Step 6 with NIM runtime] +``` diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/nim-setup.md b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/nim-setup.md new file mode 100644 index 00000000..ccfc1a5d --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/nim-setup.md @@ -0,0 +1,115 @@ +# /nim-setup Examples + +Example interactions demonstrating the nim-setup skill workflow. + +## Example 1: First-Time NIM Setup + +``` +User: "Set up NIM on my cluster" + +Agent: [Invokes /nim-setup skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand NIM platform requirements. + +Let me verify the prerequisites for NIM setup. + +Step 1 of 7: Checking GPU Operator and NFD Operator... + +GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) +NFD Operator: nfd.v4.19.0 (Succeeded) + +Both operators are installed and healthy. + +Step 2 of 7: I need your NGC credentials. + +To set up NIM, I need your NVIDIA NGC API key. +You can generate one at: https://ngc.nvidia.com/setup/api-key + +Please provide: +1. Your NGC API key +2. The target namespace for NIM resources + +User: "Here's my key: [user provides NGC API key], namespace: ai-models" + +Agent: Step 3 of 7: Creating NGC image pull secret... + +I'll create a docker-registry secret for pulling NIM images from nvcr.io: + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-image-pull-secret + namespace: ai-models + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] + +Should I create this image pull secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + Image pull secret created successfully. + +Step 4 of 7: Creating NGC API key secret... + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-api-key + namespace: ai-models + type: Opaque + stringData: + NGC_API_KEY: "[REDACTED]" + +Should I create this API key secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + API key secret created successfully. + +Step 5 of 7: Creating NIM Account CR... + + apiVersion: nim.opendatahub.io/v1 + kind: Account + metadata: + name: nim-account + namespace: ai-models + spec: + apiKeySecret: + name: ngc-api-key + imagePullSecret: + name: ngc-image-pull-secret + +Should I create this NIM Account CR in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates Account CR via resources_create_or_update] + Account CR created successfully. + +Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) + +User: "no" + +Agent: Step 7 of 7: Validating NIM platform readiness... + +NIM Platform Setup Complete + +Account CR: `nim-account` (namespace: ai-models) + Status: Ready + +NGC Credentials: + Image Pull Secret: ngc-image-pull-secret [created] + API Key Secret: ngc-api-key [created] + +NIM ServingRuntimes Available: + - nim-llama-3-1-8b-instruct + - nim-llama-3-1-70b-instruct + - nim-mistral-7b-instruct + +Next Steps: + To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" +``` diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/live-doc-lookup.md b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/live-doc-lookup.md new file mode 100644 index 00000000..f09979aa --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/live-doc-lookup.md @@ -0,0 +1,106 @@ +--- +title: Live Documentation Lookup Protocol +category: references +tags: [live-lookup, webfetch, documentation, models, runtimes] +semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] +use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] +last_updated: 2026-02-26 +--- + +# Live Documentation Lookup Protocol + +This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. + +## When to Trigger Live Lookup + +The agent MUST trigger a live doc lookup when ANY of these conditions are true: + +1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) +2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model +3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about +4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data + +## Lookup Targets + +Query these sources in order of relevance. Stop once sufficient information is found. + +### 1. Red Hat OpenShift AI Documentation + +**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` + +**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. + +**What to extract**: +- Supported serving runtime versions and configurations +- InferenceService CRD schema updates +- Model catalog entries with deployment parameters +- Known issues and workarounds + +### 2. NVIDIA NIM Model Catalog + +**URL**: `https://build.nvidia.com/models` + +**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. + +**What to extract**: +- GPU type and count requirements (e.g., "2x A100 80GB") +- Model-specific deployment parameters +- API specifications and endpoints +- Available model profiles (optimized vs. generic) + +### 3. NVIDIA NIM Supported Models Matrix + +**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + +**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. + +**What to extract**: +- GPU compatibility per model (which GPU types are supported) +- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) +- Minimum GPU memory requirements +- Tensor parallelism configuration + +## Lookup Procedure + +### Step 1: Determine the lookup target + +Based on the trigger condition, select the most relevant URL: +- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) +- Runtime configuration → RHOAI docs (#1) +- NIM GPU compatibility → NIM supported models matrix (#3) +- General deployment issues → RHOAI docs (#1) + +### Step 2: Fetch the page + +Use the **WebFetch** tool to retrieve the relevant page content. + +### Step 3: Extract relevant information + +Parse the fetched content for: +- GPU type and count requirements +- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) +- Compatible runtimes and their versions +- Known issues or special configuration notes + +### Step 4: Report to user + +**REQUIRED** (Document Consultation Transparency - Design Principle #1): + +Always report what was looked up and from where: + +``` +"I looked up [model-name] on [source-name] to confirm its hardware requirements: +- GPU: [count]x [type] ([VRAM]) +- Key parameters: [list] +- Compatible runtimes: [list]" +``` + +### Step 5: Proceed with deployment + +Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. + +## Security Considerations + +- Live lookup URLs are read-only documentation pages +- No credentials are sent to external URLs +- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/references/known-model-profiles.md b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/references/known-model-profiles.md new file mode 100644 index 00000000..40148981 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/references/known-model-profiles.md @@ -0,0 +1,83 @@ +--- +title: Known Model Hardware Profiles +category: references +tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] +semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] +use_cases: [model-deploy, debug-inference, ai-observability] +last_updated: 2026-03-10 +--- + +# Known Model Hardware Profiles + +Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). + +**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. + +## Model Source Conventions + +Each model lists a recommended `storageUri` with its authentication requirements: +- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** +- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). +- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. + +When the user does not specify a model source, use the `hf://` URI listed in the profile below. + +## Llama 3.x (Meta) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | +| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | +| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | +| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | + +- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) +- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly + +## Granite 3.x (IBM/Red Hat) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | +| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | + +- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) +- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser granite --chat-template` +- Red Hat-supported model family on RHOAI + +## Mixtral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | +| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) +- Mixture-of-Experts architecture: only ~13B/45B parameters active per token + +## Mistral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | +| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) + +## When a Model Is Not Listed + +If the requested model is not in this file, the agent MUST use the live doc lookup protocol: + +1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure +2. Fetch hardware specs from the appropriate source +3. Report findings to the user before proceeding with deployment + +Common cases requiring live lookup: +- Newly released models (after this file's last update) +- Domain-specific fine-tuned models +- Models with custom quantization +- NIM-specific optimized profiles diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/references/supported-runtimes.md b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/references/supported-runtimes.md new file mode 100644 index 00000000..9d018651 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/references/supported-runtimes.md @@ -0,0 +1,104 @@ +--- +title: Supported Serving Runtimes +category: references +tags: [runtimes, vllm, nim, caikit, tgis, serving] +semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] +use_cases: [model-deploy, serving-runtime-config, nim-setup] +last_updated: 2026-02-26 +--- + +# Supported Serving Runtimes + +This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. + +## Runtime Comparison + +| Runtime | API Type | Model Format | GPU Support | License | Setup Required | +|---------|----------|-------------|-------------|---------|----------------| +| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | +| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | +| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | +| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | + +## vLLM + +**Default runtime for most open-source models.** + +- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) +- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized +- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) +- **Model source**: S3-compatible storage, OCI registry, PVC, URI +- **Key features**: + - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) + - PagedAttention for efficient memory management + - Tensor parallelism for multi-GPU inference + - Tool/function calling support (`--tool-call-parser`) + - Continuous batching for high throughput +- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name + +## NVIDIA NIM + +**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** + +- **API**: OpenAI-compatible REST +- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog +- **GPU support**: NVIDIA only (requires specific GPU models per model profile) +- **Model source**: NGC model catalog (pulled automatically via NGC credentials) +- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) +- **Key features**: + - TensorRT-LLM optimization for lower latency + - Pre-compiled model engines (no compilation on first load) + - Optimized and generic profiles per GPU type + - Automatic model download from NGC +- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. +- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM +- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. +- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state + +## Caikit+TGIS + +**Red Hat's Caikit format with Text Generation Inference Server.** + +- **API**: gRPC (not REST) +- **Model formats**: Caikit format (requires conversion from HuggingFace) +- **GPU support**: NVIDIA +- **Model source**: S3-compatible storage +- **Key features**: + - Red Hat-supported runtime + - gRPC API for streaming inference + - Integrated with RHOAI model serving platform +- **When to choose**: When the model is already in Caikit format or when gRPC API is required. +- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name + +## Custom Runtimes + +**User-provided ServingRuntime CRs for unsupported frameworks.** + +- **API**: Defined by the custom runtime +- **Model formats**: Defined by the custom runtime +- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. +- **How to create**: Use `/serving-runtime-config` skill +- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility + +## Runtime Selection Decision Tree + +``` +Is the user's preferred runtime explicitly stated? +├── Yes → Use that runtime +└── No → Continue + +Is the model available in the NGC NIM catalog? +├── Yes → Suggest NIM (with vLLM as fallback) +│ Note: Requires /nim-setup and NVIDIA GPUs +└── No → Continue + +Is the model in Caikit format? +├── Yes → Caikit+TGIS +└── No → Continue + +Is the model a standard open-source LLM (HuggingFace-compatible)? +├── Yes → vLLM (default) +└── No → Custom runtime via /serving-runtime-config +``` diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/skill-conventions.md b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/skill-conventions.md new file mode 100644 index 00000000..ada90ecc --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/skill-conventions.md @@ -0,0 +1,85 @@ +--- +title: Skill Conventions +category: references +tags: [conventions, prerequisites, human-in-the-loop, security] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] +last_updated: 2026-03-01 +--- + +# rh-ai-engineer Skill Conventions + +Shared conventions for all skills in the rh-ai-engineer agentic collection. + +## Prerequisite Verification Protocol + +Before executing any skill, verify MCP server availability: + +1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` +2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) +3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) + +**When prerequisites fail:** + +1. Stop execution immediately +2. Report the specific missing prerequisite: + ``` + Cannot execute [skill-name]: [specific prerequisite] is not available + + Setup Instructions: + 1. [Server-specific setup steps] + 2. Set required environment variables + 3. Restart Claude Code to reload MCP servers + + Documentation: [link to server docs] + ``` +3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) +4. WAIT for user decision -- never proceed automatically + +**Common prerequisite: OpenShift MCP Server** + +Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +- Source: https://github.com/openshift/openshift-mcp-server +- Required env var: `KUBECONFIG` +- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code + +## Common Prerequisites + +All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster with Red Hat OpenShift AI operator installed +- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) +- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed + +## Human-in-the-Loop Requirements + +All rh-ai-engineer skills that create or modify Kubernetes resources MUST: + +1. **Display the resource manifest** (with credentials REDACTED) before creation +2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" +3. **WAIT for user response** -- never auto-execute +4. **On failure, present diagnostic options** -- never auto-delete or auto-retry + +**Never:** +- Create resources without user reviewing the manifest +- Display actual credential values (API keys, passwords, tokens) +- Skip confirmation for any resource creation +- Assume approval -- always wait for explicit user confirmation + +**Why This Matters:** +- GPU resources are expensive and may have associated costs +- Deployments may affect other workloads competing for cluster resources +- Credentials grant access to external services (NGC, model registries) + +## Security Conventions + +- **Credentials**: Never display actual values; only report presence/absence +- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) +- **KUBECONFIG**: Path and contents never exposed in output +- **Namespace isolation**: All resources created in user-specified namespace only +- **RBAC**: Check for sufficient permissions before attempting resource creation +- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..e7a4d11c --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +"""Mock OpenShift MCP server for SkillsBench rh-ai-engineer task. + +Simulates Kubernetes resource CRUD, pod management, logs, and events. + +Key scenario elements: +- LimitRange in namespaces: min CPU=100m, min memory=128Mi + (conflicts with KServe sidecar containers hardcoded at 10m CPU/15Mi memory) +- GPU node with custom taint ai-workload=true:NoSchedule +- NIM Account CR in ml-production: not ready (NGC credentials invalid) +- text-gen-legacy pods: OOMKilled (max-model-len=32768 on A10G) +- nim-llama-prod: no pods created (Account CR not ready) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + +# ── Cluster state ──────────────────────────────────────────────────────── + +GPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "gpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + "nvidia.com/gpu.present": "true", + "nvidia.com/gpu.product": "NVIDIA-A10G", + }, + }, + "spec": { + "taints": [ + { + "key": "ai-workload", + "value": "true", + "effect": "NoSchedule", + }, + ], + }, + "status": { + "allocatable": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "capacity": { + "cpu": "48", + "memory": "192Gi", + "nvidia.com/gpu": "2", + "pods": "250", + }, + "conditions": [ + {"type": "Ready", "status": "True"}, + ], + }, +} + +CPU_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "cpu-worker-1", + "labels": { + "node-role.kubernetes.io/worker": "", + }, + }, + "spec": {"taints": []}, + "status": { + "allocatable": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "capacity": {"cpu": "16", "memory": "64Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +MASTER_NODE = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "master-1", + "labels": { + "node-role.kubernetes.io/master": "", + "node-role.kubernetes.io/control-plane": "", + }, + }, + "spec": { + "taints": [ + {"key": "node-role.kubernetes.io/master", "effect": "NoSchedule"}, + ], + }, + "status": { + "allocatable": {"cpu": "8", "memory": "32Gi", "pods": "250"}, + "conditions": [{"type": "Ready", "status": "True"}], + }, +} + +ALL_NODES = [GPU_NODE, CPU_NODE, MASTER_NODE] + +# LimitRange applied by cluster policy to all DS project namespaces +NAMESPACE_LIMITRANGE = { + "apiVersion": "v1", + "kind": "LimitRange", + "metadata": { + "name": "default-limits", + }, + "spec": { + "limits": [ + { + "type": "Container", + "default": { + "cpu": "2", + "memory": "4Gi", + }, + "defaultRequest": { + "cpu": "500m", + "memory": "256Mi", + }, + "min": { + "cpu": "100m", + "memory": "128Mi", + }, + "max": { + "cpu": "32", + "memory": "128Gi", + }, + }, + ], + }, +} + +NIM_ACCOUNT_CR = { + "apiVersion": "nim.opendatahub.io/v1", + "kind": "Account", + "metadata": { + "name": "nim-account", + "namespace": "ml-production", + }, + "spec": { + "apiKeySecret": { + "name": "ngc-api-key", + }, + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "NGCCredentialsInvalid", + "message": "NGC API key validation failed: 401 Unauthorized. " + "The API key in secret 'ngc-api-key' is expired or invalid. " + "Re-create the secret with a valid NGC API key from " + "https://ngc.nvidia.com/setup/api-key and restart the " + "Account reconciliation.", + "lastTransitionTime": "2026-03-14T12:00:00Z", + }, + ], + "nimPullSecretStatus": "Failed", + "nimConfigStatus": "Pending", + }, +} + +SERVING_RUNTIME_VLLM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "vllm-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "vLLM", "version": "1", "autoSelect": True}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "quay.io/modh/vllm:rhoai-2.16", + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + }, + ], + }, +} + +SERVING_RUNTIME_NIM = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "name": "nim-serving-runtime", + }, + "spec": { + "supportedModelFormats": [ + {"name": "NIM", "version": "1"}, + ], + "containers": [ + { + "name": "kserve-container", + "image": "nvcr.io/nim/meta/llama-3.1-8b-instruct:latest", + "ports": [{"containerPort": 8000, "protocol": "TCP"}], + "env": [ + {"name": "NGC_API_KEY", "valueFrom": { + "secretKeyRef": {"name": "ngc-api-key", "key": "api_key"}, + }}, + ], + }, + ], + }, +} + +PODS_BY_NAMESPACE = { + "ml-production": [ + { + "name": "text-gen-legacy-predictor-00001-abc12", + "namespace": "ml-production", + "status": "CrashLoopBackOff", + "restarts": 5, + "node": "gpu-worker-1", + "containers": [ + { + "name": "kserve-container", + "state": "waiting", + "reason": "CrashLoopBackOff", + "last_termination_reason": "OOMKilled", + "last_termination_exit_code": 137, + }, + ], + "labels": { + "serving.kserve.io/inferenceservice": "text-gen-legacy", + }, + "gpu": "1", + }, + # nim-llama-prod: NO pods created (Account CR not ready) + ], +} + +POD_LOGS = { + "text-gen-legacy-predictor-00001-abc12": ( + "INFO 2026-03-01 10:00:00 vllm_engine.py:125] vLLM engine starting...\n" + "INFO 2026-03-01 10:00:01 config.py:89] Model: mistralai/Mistral-7B-Instruct-v0.3\n" + "INFO 2026-03-01 10:00:01 config.py:92] max_model_len = 32768\n" + "INFO 2026-03-01 10:00:02 gpu_executor.py:45] GPU 0: NVIDIA A10G (24576 MiB)\n" + "INFO 2026-03-01 10:00:03 model_runner.py:88] Loading model weights...\n" + "INFO 2026-03-01 10:00:15 model_runner.py:112] Model weights loaded: 13.5 GiB\n" + "INFO 2026-03-01 10:00:15 worker.py:201] Allocating KV cache...\n" + "ERROR 2026-03-01 10:00:16 worker.py:215] torch.cuda.OutOfMemoryError: " + "CUDA out of memory. Tried to allocate 28.5 GiB for KV cache but only " + "10.1 GiB available after loading model weights (13.5 GiB).\n" + "ERROR 2026-03-01 10:00:16 vllm_engine.py:178] Engine failed to start\n" + "Traceback (most recent call last):\n" + " File \"/opt/vllm/vllm/engine/engine.py\", line 175, in start\n" + " self._init_kv_cache()\n" + " File \"/opt/vllm/vllm/worker/worker.py\", line 215, in _init_kv_cache\n" + " raise torch.cuda.OutOfMemoryError(msg)\n" + "torch.cuda.OutOfMemoryError: CUDA out of memory\n" + ), +} + +EVENTS_BY_NAMESPACE = { + "ml-production": [ + { + "type": "Warning", + "reason": "BackOff", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Back-off restarting failed container kserve-container in pod " + "text-gen-legacy-predictor-00001-abc12", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Warning", + "reason": "OOMKilled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Container kserve-container was OOMKilled (exit code 137). " + "GPU memory exhausted during KV cache allocation.", + "count": 5, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-03-01T10:00:16Z", + }, + { + "type": "Normal", + "reason": "Scheduled", + "object": "Pod/text-gen-legacy-predictor-00001-abc12", + "message": "Successfully assigned ml-production/" + "text-gen-legacy-predictor-00001-abc12 to gpu-worker-1", + "count": 1, + "first_timestamp": "2026-02-28T08:00:00Z", + "last_timestamp": "2026-02-28T08:00:00Z", + }, + { + "type": "Warning", + "reason": "NIMAccountNotReady", + "object": "InferenceService/nim-llama-prod", + "message": "NIM Account 'nim-account' in namespace 'ml-production' " + "is not ready", + "count": 12, + "first_timestamp": "2026-03-14T12:00:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + { + "type": "Warning", + "reason": "ImagePullBackOff", + "object": "InferenceService/nim-llama-prod", + "message": "Failed to pull image 'nvcr.io/nim/meta/llama-3.1-8b-instruct:" + "latest': unauthorized: authentication required", + "count": 8, + "first_timestamp": "2026-03-14T12:05:00Z", + "last_timestamp": "2026-03-15T10:00:00Z", + }, + ], +} + + +# ── Resource tools ─────────────────────────────────────────────────────── + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: str = "", +) -> str: + """Get a single Kubernetes resource by apiVersion, kind, and name.""" + if kind == "Node": + for node in ALL_NODES: + if node["metadata"]["name"] == name: + return json.dumps(node, indent=2) + raise ValueError(f"Node '{name}' not found") + + if kind == "ServingRuntime": + if name == "vllm-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_VLLM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + if name == "nim-serving-runtime": + cr = json.loads(json.dumps(SERVING_RUNTIME_NIM)) + cr["metadata"]["namespace"] = namespace or "ml-production" + return json.dumps(cr, indent=2) + raise ValueError(f"ServingRuntime '{name}' not found in namespace '{namespace}'") + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps(lr, indent=2) + + if kind == "Account" and "nim" in apiVersion.lower(): + if namespace == "ml-production" and name == "nim-account": + return json.dumps(NIM_ACCOUNT_CR, indent=2) + raise ValueError( + f"Account '{name}' not found in namespace '{namespace}'" + ) + + if kind == "ClusterVersion" and apiVersion == "config.openshift.io/v1": + return json.dumps({ + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterVersion", + "metadata": {"name": "version"}, + "status": {"desired": {"version": "4.16.3"}}, + }) + + raise ValueError(f"Resource {apiVersion}/{kind}/{name} not found") + + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: str = "", + labelSelector: str = "", +) -> str: + """List Kubernetes resources by apiVersion and kind.""" + if kind == "Node": + nodes = ALL_NODES + if labelSelector: + parts = labelSelector.split("=", 1) + key = parts[0] + value = parts[1] if len(parts) > 1 else "" + nodes = [ + n for n in nodes + if n["metadata"]["labels"].get(key) == value + ] + return json.dumps(nodes, indent=2) + + if kind == "Service" and apiVersion == "serving.knative.dev/v1": + return json.dumps({ + "kind": "ServiceList", + "apiVersion": "serving.knative.dev/v1", + "items": [], + "metadata": {}, + }) + + if kind == "LimitRange": + lr = json.loads(json.dumps(NAMESPACE_LIMITRANGE)) + lr["metadata"]["namespace"] = namespace + return json.dumps({ + "kind": "LimitRangeList", + "items": [lr], + }) + + if kind == "InferenceService": + return json.dumps({ + "kind": "InferenceServiceList", + "items": [], + }) + + raise ValueError(f"Unsupported list: {apiVersion}/{kind}") + + +@mcp.tool() +def pods_list( + namespace: str, + labelSelector: str = "", +) -> str: + """List pods in a namespace with optional label selector.""" + pods = PODS_BY_NAMESPACE.get(namespace, []) + + if labelSelector: + key, _, value = labelSelector.partition("=") + pods = [p for p in pods if p.get("labels", {}).get(key) == value] + + results = [] + for pod in pods: + results.append({ + "name": pod["name"], + "namespace": pod["namespace"], + "status": pod["status"], + "restarts": pod.get("restarts", 0), + "node": pod.get("node", ""), + "containers": pod.get("containers", []), + "gpu": pod.get("gpu", "0"), + }) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def pods_log( + namespace: str, + name: str, + container: str = "", +) -> str: + """Get logs from a pod container.""" + logs = POD_LOGS.get(name) + if logs is None: + raise ValueError(f"Pod '{name}' not found in namespace '{namespace}'") + return logs + + +@mcp.tool() +def events_list(namespace: str) -> str: + """List events in a namespace.""" + events = EVENTS_BY_NAMESPACE.get(namespace, []) + return json.dumps(events, indent=2) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/mcp-servers/mock-rhoai-mcp.py b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/mcp-servers/mock-rhoai-mcp.py new file mode 100644 index 00000000..12513127 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/mcp-servers/mock-rhoai-mcp.py @@ -0,0 +1,866 @@ +#!/usr/bin/env python3 +"""Mock RHOAI MCP server for SkillsBench rh-ai-engineer task. + +Simulates Red Hat OpenShift AI operations: Data Science Projects, +model serving, data connections, serving runtimes, inference services. + +Scenario: +- ml-production: existing project with two broken deployments + - text-gen-legacy: vLLM OOMKilled (max-model-len=32768 on A10G) + - nim-llama-prod: NIM failing (Account CR not ready, NGC creds invalid) +- fraud-detection: does not exist yet (agent creates it) +""" + +import json +from fastmcp import FastMCP + +mcp = FastMCP("rhoai") + +# ── In-memory state ────────────────────────────────────────────────────── + +PROJECTS = { + "ml-production": { + "name": "ml-production", + "display_name": "ML Production", + "description": "Production ML workloads", + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": "single", + "pipeline_server": True, + }, +} + +DATA_CONNECTIONS = { + "ml-production": [ + { + "name": "prod-model-store", + "type": "S3", + "bucket": "ml-models-prod", + "endpoint": "https://s3.us-east-1.amazonaws.com", + "region": "us-east-1", + }, + ], +} + +SERVING_RUNTIMES = { + "__platform_templates__": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "REST", + "supported_model_formats": [ + {"name": "vLLM", "version": "1", "autoSelect": True} + ], + }, + { + "name": "caikit-tgis-runtime", + "display_name": "Caikit+TGIS ServingRuntime", + "model_formats": ["caikit"], + "requires_instantiation": True, + "source": "platform-template", + "api_protocol": "gRPC", + }, + ], + "ml-production": [ + { + "name": "vllm-runtime", + "display_name": "vLLM ServingRuntime for KServe", + "model_formats": ["vLLM"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + { + "name": "nim-serving-runtime", + "display_name": "NVIDIA NIM ServingRuntime", + "model_formats": ["NIM"], + "requires_instantiation": False, + "source": "nim-account", + "api_protocol": "REST", + }, + { + "name": "ovms-1", + "display_name": "OpenVINO Model Server", + "model_formats": ["openvino_ir", "onnx"], + "requires_instantiation": False, + "source": "existing", + "api_protocol": "REST", + }, + ], +} + +INFERENCE_SERVICES = { + "ml-production": { + "text-gen-legacy": { + "name": "text-gen-legacy", + "namespace": "ml-production", + "runtime": "vllm-runtime", + "model_format": "vLLM", + "storage_uri": "hf://mistralai/Mistral-7B-Instruct-v0.3", + "display_name": "Mistral 7B Legacy", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "16Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "PredictorFailed", + "message": "Predictor pod is not ready", + }, + { + "type": "PredictorReady", + "status": "False", + "reason": "ContainerCrashLoop", + "message": "Container kserve-container terminated: " + "OOMKilled (exit code 137). 5 restarts.", + }, + { + "type": "IngressReady", + "status": "True", + "reason": "IngressReady", + "message": "Ingress is ready", + }, + ], + "age": "3d", + }, + "nim-llama-prod": { + "name": "nim-llama-prod", + "namespace": "ml-production", + "runtime": "nim-serving-runtime", + "model_format": "NIM", + "storage_uri": "nim://meta/llama-3.1-8b-instruct", + "display_name": "Llama 3.1 8B (NIM)", + "gpu_count": 1, + "cpu_request": "4", + "memory_request": "16Gi", + "memory_limit": "32Gi", + "min_replicas": 1, + "max_replicas": 1, + "ready": False, + "url": "", + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "RuntimeNotReady", + "message": "ServingRuntime 'nim-serving-runtime' " + "is not in ready state", + }, + { + "type": "PredictorReady", + "status": "Unknown", + "reason": "PodNotCreated", + "message": "Predictor pod has not been created. " + "Waiting for ServingRuntime to become ready.", + }, + { + "type": "IngressReady", + "status": "Unknown", + "reason": "PredictorNotReady", + "message": "Waiting for predictor to become ready", + }, + ], + "age": "1d", + }, + }, +} + +DEPLOYED_MODELS = {} + +WORKBENCHES = { + "ml-production": [ + { + "name": "data-exploration-nb", + "display_name": "Data Exploration", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Running", + "cpu_request": "1", + "memory_request": "8Gi", + "gpu_count": 0, + "pvc_name": "data-exploration-nb-pvc", + "pvc_size": "20Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-10T09:00:00Z", + }, + { + "name": "model-training-nb", + "display_name": "Model Training", + "image": "jupyter-pytorch-ubi9-python-3.9-2024.1", + "status": "Stopped", + "cpu_request": "4", + "memory_request": "16Gi", + "gpu_count": 1, + "pvc_name": "model-training-nb-pvc", + "pvc_size": "50Gi", + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-02-15T14:00:00Z", + }, + ], +} + +PIPELINE_SERVERS = { + "ml-production": { + "configured": True, + "data_connection": "prod-model-store", + "status": "Ready", + "database": "MariaDB", + }, +} + +NOTEBOOK_IMAGES = [ + {"name": "jupyter-pytorch-ubi9-python-3.9-2024.1", "display_name": "PyTorch 2024.1", "packages": ["torch", "transformers"]}, + {"name": "jupyter-tensorflow-ubi9-python-3.9-2024.1", "display_name": "TensorFlow 2024.1", "packages": ["tensorflow"]}, + {"name": "jupyter-datascience-ubi9-python-3.9-2024.1", "display_name": "Standard Data Science", "packages": ["pandas", "scikit-learn"]}, + {"name": "jupyter-minimal-ubi9-python-3.9-2024.1", "display_name": "Minimal Python", "packages": []}, +] + + +# ── Project tools ──────────────────────────────────────────────────────── + + +@mcp.tool() +def list_data_science_projects() -> str: + """List all RHOAI Data Science Projects on the cluster.""" + projects = [] + for name, proj in PROJECTS.items(): + projects.append({ + "name": name, + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + }) + return json.dumps(projects, indent=2) + + +@mcp.tool() +def create_data_science_project( + name: str, + display_name: str, + description: str = "", +) -> str: + """Create a new RHOAI Data Science Project (namespace with dashboard labels).""" + if name in PROJECTS: + raise ValueError( + f"Project '{name}' already exists. Choose a different name " + "or configure the existing project." + ) + if not name.replace("-", "").replace("_", "").isalnum() or len(name) > 63: + raise ValueError( + f"Invalid project name '{name}'. Must be DNS-compatible: " + "lowercase alphanumeric and hyphens, max 63 chars." + ) + + PROJECTS[name] = { + "name": name, + "display_name": display_name, + "description": description, + "labels": {"opendatahub.io/dashboard": "true"}, + "model_serving_mode": None, + "pipeline_server": False, + } + DATA_CONNECTIONS[name] = [] + SERVING_RUNTIMES[name] = [] + INFERENCE_SERVICES[name] = {} + + return json.dumps({ + "status": "created", + "name": name, + "display_name": display_name, + "namespace": name, + "labels": {"opendatahub.io/dashboard": "true"}, + }) + + +@mcp.tool() +def get_project_details(name: str) -> str: + """Get detailed information about an RHOAI Data Science Project.""" + if name not in PROJECTS: + raise ValueError(f"Project '{name}' not found") + proj = PROJECTS[name] + dc_count = len(DATA_CONNECTIONS.get(name, [])) + isvc_count = len(INFERENCE_SERVICES.get(name, {})) + return json.dumps({ + "name": proj["name"], + "display_name": proj["display_name"], + "description": proj.get("description", ""), + "labels": proj["labels"], + "data_connections": dc_count, + "inference_services": isvc_count, + "model_serving_mode": proj.get("model_serving_mode"), + "pipeline_server": proj.get("pipeline_server", False), + }) + + +@mcp.tool() +def get_project_status(namespace: str) -> str: + """Get comprehensive status of an RHOAI Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Project '{namespace}' not found") + proj = PROJECTS[namespace] + dcs = DATA_CONNECTIONS.get(namespace, []) + isvcs = INFERENCE_SERVICES.get(namespace, {}) + return json.dumps({ + "namespace": namespace, + "display_name": proj["display_name"], + "status": "Active", + "components": { + "data_connections": len(dcs), + "inference_services": len(isvcs), + "model_serving_mode": proj.get("model_serving_mode", "not configured"), + "pipeline_server": "configured" if proj.get("pipeline_server") else "not configured", + }, + }) + + +# ── Data connection tools ──────────────────────────────────────────────── + + +@mcp.tool() +def create_s3_data_connection( + namespace: str, + name: str, + bucket: str, + endpoint: str, + access_key: str, + secret_key: str, + region: str = "", +) -> str: + """Create an S3-compatible data connection in an RHOAI project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + existing = DATA_CONNECTIONS.get(namespace, []) + if any(dc["name"] == name for dc in existing): + raise ValueError( + f"Data connection '{name}' already exists in namespace '{namespace}'" + ) + + dc = { + "name": name, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + "region": region, + } + DATA_CONNECTIONS.setdefault(namespace, []).append(dc) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "type": "S3", + "bucket": bucket, + "endpoint": endpoint, + }) + + +@mcp.tool() +def list_data_connections(namespace: str) -> str: + """List data connections in an RHOAI project namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + return json.dumps(dcs, indent=2) + + +# ── Model serving tools ───────────────────────────────────────────────── + + +@mcp.tool() +def set_model_serving_mode(namespace: str, mode: str) -> str: + """Enable model serving on a Data Science Project (single or multi mode).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + if mode not in ("single", "multi"): + raise ValueError(f"Invalid mode '{mode}'. Must be 'single' or 'multi'.") + + PROJECTS[namespace]["model_serving_mode"] = mode + + if not SERVING_RUNTIMES.get(namespace): + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + SERVING_RUNTIMES[namespace] = [ + {**t, "requires_instantiation": False, "source": "existing"} + for t in templates + ] + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "mode": mode, + }) + + +@mcp.tool() +def list_serving_runtimes( + namespace: str, + include_templates: bool = False, +) -> str: + """List available ServingRuntimes in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + runtimes = list(SERVING_RUNTIMES.get(namespace, [])) + if include_templates: + templates = SERVING_RUNTIMES.get("__platform_templates__", []) + existing_names = {r["name"] for r in runtimes} + for t in templates: + if t["name"] not in existing_names: + runtimes.append(t) + + return json.dumps(runtimes, indent=2) + + +# ── Inference service tools ────────────────────────────────────────────── + + +@mcp.tool() +def deploy_model( + name: str, + namespace: str, + runtime: str, + model_format: str, + storage_uri: str, + display_name: str = "", + min_replicas: int = 1, + max_replicas: int = 1, + cpu_request: str = "1", + cpu_limit: str = "2", + memory_request: str = "4Gi", + memory_limit: str = "8Gi", + gpu_count: int = 0, +) -> str: + """Deploy an AI/ML model as a KServe InferenceService.""" + if namespace not in PROJECTS: + raise ValueError( + f"Namespace '{namespace}' is not a Data Science Project. " + "Create one via create_data_science_project first." + ) + + ns_runtimes = SERVING_RUNTIMES.get(namespace, []) + runtime_names = [r["name"] for r in ns_runtimes] + if runtime not in runtime_names: + available = ", ".join(runtime_names) or "none" + raise ValueError( + f"ServingRuntime '{runtime}' not found in namespace '{namespace}'. " + f"Available runtimes: {available}" + ) + + endpoint = f"https://{name}-{namespace}.apps.ocp-cluster.example.com" + isvc = { + "name": name, + "namespace": namespace, + "runtime": runtime, + "model_format": model_format, + "storage_uri": storage_uri, + "display_name": display_name or name, + "gpu_count": gpu_count, + "cpu_request": cpu_request, + "memory_request": memory_request, + "min_replicas": min_replicas, + "max_replicas": max_replicas, + "ready": True, + "url": endpoint, + "conditions": [ + {"type": "Ready", "status": "True", "reason": "Ready", "message": ""}, + {"type": "PredictorReady", "status": "True", "reason": "PodReady", "message": ""}, + {"type": "IngressReady", "status": "True", "reason": "IngressReady", "message": ""}, + ], + "age": "0s", + } + + INFERENCE_SERVICES.setdefault(namespace, {})[name] = isvc + DEPLOYED_MODELS[f"{namespace}/{name}"] = isvc + + return json.dumps({ + "status": "deployed", + "name": name, + "namespace": namespace, + "runtime": runtime, + "endpoint": endpoint, + "ready": True, + }) + + +@mcp.tool() +def list_inference_services( + namespace: str, + verbosity: str = "standard", +) -> str: + """List deployed InferenceServices in a namespace.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + isvcs = INFERENCE_SERVICES.get(namespace, {}) + results = [] + for isvc_name, isvc in isvcs.items(): + entry = { + "name": isvc["name"], + "runtime": isvc["runtime"], + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "age": isvc.get("age", ""), + } + if verbosity == "full": + entry["conditions"] = isvc.get("conditions", []) + entry["storage_uri"] = isvc.get("storage_uri", "") + entry["gpu_count"] = isvc.get("gpu_count", 0) + results.append(entry) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def get_inference_service( + name: str, + namespace: str, + verbosity: str = "standard", +) -> str: + """Get detailed status of a specific InferenceService.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + + isvc = isvcs[name] + result = { + "name": isvc["name"], + "namespace": isvc["namespace"], + "runtime": isvc["runtime"], + "model_format": isvc.get("model_format", ""), + "storage_uri": isvc.get("storage_uri", ""), + "ready": isvc["ready"], + "url": isvc.get("url", ""), + "conditions": isvc.get("conditions", []), + "gpu_count": isvc.get("gpu_count", 0), + "replicas": {"min": isvc.get("min_replicas", 1), "max": isvc.get("max_replicas", 1)}, + "resources": { + "cpu_request": isvc.get("cpu_request", "1"), + "memory_request": isvc.get("memory_request", "4Gi"), + "memory_limit": isvc.get("memory_limit", "8Gi"), + }, + "age": isvc.get("age", ""), + } + return json.dumps(result, indent=2) + + +@mcp.tool() +def get_model_endpoint(name: str, namespace: str) -> str: + """Get the inference endpoint URL for a deployed model.""" + isvcs = INFERENCE_SERVICES.get(namespace, {}) + if name not in isvcs: + raise ValueError( + f"InferenceService '{name}' not found in namespace '{namespace}'" + ) + isvc = isvcs[name] + if not isvc["ready"]: + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": "", + "error": "InferenceService is not ready. Check conditions for details.", + }) + return json.dumps({ + "name": name, + "namespace": namespace, + "endpoint": isvc["url"], + }) + + +# ── Workbench tools ────────────────────────────────────────────────────── + + +@mcp.tool() +def list_workbenches(namespace: str) -> str: + """List workbenches (Jupyter notebooks) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + return json.dumps(wbs, indent=2) + + +@mcp.tool() +def create_workbench( + namespace: str, + name: str, + display_name: str = "", + image: str = "jupyter-datascience-ubi9-python-3.9-2024.1", + cpu_request: str = "1", + memory_request: str = "4Gi", + gpu_count: int = 0, + pvc_size: str = "20Gi", +) -> str: + """Create a new workbench (Jupyter notebook) in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + valid_images = [img["name"] for img in NOTEBOOK_IMAGES] + if image not in valid_images: + raise ValueError( + f"Image '{image}' not found. Available: {', '.join(valid_images)}" + ) + + wb = { + "name": name, + "display_name": display_name or name, + "image": image, + "status": "Running", + "cpu_request": cpu_request, + "memory_request": memory_request, + "gpu_count": gpu_count, + "pvc_name": f"{name}-pvc", + "pvc_size": pvc_size, + "pvc_access_mode": "ReadWriteOnce", + "creation": "2026-03-02T12:00:00Z", + } + WORKBENCHES.setdefault(namespace, []).append(wb) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "image": image, + "pvc": f"{name}-pvc", + }) + + +@mcp.tool() +def stop_workbench(namespace: str, name: str) -> str: + """Stop a running workbench (preserves data).""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Stopped" + return json.dumps({"status": "stopped", "name": name, "namespace": namespace}) + + +@mcp.tool() +def start_workbench(namespace: str, name: str) -> str: + """Start a stopped workbench.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wb["status"] = "Running" + return json.dumps({"status": "running", "name": name, "namespace": namespace}) + + +@mcp.tool() +def get_workbench_url(namespace: str, name: str) -> str: + """Get the URL for accessing a running workbench.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + if wb["status"] != "Running": + return json.dumps({ + "namespace": namespace, + "name": name, + "url": "", + "error": f"Workbench is not running (status: {wb['status']}). Start it first.", + }) + url = f"https://{name}-{namespace}.apps.ocp-cluster.example.com" + return json.dumps({ + "namespace": namespace, + "name": name, + "url": url, + "status": wb["status"], + }) + + +@mcp.tool() +def list_workbench_storage(namespace: str, name: str) -> str: + """List PVC details for a workbench including size, usage, access mode.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + volumes = [ + { + "pvc_name": wb.get("pvc_name", f"{name}-pvc"), + "size": wb.get("pvc_size", "20Gi"), + "usage": "12Gi", # Mock usage + "access_mode": wb.get("pvc_access_mode", "ReadWriteOnce"), + "mount_path": "/opt/app-root/data", + }, + ] + # Include additional volumes if any + for extra in wb.get("extra_volumes", []): + volumes.append(extra) + return json.dumps({ + "namespace": namespace, + "workbench": name, + "volumes": volumes, + }, indent=2) + + +@mcp.tool() +def add_workbench_storage( + namespace: str, + workbench_name: str, + pvc_name: str, + mount_path: str, + size: str, +) -> str: + """Add additional storage volume to a workbench.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == workbench_name), None) + if not wb: + raise ValueError(f"Workbench '{workbench_name}' not found in '{namespace}'") + extra = wb.setdefault("extra_volumes", []) + extra.append({ + "pvc_name": pvc_name, + "size": size, + "usage": "0", + "access_mode": "ReadWriteOnce", + "mount_path": mount_path, + }) + return json.dumps({ + "status": "added", + "namespace": namespace, + "workbench": workbench_name, + "pvc_name": pvc_name, + "mount_path": mount_path, + "size": size, + }) + + +@mcp.tool() +def delete_workbench(namespace: str, name: str) -> str: + """Delete a workbench. WARNING: PVC data may be lost if not backed up.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + wbs = WORKBENCHES.get(namespace, []) + wb = next((w for w in wbs if w["name"] == name), None) + if not wb: + raise ValueError(f"Workbench '{name}' not found in '{namespace}'") + wbs.remove(wb) + return json.dumps({ + "status": "deleted", + "name": name, + "namespace": namespace, + "warning": "Associated PVC data has been deleted", + }) + + +@mcp.tool() +def list_notebook_images() -> str: + """List available notebook images for workbench creation.""" + return json.dumps(NOTEBOOK_IMAGES, indent=2) + + +# ── Pipeline server tools ─────────────────────────────────────────────── + + +@mcp.tool() +def configure_pipeline_server( + namespace: str, + data_connection: str, + database: str = "MariaDB", +) -> str: + """Configure a pipeline server for a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + dcs = DATA_CONNECTIONS.get(namespace, []) + if not any(dc["name"] == data_connection for dc in dcs): + available = [dc["name"] for dc in dcs] + raise ValueError( + f"Data connection '{data_connection}' not found. Available: {available}" + ) + + PIPELINE_SERVERS[namespace] = { + "configured": True, + "data_connection": data_connection, + "status": "Ready", + "database": database, + } + PROJECTS[namespace]["pipeline_server"] = True + + return json.dumps({ + "status": "configured", + "namespace": namespace, + "data_connection": data_connection, + "database": database, + }) + + +@mcp.tool() +def get_pipeline_server_status(namespace: str) -> str: + """Get the status of the pipeline server in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + ps = PIPELINE_SERVERS.get(namespace) + if not ps: + return json.dumps({"namespace": namespace, "configured": False}) + return json.dumps({ + "namespace": namespace, + "configured": ps["configured"], + "data_connection": ps["data_connection"], + "status": ps["status"], + "database": ps["database"], + }) + + +# ── Serving runtime creation ──────────────────────────────────────────── + + +@mcp.tool() +def create_serving_runtime( + namespace: str, + name: str, + display_name: str = "", + model_formats: list = None, + container_image: str = "", + container_port: int = 8080, + multi_model: bool = False, + api_protocol: str = "REST", +) -> str: + """Create a custom ServingRuntime in a Data Science Project.""" + if namespace not in PROJECTS: + raise ValueError(f"Namespace '{namespace}' is not a Data Science Project") + + if not model_formats: + raise ValueError("model_formats must specify at least one model format") + + runtime = { + "name": name, + "display_name": display_name or name, + "model_formats": model_formats, + "requires_instantiation": False, + "source": "custom", + "api_protocol": api_protocol, + "container_image": container_image, + "container_port": container_port, + "multi_model": multi_model, + } + SERVING_RUNTIMES.setdefault(namespace, []).append(runtime) + + return json.dumps({ + "status": "created", + "name": name, + "namespace": namespace, + "model_formats": model_formats, + }) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/instruction.md b/evaluation/without_skills/rh-ai-engineer__workbench-manage/instruction.md new file mode 100644 index 00000000..39b97c27 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__workbench-manage/instruction.md @@ -0,0 +1,13 @@ +# Workbench Management Task + +You are an AI engineer on Red Hat OpenShift AI. Your data science team needs workbenches set up for model development, and some existing workbenches need cleanup. + +## Requirements +- Review existing workbenches in the project: their status, resource usage, and notebook images +- Plan a new workbench for a data scientist who needs PyTorch with 4 CPUs, 16Gi memory, and 50Gi persistent storage +- Identify any stopped or unused workbenches that should be cleaned up to free resources +- Document the lifecycle procedures: how to stop a workbench to save resources, restart it, and safely delete one + +Document your workbench assessment, creation plan, and cleanup recommendations in `/root/report.md`. + +Use MCP tools to interact with the platform. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/solution/solve.sh b/evaluation/without_skills/rh-ai-engineer__workbench-manage/solution/solve.sh new file mode 100644 index 00000000..49e5cc92 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__workbench-manage/solution/solve.sh @@ -0,0 +1,25 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Workbench Creation Plan + +## Workbench: fraud-analysis +Project/Namespace: fraud-detection + +### Storage (create_storage) +- PVC: 20Gi, access mode: ReadWriteOnce +- Namespace validated via list_data_science_projects + +### Configuration (create_workbench) +- Image: Jupyter Data Science Notebook (from list_notebook_images) +- CPU: 2 +- Memory: 8Gi +- Storage: 20Gi + +### Lifecycle +- start_workbench / stop_workbench for running/stopped state +- get_workbench_url: OAuth-protected notebook URL for access + +### Delete Warnings +- delete_workbench: Data loss warning — unsaved work lost, action cannot be undone +- delete_storage: Separate confirmation for PVC deletion — permanent data loss +REPORT_EOF diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/task.toml b/evaluation/without_skills/rh-ai-engineer__workbench-manage/task.toml new file mode 100644 index 00000000..6c538b09 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__workbench-manage/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-ai-engineer__workbench-manage" +name = "rh-ai-engineer Workbench Management Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-ai-engineer", "workbench-manage", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/tests/llm_judge.py b/evaluation/without_skills/rh-ai-engineer__workbench-manage/tests/llm_judge.py new file mode 100644 index 00000000..b7792ec1 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__workbench-manage/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "stop_vs_delete_data", "file": "/root/report.md", "question": "Does the report explain that stopping a workbench preserves PVC data while deleting requires separate storage decision?", "reference": "A skilled report distinguishes stop (preserves) from delete (data loss risk). An unskilled report treats stop and delete equivalently."}, + {"id": "notebook_image_discovery", "file": "/root/report.md", "question": "Does the report describe discovering or listing available notebook images before creating a workbench?", "reference": "A skilled report lists available notebook images (via list_notebook_images or equivalent) to guide workbench creation. An unskilled report skips image discovery and assumes a default."}, + {"id": "storage_access_mode_awareness", "file": "/root/report.md", "question": "Does the report mention the PVC access mode (ReadWriteOnce or RWO) when describing workbench storage configuration or provisioning?", "reference": "A skilled report specifies the storage access mode (ReadWriteOnce) for PVC provisioning, showing awareness of storage class constraints. An unskilled report describes storage size but omits access mode details."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/tests/test.sh b/evaluation/without_skills/rh-ai-engineer__workbench-manage/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__workbench-manage/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/tests/test_outputs.py b/evaluation/without_skills/rh-ai-engineer__workbench-manage/tests/test_outputs.py new file mode 100644 index 00000000..59f74eec --- /dev/null +++ b/evaluation/without_skills/rh-ai-engineer__workbench-manage/tests/test_outputs.py @@ -0,0 +1,73 @@ +""" +Tests for rh-ai-engineer__workbench-manage per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ["workbench", "notebook"]), ( + "report should mention workbench or notebook" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 200, "report should have substantial content" + + +class TestSkillDependent: + def test_stop_preserves_data(self): + """Skill teaches: stopping a workbench preserves PVC data; only delete removes it.""" + c = read_report().lower() + assert any(t in c for t in [ + "stop", "preserve", "data", "pvc", "storage", + "stopped", "restart", "start again", + ]), "should explain that stop preserves data vs delete" + + def test_delete_pvc_warning(self): + """Skill teaches: deleting workbench requires separate confirmation for PVC; warn about permanent data loss.""" + c = read_report().lower() + assert any(t in c for t in [ + "pvc", "delete", "data loss", "permanent", "warning", + "volume", "storage", "backup", "cannot be undone", + ]), "should warn about PVC/data loss on deletion" + + def test_lifecycle_operations(self): + """Skill teaches: create, start, stop, delete with distinct implications.""" + c = read_report().lower() + ops = sum(1 for t in ["start", "stop", "delet", "creat"] if t in c) + assert ops >= 2, "should describe lifecycle operations (create, start, stop, delete)" + + def test_list_notebook_images_tool(self): + """Skill teaches: list_notebook_images MCP tool to discover available notebook images.""" + c = read_report().lower() + assert any(t in c for t in ["list_notebook_images", "notebook images", "available images"]), ( + "should reference list_notebook_images tool (skill)" + ) + + def test_gpu_tuning_awareness(self): + """Docs teach GPU scheduling triage and OOM mitigation using + model/context-size controls for workbenches with GPU resources. + Without docs, agents don't address GPU resource tuning.""" + c = read_report().lower() + assert any(t in c for t in [ + "gpu", "oom", "context size", "max-model-len", "memory", + ]) and any(t in c for t in ["workbench", "notebook", "resource", "gpu"]), ( + "should address GPU/OOM tuning for workbench resources" + ) diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/Dockerfile b/evaluation/without_skills/rh-developer__containerize-deploy/environment/Dockerfile new file mode 100644 index 00000000..b01cae66 --- /dev/null +++ b/evaluation/without_skills/rh-developer__containerize-deploy/environment/Dockerfile @@ -0,0 +1,63 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-developer__containerize-deploy/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/without_skills/rh-developer__containerize-deploy/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/instruction.md b/evaluation/without_skills/rh-developer__containerize-deploy/instruction.md new file mode 100644 index 00000000..42797f3c --- /dev/null +++ b/evaluation/without_skills/rh-developer__containerize-deploy/instruction.md @@ -0,0 +1,15 @@ +# Containerization and Deployment Task + +You are a Red Hat developer. Your team has a Python web application that needs to be containerized and deployed to OpenShift. You need to evaluate the available approaches and recommend the best one. + +## Requirements +- Examine the application source and determine its language, dependencies, and build requirements +- Compare containerization strategies (e.g., S2I, Dockerfile, Helm chart) and explain the trade-offs of each for this application +- If a multi-stage Dockerfile approach is viable, include a working example showing build and runtime stages +- Recommend the best approach with a clear justification +- Define the deployment configuration including: resource requests/limits, all three probe types (startup, liveness, readiness), autoscaling (HPA), and how external traffic will reach the application +- Address application-specific concerns like database connection pooling configuration + +Document your strategy evaluation, recommendation, and deployment plan in `/root/report.md`. + +Use MCP tools to examine the environment. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/solution/solve.sh b/evaluation/without_skills/rh-developer__containerize-deploy/solution/solve.sh new file mode 100644 index 00000000..713efa82 --- /dev/null +++ b/evaluation/without_skills/rh-developer__containerize-deploy/solution/solve.sh @@ -0,0 +1,23 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Containerize and Deploy Plan + +## Phase 1: Detect +- Language: Python +- Framework: Django +- Entry point: manage.py + +## Phase 2: Strategy +- Target: OpenShift +- Strategy: S2I (recommended for Python on OpenShift) +- Alternative: Dockerfile with multi-stage build + +## Phase 3: Build +- Builder image: ubi9/python-311 +- APP_MODULE: myproject.wsgi:application + +## Phase 4: Deploy +- Deployment + Service + Route +- Port: 8000 (Django default) +- On failure: /debug-pod, /debug-build, /debug-network +REPORT_EOF diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/task.toml b/evaluation/without_skills/rh-developer__containerize-deploy/task.toml new file mode 100644 index 00000000..9022cd22 --- /dev/null +++ b/evaluation/without_skills/rh-developer__containerize-deploy/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__containerize-deploy" +name = "rh-developer End-to-End Containerize and Deploy Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "containerize-deploy", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/tests/llm_judge.py b/evaluation/without_skills/rh-developer__containerize-deploy/tests/llm_judge.py new file mode 100644 index 00000000..0dc24c7f --- /dev/null +++ b/evaluation/without_skills/rh-developer__containerize-deploy/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "multistage_and_startup_probe", + "file": "/root/report.md", + "question": "Does the report include both a multi-stage Dockerfile example (with COPY --from=builder or AS builder) AND a startup probe configuration?", + "reference": "A skilled report includes a multi-stage Dockerfile showing build and runtime stages with COPY --from=builder, and configures a startupProbe in addition to liveness/readiness probes. An unskilled report provides only a single-stage Dockerfile and only liveness/readiness probes without startup probe." + }, + { + "id": "hpa_and_pool_config", + "file": "/root/report.md", + "question": "Does the report include a HorizontalPodAutoscaler manifest (with autoscaling/v2 API) AND database connection pool configuration (SQLALCHEMY_POOL or equivalent)?", + "reference": "A skilled report includes a complete HPA YAML with kind: HorizontalPodAutoscaler and autoscaling/v2 API, plus SQLAlchemy connection pool settings (pool_size, pool_recycle). An unskilled report mentions autoscaling conceptually without the manifest, and skips connection pool configuration." + }, + { + "id": "strategy_comparison_depth", + "file": "/root/report.md", + "question": "Does the report compare at least 3 containerization strategies (S2I, Dockerfile, Helm) with specific trade-offs and a justified recommendation?", + "reference": "A skilled report provides a detailed comparison table of S2I, Dockerfile, and Helm with pros/cons/trade-offs for each, leading to a justified recommendation. An unskilled report may compare strategies superficially without detailed trade-offs." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/tests/test.sh b/evaluation/without_skills/rh-developer__containerize-deploy/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-developer__containerize-deploy/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/tests/test_outputs.py b/evaluation/without_skills/rh-developer__containerize-deploy/tests/test_outputs.py new file mode 100644 index 00000000..5f7eec38 --- /dev/null +++ b/evaluation/without_skills/rh-developer__containerize-deploy/tests/test_outputs.py @@ -0,0 +1,110 @@ +""" +Tests for rh-developer__containerize-deploy per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_containerization(self): + content = read_report().lower() + assert any(t in content for t in ["container", "deploy", "image"]), ( + "report should mention containerization or deployment" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_startup_probe(self): + """Skill docs teach startup probe in addition to liveness/readiness. + Without skill, agents typically only include liveness and readiness probes.""" + c = read_report() + assert "startupProbe" in c or "startup probe" in c.lower() or "startupprobe" in c.lower(), ( + "should include startup probe configuration (startupProbe YAML key)" + ) + + def test_multistage_dockerfile_example(self): + """Skill docs teach multi-stage Dockerfile with COPY --from=builder pattern. + Without skill, agents mention multi-stage conceptually but don't provide the example.""" + c = read_report() + assert "COPY --from=" in c or "AS builder" in c or "copy --from=" in c.lower(), ( + "should include a multi-stage Dockerfile example with COPY --from= or AS builder syntax" + ) + + def test_hpa_autoscaling_config(self): + """Skill docs teach complete HPA configuration with autoscaling API. + Without skill, agents mention autoscaling conceptually but skip the manifest.""" + c = read_report() + assert "HorizontalPodAutoscaler" in c or "autoscaling/v2" in c, ( + "should include HorizontalPodAutoscaler manifest or autoscaling/v2 API reference" + ) + + def test_connection_pool_config(self): + """Skill docs teach application-specific database connection pooling with + SQLAlchemy settings. Without skill, agents skip pool configuration details.""" + c = read_report() + assert any(t in c for t in [ + "SQLALCHEMY_POOL", "pool_size", "POOL_SIZE", + "pool_recycle", "POOL_RECYCLE", + ]), "should include SQLAlchemy connection pool settings (pool_size, pool_recycle)" + + def test_strategy_comparison(self): + """Skill teaches comparing at least 2 containerization strategies with trade-offs.""" + c = read_report().lower() + strategies = ["s2i", "dockerfile", "helm", "podman", "source-to-image"] + mentioned = sum(1 for s in strategies if s in c) + assert mentioned >= 2, "should compare at least 2 containerization strategies" + + def test_session_affinity_config(self): + """Skill docs teach explicit sessionAffinity configuration in Service spec. + Without skill, agents skip this detail in the Service definition.""" + c = read_report().lower() + assert "sessionaffinity" in c or "session affinity" in c, ( + "should specify sessionAffinity in Service configuration" + ) + + def test_app_module_s2i_entrypoint(self): + """Skill teaches APP_MODULE environment variable for S2I Python startup + (e.g., app:app). Without skill, agents don't know this S2I-specific + configuration for WSGI entry point discovery.""" + c = read_report() + assert "APP_MODULE" in c or "app:app" in c or "APP_SCRIPT" in c, ( + "should reference APP_MODULE or app:app S2I entrypoint configuration" + ) + + def test_gunicorn_worker_formula(self): + """Skill teaches Gunicorn worker count formula: (2 × CPU cores) + 1. + Without skill, agents hardcode worker count without the sizing formula.""" + c = read_report() + assert any(t in c for t in [ + "2 * cores", "2 × CPU", "(2 * cores) + 1", "2 × cores", + "2*cores", "2 * cpu", "2x CPU", "2 x cores", + ]) or ("worker" in c.lower() and ("formula" in c.lower() or "cores" in c.lower())), ( + "should include Gunicorn worker count formula based on CPU cores" + ) + + def test_sqlalchemy_engine_options(self): + """Skill teaches SQLALCHEMY_ENGINE_OPTIONS configuration for advanced + pool tuning. Without skill, agents configure individual pool parameters + but miss the unified engine options dict.""" + c = read_report() + assert "SQLALCHEMY_ENGINE_OPTIONS" in c or "engine_options" in c, ( + "should include SQLALCHEMY_ENGINE_OPTIONS for advanced pool configuration" + ) diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/Dockerfile b/evaluation/without_skills/rh-developer__debug-build/environment/Dockerfile new file mode 100644 index 00000000..b01cae66 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-build/environment/Dockerfile @@ -0,0 +1,63 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__debug-build/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-build/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__debug-build/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-build/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__debug-build/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-build/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__debug-build/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-build/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__debug-build/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-build/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__debug-build/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-build/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__debug-build/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-build/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__debug-build/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-build/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__debug-build/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-build/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-developer__debug-build/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..5f7e49b1 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-build/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,755 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + { + "name": "api-service-2", + "namespace": "api-platform", + "status": "Failed", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "1m48s", + "reason": "AssembleFailed", + "message": "Assemble script failed with exit code 1", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "api-service-2": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.110.0\n" + "Collecting uvicorn==0.27.1\n" + "Collecting pydantic==2.6.0\n" + "Collecting psycopg2==2.9.9\n" + " ERROR: Could not build wheels for psycopg2, which is required to install pyproject.toml-based projects\n" + " error: subprocess-exited-with-error\n" + " × Running setup.py install for psycopg2 did not run successfully.\n" + " │ exit code: 1\n" + " ╰─> [25 lines of output]\n" + " Error: pg_config executable not found.\n" + " pg_config is required to build psycopg2 from source.\n" + " Please add the directory containing pg_config to the $PATH\n" + " or specify the full executable path with the option:\n" + " python setup.py build_ext --pg-config /path/to/pg_config\n" + " note: This error originates from a subprocess, and is likely not a problem with pip.\n" + "error: legacy-install-failure\n" + "---> Assemble script FAILED with exit code 1\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-developer__debug-build/instruction.md b/evaluation/without_skills/rh-developer__debug-build/instruction.md new file mode 100644 index 00000000..2cfea7f9 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-build/instruction.md @@ -0,0 +1,14 @@ +# Build Debugging Task + +You are a Red Hat developer. An OpenShift Source-to-Image (S2I) build is failing. Investigate the build process to identify and fix the issue. + +## Requirements +- Examine the build configuration and logs +- Identify which S2I build phase is failing (fetch, pull, assemble, commit, push) +- If the fix involves S2I customization, explain how S2I assemble scripts can be extended or overridden +- Provide multiple fix options with concrete commands or file changes, using the appropriate package manager for UBI-based builder images +- Recommend a fix + +Use MCP tools to examine the cluster. Document your methodology, findings, and recommendations in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-developer__debug-build/solution/solve.sh b/evaluation/without_skills/rh-developer__debug-build/solution/solve.sh new file mode 100644 index 00000000..1e0579ec --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-build/solution/solve.sh @@ -0,0 +1,21 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Build Debug Report + +## Build Failure Analysis + +### S2I Build Phases +1. Fetching source ✓ +2. Pulling builder image ✓ +3. **Assemble** ✗ (FAILED) +4. Commit (not reached) +5. Push (not reached) + +### Root Cause +Assemble phase failed — likely dependency installation error in pip install. + +### Fix +- Check requirements.txt for version conflicts (gunicorn, APP_MODULE) +- Verify builder image compatibility (python:3.11-ubi9) +- Retry: `oc start-build flask-app -n myproject --follow` +REPORT_EOF diff --git a/evaluation/without_skills/rh-developer__debug-build/task.toml b/evaluation/without_skills/rh-developer__debug-build/task.toml new file mode 100644 index 00000000..af5ff817 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-build/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__debug-build" +name = "rh-developer Build Debugging Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "debug-build", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-developer__debug-build/tests/llm_judge.py b/evaluation/without_skills/rh-developer__debug-build/tests/llm_judge.py new file mode 100644 index 00000000..7bfd7911 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-build/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "s2i_custom_assemble", + "file": "/root/report.md", + "question": "Does the report mention .s2i/bin/assemble as a way to customize the S2I build process, and reference the default assemble script path at /usr/libexec/s2i/assemble?", + "reference": "A skilled report shows creating a .s2i/bin/assemble script that installs missing packages and then calls /usr/libexec/s2i/assemble (the default assemble script). An unskilled report recommends a custom Dockerfile or builder image instead of using S2I customization hooks." + }, + { + "id": "phase_diagnosis_and_remediation", + "file": "/root/report.md", + "question": "Does the report identify which S2I phase (fetch, assemble, commit, push) failed and provide concrete oc commands for remediation?", + "reference": "A skilled report breaks down the build into phases, identifies the failing phase, and provides actionable commands like 'oc start-build' to retry. An unskilled report gives a generic build failure description." + }, + { + "id": "systematic_build_analysis", + "file": "/root/report.md", + "question": "Does the report follow a systematic approach: inspecting the BuildConfig, analyzing build logs by phase, checking related resources (secrets, imagestreams), and providing structured findings with concrete remediation?", + "reference": "A skilled report follows a structured debugging workflow: BuildConfig analysis, phase-by-phase log analysis, related resource checks, and categorized findings with concrete remediation commands. An unskilled report gives ad-hoc observations without systematic investigation." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-developer__debug-build/tests/test.sh b/evaluation/without_skills/rh-developer__debug-build/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-build/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-developer__debug-build/tests/test_outputs.py b/evaluation/without_skills/rh-developer__debug-build/tests/test_outputs.py new file mode 100644 index 00000000..c3ac3895 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-build/tests/test_outputs.py @@ -0,0 +1,77 @@ +""" +Tests for rh-developer__debug-build per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_build(self): + content = read_report().lower() + assert "build" in content, "report should mention builds" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_s2i_custom_assemble_script(self): + """Skill teaches creating .s2i/bin/assemble to extend the S2I build process. + Without skill, agents recommend Dockerfile or custom builder image instead.""" + c = read_report() + assert ".s2i/bin/assemble" in c or ".s2i/bin" in c, ( + "should mention .s2i/bin/assemble as a way to customize the S2I build" + ) + + def test_default_assemble_path(self): + """Skill teaches invoking the default S2I assemble script at /usr/libexec/s2i/assemble. + Without skill, agents don't know the default script path.""" + c = read_report() + assert "/usr/libexec/s2i/" in c or "libexec/s2i" in c, ( + "should reference the default S2I assemble script at /usr/libexec/s2i/" + ) + + def test_package_manager_awareness(self): + """Report should mention package installation approach for the builder image.""" + c = read_report().lower() + assert any(t in c for t in ["microdnf", "dnf", "yum", "package manager", "install package"]), ( + "should mention package installation approach for the builder image" + ) + + def test_s2i_phase_breakdown(self): + """Skill teaches S2I phases (fetch, pull, assemble, commit, push).""" + c = read_report().lower() + phases = ["assemble", "fetch", "pull", "push", "commit"] + mentioned = sum(1 for p in phases if p in c) + assert mentioned >= 2, ( + "should identify S2I build phases (skill teaches phase-by-phase diagnosis)" + ) + + def test_concrete_remediation_command(self): + """Skill teaches providing concrete oc/command remediation.""" + c = read_report().lower() + assert any(t in c for t in ["oc ", "oc start-build", "oc create", "oc import", "retry"]) or ( + "```" in read_report() and ("oc" in c or "bash" in c) + ), "should include concrete remediation commands" + + def test_dependency_fix_suggestion(self): + """Report should suggest concrete dependency fixes for the failing build.""" + c = read_report().lower() + assert any(t in c for t in [ + "psycopg", "pip install", "requirements", "dependency", "package" + ]), "should suggest concrete dependency fixes for the failing build" diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/Dockerfile b/evaluation/without_skills/rh-developer__debug-container/environment/Dockerfile new file mode 100644 index 00000000..257a1441 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-container/environment/Dockerfile @@ -0,0 +1,67 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "podman": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-podman-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__debug-container/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-container/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__debug-container/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-container/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__debug-container/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-container/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__debug-container/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-container/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__debug-container/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-container/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__debug-container/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-container/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__debug-container/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-container/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__debug-container/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-container/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__debug-container/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-container/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-developer__debug-container/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-container/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/mcp-servers/mock-podman-mcp.py b/evaluation/without_skills/rh-developer__debug-container/environment/mcp-servers/mock-podman-mcp.py new file mode 100644 index 00000000..3d86ba08 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-container/environment/mcp-servers/mock-podman-mcp.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python3 +"""Mock Podman MCP Server for container debugging evaluation. + +Simulates a local Podman environment with several containers, including +one that is crashing (OOMKilled) and one that has an entrypoint error. + +Scenario: + - myapp-web: Exited (137) - OOMKilled, memory limit 256m too low + - myapp-worker: Exited (1) - missing Python dependency 'celery' + - nginx-proxy: Running, healthy + - postgres-db: Running, healthy +""" + +import json +from typing import Optional + +from fastmcp import FastMCP + +mcp = FastMCP("podman") + +NOW = "2026-03-02T12:00:00Z" + +CONTAINERS = { + "a1b2c3d4e5f6": { + "Id": "a1b2c3d4e5f67890abcdef1234567890abcdef1234567890abcdef1234567890", + "Names": ["myapp-web"], + "Image": "myapp:latest", + "ImageID": "sha256:abc123def456789012345678901234567890abcdef1234567890abcdef123456", + "Created": "2026-03-01T10:00:00Z", + "State": { + "Status": "exited", + "Running": False, + "Paused": False, + "Restarting": False, + "OOMKilled": True, + "Dead": False, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-03-01T10:00:05Z", + "FinishedAt": "2026-03-02T08:45:12Z", + }, + "Config": { + "Entrypoint": ["python3"], + "Cmd": ["-m", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"], + "WorkingDir": "/app", + "User": "1001", + "Env": [ + "APP_ENV=production", + "DATABASE_URL=postgresql://db:5432/myapp", + "WORKERS=4", + "MAX_REQUESTS=1000", + ], + "ExposedPorts": {"8080/tcp": {}}, + }, + "HostConfig": { + "Memory": 268435456, + "MemorySwap": 268435456, + "CpuQuota": 100000, + "CpuPeriod": 100000, + "PortBindings": {"8080/tcp": [{"HostIp": "0.0.0.0", "HostPort": "8080"}]}, + "Binds": ["/data/myapp:/app/data:rw"], + }, + "Mounts": [ + {"Type": "bind", "Source": "/data/myapp", "Destination": "/app/data", "Mode": "rw"}, + ], + }, + "b2c3d4e5f6a7": { + "Id": "b2c3d4e5f6a7890123456789abcdef1234567890abcdef1234567890abcdef12", + "Names": ["myapp-worker"], + "Image": "myapp:latest", + "ImageID": "sha256:abc123def456789012345678901234567890abcdef1234567890abcdef123456", + "Created": "2026-03-01T10:00:00Z", + "State": { + "Status": "exited", + "Running": False, + "Paused": False, + "Restarting": False, + "OOMKilled": False, + "Dead": False, + "Pid": 0, + "ExitCode": 1, + "Error": "", + "StartedAt": "2026-03-01T10:00:08Z", + "FinishedAt": "2026-03-01T10:00:12Z", + }, + "Config": { + "Entrypoint": ["python3"], + "Cmd": ["-m", "celery", "-A", "tasks", "worker", "--loglevel=info"], + "WorkingDir": "/app", + "User": "1001", + "Env": [ + "APP_ENV=production", + "DATABASE_URL=postgresql://db:5432/myapp", + "CELERY_BROKER_URL=redis://redis:6379/0", + ], + }, + "HostConfig": { + "Memory": 536870912, + "MemorySwap": 1073741824, + "CpuQuota": 0, + "CpuPeriod": 0, + }, + "Mounts": [], + }, + "c3d4e5f6a7b8": { + "Id": "c3d4e5f6a7b8901234567890abcdef1234567890abcdef1234567890abcdef12", + "Names": ["nginx-proxy"], + "Image": "nginx:1.25", + "ImageID": "sha256:def456789012345678901234567890abcdef1234567890abcdef1234567890ab", + "Created": "2026-02-28T08:00:00Z", + "State": { + "Status": "running", + "Running": True, + "Paused": False, + "Restarting": False, + "OOMKilled": False, + "Dead": False, + "Pid": 12345, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-02-28T08:00:05Z", + "FinishedAt": "0001-01-01T00:00:00Z", + }, + "Config": { + "Entrypoint": ["/docker-entrypoint.sh"], + "Cmd": ["nginx", "-g", "daemon off;"], + "WorkingDir": "", + "User": "", + "Env": ["NGINX_PORT=80"], + "ExposedPorts": {"80/tcp": {}, "443/tcp": {}}, + }, + "HostConfig": { + "Memory": 0, + "MemorySwap": 0, + "CpuQuota": 0, + "CpuPeriod": 0, + "PortBindings": { + "80/tcp": [{"HostIp": "0.0.0.0", "HostPort": "80"}], + "443/tcp": [{"HostIp": "0.0.0.0", "HostPort": "443"}], + }, + }, + "Mounts": [ + {"Type": "bind", "Source": "/etc/nginx/conf.d", "Destination": "/etc/nginx/conf.d", "Mode": "ro"}, + ], + }, + "d4e5f6a7b8c9": { + "Id": "d4e5f6a7b8c9012345678901abcdef1234567890abcdef1234567890abcdef12", + "Names": ["postgres-db"], + "Image": "postgres:15", + "ImageID": "sha256:789012345678901234567890abcdef1234567890abcdef1234567890abcdef12", + "Created": "2026-02-25T12:00:00Z", + "State": { + "Status": "running", + "Running": True, + "Paused": False, + "Restarting": False, + "OOMKilled": False, + "Dead": False, + "Pid": 23456, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-02-25T12:00:10Z", + "FinishedAt": "0001-01-01T00:00:00Z", + }, + "Config": { + "Entrypoint": ["docker-entrypoint.sh"], + "Cmd": ["postgres"], + "WorkingDir": "", + "User": "postgres", + "Env": [ + "POSTGRES_DB=myapp", + "POSTGRES_USER=app", + "PGDATA=/var/lib/postgresql/data", + ], + "ExposedPorts": {"5432/tcp": {}}, + }, + "HostConfig": { + "Memory": 1073741824, + "MemorySwap": 2147483648, + "CpuQuota": 0, + "CpuPeriod": 0, + "PortBindings": {"5432/tcp": [{"HostIp": "127.0.0.1", "HostPort": "5432"}]}, + }, + "Mounts": [ + {"Type": "volume", "Source": "pgdata", "Destination": "/var/lib/postgresql/data", "Mode": "rw"}, + ], + }, +} + +LOGS = { + "myapp-web": ( + "INFO: Started server process [1]\n" + "INFO: Waiting for application startup.\n" + "INFO: Application startup complete.\n" + "INFO: Uvicorn running on http://0.0.0.0:8080\n" + "INFO: Loading ML model into memory...\n" + "INFO: Model size: 1.2GB\n" + "WARNING: Memory usage at 89% of limit (237MB/256MB)\n" + "INFO: Processing request batch (32 items)\n" + "WARNING: Memory usage at 95% of limit (248MB/256MB)\n" + "WARNING: Memory pressure detected, attempting GC\n" + "INFO: GC freed 12MB, usage now at 92%\n" + "INFO: Processing request batch (64 items)\n" + "CRITICAL: Memory usage exceeded limit\n" + "Killed\n" + ), + "myapp-worker": ( + "Traceback (most recent call last):\n" + ' File "/usr/lib/python3.11/runpy.py", line 198, in _run_module_as_main\n' + ' return _run_code(code, main_globals, None,\n' + ' File "/usr/lib/python3.11/runpy.py", line 88, in _run_code\n' + ' exec(code, run_globals)\n' + "ModuleNotFoundError: No module named 'celery'\n" + ), + "nginx-proxy": ( + "2026/02/28 08:00:05 [notice] 1#1: nginx/1.25.4\n" + "2026/02/28 08:00:05 [notice] 1#1: built by gcc 12.2.0\n" + "2026/02/28 08:00:05 [notice] 1#1: OS: Linux 5.14.0-362.el9.x86_64\n" + "2026/02/28 08:00:05 [notice] 1#1: start worker processes\n" + "2026/02/28 08:00:05 [notice] 1#1: start worker process 29\n" + "2026/02/28 08:00:05 [notice] 1#1: start worker process 30\n" + ), + "postgres-db": ( + "PostgreSQL init process complete; ready for start up.\n" + '2026-02-25 12:00:10.123 UTC [1] LOG: starting PostgreSQL 15.5\n' + '2026-02-25 12:00:10.456 UTC [1] LOG: listening on IPv4 address "0.0.0.0", port 5432\n' + '2026-02-25 12:00:10.789 UTC [1] LOG: database system is ready to accept connections\n' + ), +} + +IMAGES = [ + { + "Id": "sha256:abc123def456789012345678901234567890abcdef1234567890abcdef123456", + "RepoTags": ["myapp:latest"], + "Created": "2026-02-28T15:30:00Z", + "Size": 1345678901, + "VirtualSize": 1345678901, + "Labels": {"maintainer": "dev@myapp.io", "version": "2.1.0"}, + "Config": { + "Entrypoint": ["python3"], + "Cmd": ["-m", "uvicorn", "main:app"], + "WorkingDir": "/app", + "ExposedPorts": {"8080/tcp": {}}, + "Env": ["PYTHONDONTWRITEBYTECODE=1", "PYTHONUNBUFFERED=1"], + }, + }, + { + "Id": "sha256:def456789012345678901234567890abcdef1234567890abcdef1234567890ab", + "RepoTags": ["nginx:1.25"], + "Created": "2026-01-15T10:00:00Z", + "Size": 187654321, + "VirtualSize": 187654321, + "Labels": {"maintainer": "NGINX Docker Maintainers"}, + "Config": { + "Entrypoint": ["/docker-entrypoint.sh"], + "Cmd": ["nginx", "-g", "daemon off;"], + "ExposedPorts": {"80/tcp": {}}, + }, + }, + { + "Id": "sha256:789012345678901234567890abcdef1234567890abcdef1234567890abcdef12", + "RepoTags": ["postgres:15"], + "Created": "2026-01-20T12:00:00Z", + "Size": 412345678, + "VirtualSize": 412345678, + "Labels": {"maintainer": "PostgreSQL Docker Maintainers"}, + "Config": { + "Entrypoint": ["docker-entrypoint.sh"], + "Cmd": ["postgres"], + "ExposedPorts": {"5432/tcp": {}}, + }, + }, +] + + +def _find_container(name_or_id: str): + for cid, c in CONTAINERS.items(): + if name_or_id in (cid, c["Id"]): + return c + if name_or_id in c["Names"]: + return c + return None + + +@mcp.tool() +def container_list(all: bool = True) -> str: + """List containers. Set all=True to include stopped containers.""" + results = [] + for cid, c in CONTAINERS.items(): + if not all and not c["State"]["Running"]: + continue + status = c["State"]["Status"] + if c["State"]["OOMKilled"]: + status = f"Exited (137) OOMKilled" + elif c["State"]["ExitCode"] != 0 and not c["State"]["Running"]: + status = f"Exited ({c['State']['ExitCode']})" + elif c["State"]["Running"]: + status = "Up 2 days" + results.append({ + "Id": cid, + "Names": c["Names"], + "Image": c["Image"], + "Status": status, + "Created": c["Created"], + "Ports": list(c["Config"].get("ExposedPorts", {}).keys()), + }) + return json.dumps(results, indent=2) + + +@mcp.tool() +def container_inspect(name: str) -> str: + """Inspect a container by name or ID. Returns detailed configuration and state.""" + c = _find_container(name) + if not c: + raise ValueError(f"no container with name or ID \"{name}\": no such container") + return json.dumps(c, indent=2) + + +@mcp.tool() +def container_logs(name: str, tail: int = 100) -> str: + """Get logs from a container by name or ID.""" + c = _find_container(name) + if not c: + raise ValueError(f"no container with name or ID \"{name}\": no such container") + cname = c["Names"][0] + log = LOGS.get(cname, f"No logs available for {cname}") + return log + + +@mcp.tool() +def container_stats(name: Optional[str] = None) -> str: + """Get resource usage statistics for running containers.""" + results = [] + for cid, c in CONTAINERS.items(): + if name and name not in c["Names"] and name != cid: + continue + if not c["State"]["Running"]: + continue + mem_limit = c["HostConfig"]["Memory"] or 8589934592 + results.append({ + "Id": cid, + "Name": c["Names"][0], + "CPUPerc": "12.5%", + "MemUsage": f"{mem_limit // 4} / {mem_limit}", + "MemPerc": "25.0%", + "NetIO": "1.2MB / 500KB", + "BlockIO": "50MB / 10MB", + "PIDs": 15, + }) + if not results: + return "No running containers found" + (f" matching '{name}'" if name else "") + return json.dumps(results, indent=2) + + +@mcp.tool() +def container_top(name: str) -> str: + """Display the running processes of a container.""" + c = _find_container(name) + if not c: + raise ValueError(f"no container with name or ID \"{name}\": no such container") + if not c["State"]["Running"]: + raise ValueError(f"container {c['Names'][0]} is not running") + return ( + "UID PID PPID C STIME TTY TIME CMD\n" + f"1001 12345 1 0 08:00 ? 00:05:00 {' '.join(c['Config'].get('Cmd', ['']))}\n" + ) + + +@mcp.tool() +def image_list() -> str: + """List all container images.""" + results = [] + for img in IMAGES: + size_mb = img["Size"] // (1024 * 1024) + results.append({ + "Id": img["Id"][:19], + "RepoTags": img["RepoTags"], + "Created": img["Created"], + "Size": f"{size_mb}MB", + "Labels": img.get("Labels", {}), + }) + return json.dumps(results, indent=2) + + +@mcp.tool() +def image_inspect(name: str) -> str: + """Inspect a container image by name or ID.""" + for img in IMAGES: + if name in img["RepoTags"] or name == img["Id"] or img["Id"].startswith(f"sha256:{name}"): + return json.dumps(img, indent=2) + raise ValueError(f"image \"{name}\" not found") + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/without_skills/rh-developer__debug-container/instruction.md b/evaluation/without_skills/rh-developer__debug-container/instruction.md new file mode 100644 index 00000000..52862c6a --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-container/instruction.md @@ -0,0 +1,16 @@ +# Container Debugging Task + +You are a Red Hat developer. Two containers in your local environment have stopped working -- one exited with code 137 and another exited with code 1. Investigate why each container failed and recommend fixes. + +## Requirements +- List all containers (including stopped ones) and identify which are failing +- For each failing container: inspect its configuration, review logs, and check resource limits +- Determine the root cause of each failure (e.g., memory exhaustion, missing dependency, misconfigured entrypoint) +- Recommend a specific fix for each container, including the corrected run command with proper cleanup of the failed container first +- Follow container security best practices (e.g., non-root user) in your fix commands +- Include verification commands to confirm the fix resolved the issue (e.g., checking container state for OOM status) +- If separate image variants would be a better long-term solution, explain that approach + +Document your investigation and fixes in `/root/report.md`. + +Use available tools to examine the environment. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-developer__debug-container/solution/solve.sh b/evaluation/without_skills/rh-developer__debug-container/solution/solve.sh new file mode 100644 index 00000000..421b9a1a --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-container/solution/solve.sh @@ -0,0 +1,18 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Container Debug Report + +## Issue: Container exits immediately + +### Diagnosis +1. `podman inspect` → State.ExitCode: 1, State.OOMKilled: false +2. `podman logs` → Error: entrypoint not found +3. Check image entrypoint/CMD + +### Root Cause +Image entrypoint points to a binary that doesn't exist in the container. + +### Fix +- Override entrypoint: `podman run --entrypoint /bin/sh myimage` +- Or fix Dockerfile CMD/ENTRYPOINT +REPORT_EOF diff --git a/evaluation/without_skills/rh-developer__debug-container/task.toml b/evaluation/without_skills/rh-developer__debug-container/task.toml new file mode 100644 index 00000000..cd098d3a --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-container/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__debug-container" +name = "rh-developer Container Debugging Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "debug-container", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-developer__debug-container/tests/llm_judge.py b/evaluation/without_skills/rh-developer__debug-container/tests/llm_judge.py new file mode 100644 index 00000000..c11e081d --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-container/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "nonroot_user_and_cleanup", + "file": "/root/report.md", + "question": "Does the report include --user 1001 (non-root) in the corrected podman run command AND proper container cleanup (podman stop/rm) before rerunning?", + "reference": "A skilled report includes --user 1001 for container security and shows 'podman stop/rm' cleanup (often with 2>/dev/null || true error suppression) before the corrected run command. An unskilled report omits the --user flag and skips cleanup steps." + }, + { + "id": "image_variant_strategy", + "file": "/root/report.md", + "question": "Does the report recommend separate image variants/tags (e.g., using --build-arg VARIANT=web/worker) for different container roles as a long-term solution?", + "reference": "A skilled report explains that web and worker containers should use separate image tags built with --build-arg VARIANT, rather than sharing a single image. An unskilled report only suggests adding the missing dependency to the shared image." + }, + { + "id": "oomkilled_verification", + "file": "/root/report.md", + "question": "Does the report include verification commands using jq to inspect container state (e.g., podman inspect | jq '.State.OOMKilled')?", + "reference": "A skilled report includes 'podman inspect | jq .State.OOMKilled' to programmatically verify OOM status after fixing. An unskilled report checks logs or status manually without jq-based state inspection." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-developer__debug-container/tests/test.sh b/evaluation/without_skills/rh-developer__debug-container/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-container/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-developer__debug-container/tests/test_outputs.py b/evaluation/without_skills/rh-developer__debug-container/tests/test_outputs.py new file mode 100644 index 00000000..34782966 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-container/tests/test_outputs.py @@ -0,0 +1,93 @@ +""" +Tests for rh-developer__debug-container per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_container(self): + content = read_report().lower() + assert "container" in content, "report should mention container" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_nonroot_user(self): + """Skill teaches running containers as non-root user (--user 1001). + Without skill, agents omit the --user flag in fix commands.""" + c = read_report() + assert "--user" in c or "user 1001" in c.lower(), ( + "should include --user flag for non-root container execution" + ) + + def test_image_variant_strategy(self): + """Skill teaches separate image tags/variants (--build-arg VARIANT=) for + different container roles. Without skill, agents use same image for all roles.""" + c = read_report() + assert "--build-arg" in c or "VARIANT=" in c or "separate image" in c.lower(), ( + "should recommend separate image variants for different roles (web vs worker)" + ) + + def test_oomkilled_state_inspection(self): + """Skill teaches verifying OOMKilled state via container inspect. + Without skill, agents infer OOM from exit code only without inspecting state.""" + c = read_report() + assert any(t in c for t in [ + ".State.OOMKilled", "OOMKilled", "oomkilled", + "State.OOMKilled", "OOMKilled=true", "oomkilled=true", + ]) and any(t in c for t in [ + "inspect", "Inspect", "state", "State", + ]), "should inspect container state to verify OOMKilled" + + def test_cleanup_before_rerun(self): + """Skill teaches proper cleanup (stop + rm with error suppression) before + rerunning a failed container. Without skill, agents skip cleanup.""" + c = read_report() + assert "2>/dev/null" in c or ("podman stop" in c and "podman rm" in c) or ( + "podman rm" in c.lower() and "podman run" in c.lower() + ), "should include container cleanup before rerunning (stop/rm pattern)" + + def test_exit_code_137_oom_mapping(self): + """Skill teaches exit code 137 = OOMKilled, recommend memory increase.""" + c = read_report().lower() + assert ("137" in c or "oom" in c) and "memory" in c, ( + "should map exit 137 to OOM and address memory" + ) + + def test_memory_swap_configuration(self): + """Skill teaches --memory-swap flag for Podman to control total memory + (RAM + swap). Without skill, agents only adjust --memory without swap.""" + c = read_report().lower() + assert "memory-swap" in c or "swap" in c or "memory+swap" in c, ( + "should address memory-swap configuration for container memory limits" + ) + + def test_separate_worker_image(self): + """Skill teaches creating separate container images for different roles + (web vs worker) rather than running all roles from a single image. + Without skill, agents patch the existing single image.""" + c = read_report().lower() + assert any(t in c for t in [ + "separate image", "worker image", "dockerfile.worker", + "dedicated image", "purpose-built", "role-specific", + ]) or ("web" in c and "worker" in c and "image" in c), ( + "should recommend separate images for different container roles" + ) diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/Dockerfile b/evaluation/without_skills/rh-developer__debug-network/environment/Dockerfile new file mode 100644 index 00000000..b01cae66 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-network/environment/Dockerfile @@ -0,0 +1,63 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__debug-network/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-network/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__debug-network/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-network/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__debug-network/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-network/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__debug-network/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-network/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__debug-network/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-network/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__debug-network/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-network/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__debug-network/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-network/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__debug-network/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-network/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__debug-network/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-network/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-developer__debug-network/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-network/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-developer__debug-network/instruction.md b/evaluation/without_skills/rh-developer__debug-network/instruction.md new file mode 100644 index 00000000..c74e95ff --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-network/instruction.md @@ -0,0 +1,12 @@ +# Network Debugging Task + +You are a Red Hat developer. An application is returning HTTP 503 errors when accessed via its Route. Investigate the networking configuration to find the issue. + +## Requirements +- Trace the request path (Route → Service → Pod) +- Identify the network misconfiguration +- Recommend a fix + +Use MCP tools to examine the cluster. Document your methodology, findings, and recommendations in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-developer__debug-network/solution/solve.sh b/evaluation/without_skills/rh-developer__debug-network/solution/solve.sh new file mode 100644 index 00000000..ef071a06 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-network/solution/solve.sh @@ -0,0 +1,19 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Network Debug Report + +## Issue: Route 503 for order-service + +### Root Cause +**Service selector mismatch**: Service selector `app: order-svc` does not match pod label `app: order-service`. + +### Diagnosis +1. Route status: Admitted ✓ +2. Service selector: `app: order-svc` +3. Pod labels: `app: order-service` +4. Endpoints: 0 (no matching pods) +5. Test: `oc run test-curl --rm -i --tty --image=curlimages/curl -- curl -v http://order-service.myns.svc.cluster.local:8080` + +### Fix +Update Service selector to match pod labels: `app: order-service` +REPORT_EOF diff --git a/evaluation/without_skills/rh-developer__debug-network/task.toml b/evaluation/without_skills/rh-developer__debug-network/task.toml new file mode 100644 index 00000000..d8399696 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-network/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__debug-network" +name = "rh-developer Network Debugging Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "debug-network", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-developer__debug-network/tests/llm_judge.py b/evaluation/without_skills/rh-developer__debug-network/tests/llm_judge.py new file mode 100644 index 00000000..3eaeb7d0 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-network/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "route_admitted_vs_exists", + "file": "/root/report.md", + "question": "Does the report check the Route Admitted condition (from the router) rather than just verifying the Route resource exists?", + "reference": "A skilled report checks the Route's Admitted condition which indicates the router has accepted and configured the route. An unskilled report only verifies the Route exists without checking its admission status." + }, + { + "id": "tls_termination_nuances", + "file": "/root/report.md", + "question": "Does the report address TLS termination nuances such as reencrypt requiring destinationCA or passthrough with HTTP backend mismatch?", + "reference": "A skilled report explains that reencrypt TLS termination requires a destinationCA certificate, and that passthrough routes with HTTP-only backends will fail. An unskilled report treats all TLS types as equivalent." + }, + { + "id": "in_cluster_debug_pattern", + "file": "/root/report.md", + "question": "Does the report use a disposable in-cluster curl pod to test internal Service connectivity?", + "reference": "A skilled report creates a temporary curl pod inside the cluster to test Service connectivity from within. An unskilled report only tests external Route access." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-developer__debug-network/tests/test.sh b/evaluation/without_skills/rh-developer__debug-network/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-network/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-developer__debug-network/tests/test_outputs.py b/evaluation/without_skills/rh-developer__debug-network/tests/test_outputs.py new file mode 100644 index 00000000..60293420 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-network/tests/test_outputs.py @@ -0,0 +1,95 @@ +""" +Tests for rh-developer__debug-network per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_network_issue(self): + content = read_report().lower() + assert "503" in content or "network" in content or "route" in content, ( + "report should mention the network issue" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_route_admitted_condition(self): + """Skill teaches Route Admitted condition (from the router) is distinct from + Route just existing. Without skill, agents only check if Route exists.""" + c = read_report().lower() + assert "admitted" in c or "route admitted" in c or ("condition" in c and "route" in c), ( + "should check Route Admitted condition (not just Route existence)" + ) + + def test_empty_endpoints_diagnosis(self): + """Skill teaches checking Endpoints object for empty subsets as the root + cause of 503 errors. Without skill, agents check pod status but not the + Endpoints object directly.""" + c = read_report().lower() + assert ("endpoint" in c and any(t in c for t in [ + "empty", "no endpoint", "none", "no backend", "no subsets", + "0 endpoint", "missing", + ])) or "oc get endpoints" in c or "get ep " in c, ( + "should diagnose empty Endpoints as root cause of 503" + ) + + def test_curl_pod_in_cluster_debug(self): + """Skill teaches using a disposable in-cluster curl pod for debugging + internal connectivity. Without skill, agents test externally only.""" + c = read_report().lower() + assert ("curl" in c and "pod" in c) or "debug pod" in c or "run.*curl" in c or ( + "cluster" in c and "curl" in c + ), "should use in-cluster curl pod for connectivity debugging" + + def test_connectivity_path_tracing(self): + """Skill teaches tracing Route → Service → Endpoints → Pod path.""" + c = read_report().lower() + path_terms = ["route", "service", "endpoint", "pod"] + mentioned = sum(1 for t in path_terms if t in c) + assert mentioned >= 3, "should trace connectivity path (Route→Service→Endpoints→Pod)" + + def test_selector_label_mismatch(self): + """Skill teaches 503 often means selector doesn't match pod labels.""" + c = read_report().lower() + assert any(t in c for t in ["selector", "label", "match", "mismatch"]) and any(t in c for t in [ + "endpoint", "503" + ]), "should identify selector/label mismatch causing no endpoints" + + def test_oc_patch_fix_command(self): + """Skill teaches using oc patch or oc edit for Service selector fixes. + Without skill, agents describe the fix narratively without the actual + command to apply it.""" + c = read_report().lower() + assert any(t in c for t in [ + "oc patch", "oc edit", "kubectl patch", "oc label", + ]) or ("patch" in c and "service" in c), ( + "should include oc patch/edit command for Service selector fix" + ) + + def test_network_policy_awareness(self): + """Skill teaches checking NetworkPolicy as a potential cause of network + issues. Without skill, agents focus only on Service/Route without + considering NetworkPolicy restrictions.""" + c = read_report() + assert "NetworkPolicy" in c or "network policy" in c.lower() or ( + "networkpolic" in c.lower() + ), "should check NetworkPolicy as potential network restriction" diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/Dockerfile b/evaluation/without_skills/rh-developer__debug-pipeline/environment/Dockerfile new file mode 100644 index 00000000..b01cae66 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pipeline/environment/Dockerfile @@ -0,0 +1,63 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-developer__debug-pipeline/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pipeline/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/instruction.md b/evaluation/without_skills/rh-developer__debug-pipeline/instruction.md new file mode 100644 index 00000000..e65370d4 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pipeline/instruction.md @@ -0,0 +1,12 @@ +# Pipeline Debugging Task + +You are a Red Hat developer. A Tekton PipelineRun has failed. Investigate the pipeline to identify which task failed and why. + +## Requirements +- Examine the PipelineRun status and task results +- Identify the failing task and step +- Recommend a fix or retry strategy + +Use MCP tools to examine the cluster. Document your methodology, findings, and recommendations in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/solution/solve.sh b/evaluation/without_skills/rh-developer__debug-pipeline/solution/solve.sh new file mode 100644 index 00000000..f879ab73 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pipeline/solution/solve.sh @@ -0,0 +1,21 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Pipeline Debug Report + +## Failed PipelineRun Analysis + +### Failure Location +- PipelineRun: build-and-deploy-run +- Failed Task: integration-test +- Failed Step: `step-test` (Tekton names step containers as `step-`) + +### Step Logs +Extract from TaskRun pod, container `step-test`. + +### Root Cause +Integration test failed because the service endpoint returned 503. + +### Fix +- Fix the underlying service issue first +- Retry: `tkn pipeline start build-and-deploy --use-pipelinerun build-and-deploy-run` +REPORT_EOF diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/task.toml b/evaluation/without_skills/rh-developer__debug-pipeline/task.toml new file mode 100644 index 00000000..d6025adc --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pipeline/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__debug-pipeline" +name = "rh-developer Pipeline Debugging Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "debug-pipeline", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/tests/llm_judge.py b/evaluation/without_skills/rh-developer__debug-pipeline/tests/llm_judge.py new file mode 100644 index 00000000..ed51f96a --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pipeline/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "step_container_naming", + "file": "/root/report.md", + "question": "Does the report reference the step- container naming convention used in TaskRun pods for targeting logs?", + "reference": "A skilled report knows that Tekton step containers are named step- and uses this to target specific step logs. An unskilled report retrieves pod logs generically without step-level targeting." + }, + { + "id": "taskrun_label_filtering", + "file": "/root/report.md", + "question": "Does the report describe filtering or selecting TaskRuns by their parent PipelineRun (e.g., using tekton.dev/pipelineRun label or equivalent selector), rather than listing all TaskRuns in the namespace?", + "reference": "A skilled report filters TaskRuns by the parent PipelineRun label (tekton.dev/pipelineRun=) to isolate the relevant failure. An unskilled report lists all TaskRuns or checks them one by one without label-based filtering." + }, + { + "id": "hierarchy_diagnosis", + "file": "/root/report.md", + "question": "Does the report systematically drill from PipelineRun → failed TaskRun → step container logs to isolate the failure?", + "reference": "A skilled report follows the PipelineRun→TaskRun→Step hierarchy. An unskilled report checks PipelineRun status without drilling into TaskRun step-level details." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/tests/test.sh b/evaluation/without_skills/rh-developer__debug-pipeline/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pipeline/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/tests/test_outputs.py b/evaluation/without_skills/rh-developer__debug-pipeline/tests/test_outputs.py new file mode 100644 index 00000000..8112bbd2 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pipeline/tests/test_outputs.py @@ -0,0 +1,53 @@ +""" +Tests for rh-developer__debug-pipeline per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_pipeline(self): + content = read_report().lower() + assert "pipeline" in content, "report should mention pipeline" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_pipelinerun_taskrun_hierarchy(self): + """Skill teaches PipelineRun → TaskRun → Step hierarchy to find failure.""" + c = read_report().lower() + assert any(t in c for t in ["pipelinerun", "pipeline run"]) and any(t in c for t in [ + "taskrun", "task run", "task" + ]), "should drill PipelineRun→TaskRun hierarchy" + + def test_concrete_remediation(self): + """Skill teaches distinguishing transient vs config fix needed.""" + c = read_report().lower() + assert any(t in c for t in ["retry", "rerun", "fix", "remediat", "resolv"]), ( + "should provide remediation guidance" + ) + + def test_taskrun_label_filter(self): + """Docs teach filtering TaskRuns by parent pipeline using + tekton.dev/pipelineRun= label. Without docs, agents list all TaskRuns.""" + c = read_report().lower() + assert "tekton.dev/pipelinerun" in c or ("label" in c and "pipelinerun" in c) or ( + "filter" in c and "taskrun" in c + ), "should filter TaskRuns by pipelineRun label" diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/Dockerfile b/evaluation/without_skills/rh-developer__debug-pod/environment/Dockerfile new file mode 100644 index 00000000..b01cae66 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pod/environment/Dockerfile @@ -0,0 +1,63 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-developer__debug-pod/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pod/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-developer__debug-pod/instruction.md b/evaluation/without_skills/rh-developer__debug-pod/instruction.md new file mode 100644 index 00000000..9a983f81 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pod/instruction.md @@ -0,0 +1,14 @@ +# Pod Debugging Task + +You are a Red Hat developer. A pod in the `web-frontend` namespace keeps crashing and restarting. Your team needs you to investigate, identify the root cause, and recommend a fix. + +## Requirements +- Check the pod status and identify the failure pattern (exit code, restart count, state) +- Examine container logs, including logs from previous crashed containers +- Analyze resource limits and requests to determine if the crash is resource-related +- Review namespace events for warnings or errors related to the pod +- Identify the root cause and recommend a specific fix + +Use MCP tools to examine the cluster. Document your methodology, findings, and recommended remediation in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-developer__debug-pod/solution/solve.sh b/evaluation/without_skills/rh-developer__debug-pod/solution/solve.sh new file mode 100644 index 00000000..dca1ff71 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pod/solution/solve.sh @@ -0,0 +1,39 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Pod Debug Report + +## Investigation Summary +A pod in the web-frontend namespace is crashing repeatedly. + +## Pod Status +- Namespace: web-frontend +- Pod: web-frontend (CrashLoopBackOff) +- Exit code: 137 (OOMKilled — SIGKILL, memory limit exceeded) +- Restart count: 8 + +## Diagnosis Methodology +1. Listed pods in web-frontend namespace — found pod in CrashLoopBackOff +2. Examined container status — exit code 137, reason: OOMKilled +3. Checked previous container logs — server starts but gets Killed +4. Reviewed events — OOMKilled warning with memory limit 64Mi +5. Analyzed resource limits — memory limit 64Mi is too low for Node.js + +## Root Cause +Exit 137 = 128 + 9 (SIGKILL). The container was OOMKilled because the memory limit of 64Mi is insufficient for a Node.js application. The application starts normally but is killed when memory usage exceeds the limit during initialization of middleware. + +## Events Analysis +- Warning: OOMKilled — Container exceeded memory limit of 64Mi +- Warning: BackOff — Back-off restarting failed container + +## Recommended Fix +Increase the memory limit for the web-frontend deployment: +- Current: requests=32Mi, limits=64Mi +- Recommended: requests=128Mi, limits=256Mi (or higher depending on app needs) + +This can be applied by patching the deployment resource limits. + +## Additional Notes +- The application logs show it starts successfully but is killed during middleware initialization +- No memory leak — the base memory requirement simply exceeds the configured limit +- Consider monitoring memory usage after the fix to right-size the limits +REPORT_EOF diff --git a/evaluation/without_skills/rh-developer__debug-pod/task.toml b/evaluation/without_skills/rh-developer__debug-pod/task.toml new file mode 100644 index 00000000..89bac572 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pod/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__debug-pod" +name = "rh-developer Pod Debugging Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "debug-pod", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-developer__debug-pod/tests/llm_judge.py b/evaluation/without_skills/rh-developer__debug-pod/tests/llm_judge.py new file mode 100644 index 00000000..3bad1517 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pod/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "previous_logs_methodology", + "file": "/root/report.md", + "question": "Does the report use --previous flag to retrieve logs from crashed containers when restarts are detected?", + "reference": "A skilled report uses --previous to get logs from the terminated container instance when restart count > 0. An unskilled report only checks current container logs, missing crash context." + }, + { + "id": "readiness_endpoint_link", + "file": "/root/report.md", + "question": "Does the report explain that readiness probe failures remove the pod from Service endpoints, causing traffic loss?", + "reference": "A skilled report explains the readiness→endpoints relationship: failed readiness probes remove the pod from Service endpoints. An unskilled report treats readiness as only affecting pod status." + }, + { + "id": "oom_diagnosis_and_fix", + "file": "/root/report.md", + "question": "Does the report map exit code 137 to OOMKilled and provide concrete oc set resources or oc patch commands to increase memory limits?", + "reference": "A skilled report maps 137→OOM and provides actionable oc commands to fix resource limits. An unskilled report may identify OOM but gives vague recommendations." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-developer__debug-pod/tests/test.sh b/evaluation/without_skills/rh-developer__debug-pod/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pod/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-developer__debug-pod/tests/test_outputs.py b/evaluation/without_skills/rh-developer__debug-pod/tests/test_outputs.py new file mode 100644 index 00000000..fda1b3ed --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-pod/tests/test_outputs.py @@ -0,0 +1,75 @@ +""" +Tests for rh-developer__debug-pod per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_pod_or_container(self): + content = read_report().lower() + assert "pod" in content or "container" in content, "report should mention pod or container" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 200, "report should have substantial content" + + +class TestSkillDependent: + def test_previous_logs_flag(self): + """Skill teaches using --previous to get logs from crashed container + when restarts > 0. Without skill, agents only check current logs.""" + c = read_report() + assert "--previous" in c or "previous" in c.lower(), ( + "should use --previous flag to get logs from crashed container" + ) + + def test_readiness_removes_endpoints(self): + """Skill teaches that readiness probe failures remove pod from Service + endpoints, causing traffic loss. Without skill, agents miss this link.""" + c = read_report().lower() + assert ("readiness" in c and "endpoint" in c) or ("readiness" in c and "service" in c) or ( + "readiness" in c and "traffic" in c + ), "should explain readiness failures remove Service endpoints" + + def test_exit_137_oomkilled_mapping(self): + """Skill teaches exit code 137 = OOMKilled, map to memory limit.""" + c = read_report().lower() + assert ("137" in c or "oom" in c or "oomkill" in c) and any(t in c for t in [ + "memory", "limit", "increase" + ]), "should map exit 137 to OOMKilled and memory limit" + + def test_concrete_remediation_command(self): + """Skill teaches oc set resources deployment/... --limits=memory=.""" + c = read_report().lower() + assert any(t in c for t in ["oc set resources", "oc patch", "memory=", "limits"]) or ( + "```" in read_report() and "oc" in c + ), "should include concrete oc remediation command" + + def test_resource_analysis(self): + """Skill teaches analyzing memory request/limit for OOM remediation.""" + c = read_report().lower() + assert any(t in c for t in ["limit", "request"]) and any(t in c for t in [ + "memory", "resource", "increase" + ]), "should analyze resource limits for OOM" + + def test_events_correlation(self): + """Skill teaches checking events for scheduling, OOM, and image pull failures.""" + c = read_report().lower() + assert "event" in c and any(t in c for t in [ + "oom", "schedule", "pull", "fail", "kill", "backoff" + ]), "should correlate pod events with failure cause" diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/Dockerfile b/evaluation/without_skills/rh-developer__debug-rhel/environment/Dockerfile new file mode 100644 index 00000000..4544bdf2 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-rhel/environment/Dockerfile @@ -0,0 +1,67 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "rhel-system": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-rhel-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-developer__debug-rhel/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-rhel/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/mcp-servers/mock-rhel-mcp.py b/evaluation/without_skills/rh-developer__debug-rhel/environment/mcp-servers/mock-rhel-mcp.py new file mode 100644 index 00000000..314f0e3b --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-rhel/environment/mcp-servers/mock-rhel-mcp.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +"""Mock RHEL System MCP Server for RHEL debugging evaluation. + +Simulates a RHEL 9 host with a failing service. Exposes system-level +diagnostic tools (systemctl, journalctl, getenforce, firewall-cmd, ausearch) +as MCP tools so the agent can diagnose the issue. + +Scenario: + Host: app-server-01.example.com (RHEL 9.3) + Failing service: myapp.service + Root causes: + 1. SELinux denial: httpd_t cannot bind to port 9090 + 2. Firewall: port 9090/tcp is not open + 3. Service configuration references correct binary but SELinux blocks it +""" + +import json +from typing import Optional + +from fastmcp import FastMCP + +mcp = FastMCP("rhel-system") + +HOST = "app-server-01.example.com" +RHEL_VER = "9.3" + +SERVICES = { + "myapp.service": { + "loaded": True, + "enabled": True, + "active": "failed", + "sub": "failed", + "description": "My Application Service", + "main_pid": 0, + "exit_code": "exited", + "exit_status": 1, + "exec_start": "/opt/myapp/bin/myapp-server --port 9090 --config /etc/myapp/config.yaml", + "user": "myapp", + "group": "myapp", + "working_directory": "/opt/myapp", + "environment": "APP_ENV=production DB_HOST=localhost DB_PORT=5432", + "restart": "on-failure", + "restart_sec": 5, + "status_output": ( + "● myapp.service - My Application Service\n" + " Loaded: loaded (/etc/systemd/system/myapp.service; enabled; preset: disabled)\n" + " Active: failed (Result: exit-code) since Sun 2026-03-01 18:30:45 UTC; 17h ago\n" + " Process: 45678 ExecStart=/opt/myapp/bin/myapp-server --port 9090 --config /etc/myapp/config.yaml (code=exited, status=1/FAILURE)\n" + " Main PID: 45678 (code=exited, status=1/FAILURE)\n" + " CPU: 125ms\n" + "\n" + "Mar 01 18:30:44 app-server-01 systemd[1]: Starting My Application Service...\n" + "Mar 01 18:30:44 app-server-01 myapp-server[45678]: Starting myapp-server v2.1.0\n" + "Mar 01 18:30:44 app-server-01 myapp-server[45678]: Loading configuration from /etc/myapp/config.yaml\n" + "Mar 01 18:30:45 app-server-01 myapp-server[45678]: Configuration loaded successfully\n" + "Mar 01 18:30:45 app-server-01 myapp-server[45678]: Attempting to bind to 0.0.0.0:9090\n" + "Mar 01 18:30:45 app-server-01 myapp-server[45678]: Error: Permission denied: bind to 0.0.0.0:9090\n" + "Mar 01 18:30:45 app-server-01 myapp-server[45678]: Fatal: Cannot start server, exiting\n" + "Mar 01 18:30:45 app-server-01 systemd[1]: myapp.service: Main process exited, code=exited, status=1/FAILURE\n" + "Mar 01 18:30:45 app-server-01 systemd[1]: myapp.service: Failed with result 'exit-code'.\n" + ), + }, + "sshd.service": { + "loaded": True, + "enabled": True, + "active": "active", + "sub": "running", + "description": "OpenSSH server daemon", + "main_pid": 1234, + "exit_code": "", + "exit_status": 0, + }, + "firewalld.service": { + "loaded": True, + "enabled": True, + "active": "active", + "sub": "running", + "description": "firewalld - dynamic firewall daemon", + "main_pid": 2345, + "exit_code": "", + "exit_status": 0, + }, + "postgresql.service": { + "loaded": True, + "enabled": True, + "active": "active", + "sub": "running", + "description": "PostgreSQL database server", + "main_pid": 3456, + "exit_code": "", + "exit_status": 0, + }, +} + +JOURNAL_LOGS = { + "myapp.service": ( + "-- Journal begins at Sat 2026-02-28 00:00:00 UTC, ends at Sun 2026-03-02 12:00:00 UTC. --\n" + "Mar 01 18:30:44 app-server-01 systemd[1]: Starting My Application Service...\n" + "Mar 01 18:30:44 app-server-01 myapp-server[45678]: Starting myapp-server v2.1.0\n" + "Mar 01 18:30:44 app-server-01 myapp-server[45678]: Loading configuration from /etc/myapp/config.yaml\n" + "Mar 01 18:30:45 app-server-01 myapp-server[45678]: Configuration loaded successfully\n" + "Mar 01 18:30:45 app-server-01 myapp-server[45678]: Connecting to database at localhost:5432... OK\n" + "Mar 01 18:30:45 app-server-01 myapp-server[45678]: Attempting to bind to 0.0.0.0:9090\n" + "Mar 01 18:30:45 app-server-01 myapp-server[45678]: Error: Permission denied: bind to 0.0.0.0:9090\n" + "Mar 01 18:30:45 app-server-01 myapp-server[45678]: Fatal: Cannot start server, exiting\n" + "Mar 01 18:30:45 app-server-01 systemd[1]: myapp.service: Main process exited, code=exited, status=1/FAILURE\n" + "Mar 01 18:30:45 app-server-01 systemd[1]: myapp.service: Failed with result 'exit-code'.\n" + "Mar 01 18:30:50 app-server-01 systemd[1]: myapp.service: Scheduled restart job, restart counter is at 1.\n" + "Mar 01 18:30:50 app-server-01 systemd[1]: Starting My Application Service...\n" + "Mar 01 18:30:50 app-server-01 myapp-server[45690]: Starting myapp-server v2.1.0\n" + "Mar 01 18:30:51 app-server-01 myapp-server[45690]: Loading configuration from /etc/myapp/config.yaml\n" + "Mar 01 18:30:51 app-server-01 myapp-server[45690]: Configuration loaded successfully\n" + "Mar 01 18:30:51 app-server-01 myapp-server[45690]: Connecting to database at localhost:5432... OK\n" + "Mar 01 18:30:51 app-server-01 myapp-server[45690]: Attempting to bind to 0.0.0.0:9090\n" + "Mar 01 18:30:51 app-server-01 myapp-server[45690]: Error: Permission denied: bind to 0.0.0.0:9090\n" + "Mar 01 18:30:51 app-server-01 myapp-server[45690]: Fatal: Cannot start server, exiting\n" + "Mar 01 18:30:51 app-server-01 systemd[1]: myapp.service: Main process exited, code=exited, status=1/FAILURE\n" + "Mar 01 18:30:51 app-server-01 systemd[1]: myapp.service: Failed with result 'exit-code'.\n" + "Mar 01 18:30:56 app-server-01 systemd[1]: myapp.service: Scheduled restart job, restart counter is at 2.\n" + "Mar 01 18:30:56 app-server-01 systemd[1]: Starting My Application Service...\n" + "Mar 01 18:30:56 app-server-01 myapp-server[45705]: Starting myapp-server v2.1.0\n" + "Mar 01 18:30:57 app-server-01 myapp-server[45705]: Error: Permission denied: bind to 0.0.0.0:9090\n" + "Mar 01 18:30:57 app-server-01 myapp-server[45705]: Fatal: Cannot start server, exiting\n" + "Mar 01 18:30:57 app-server-01 systemd[1]: myapp.service: Main process exited, code=exited, status=1/FAILURE\n" + "Mar 01 18:30:57 app-server-01 systemd[1]: myapp.service: Failed with result 'exit-code'.\n" + "Mar 01 18:30:57 app-server-01 systemd[1]: myapp.service: Start request repeated too quickly.\n" + "Mar 01 18:30:57 app-server-01 systemd[1]: myapp.service: Failed with result 'exit-code'.\n" + ), +} + + +@mcp.tool() +def systemctl_status(service: str) -> str: + """Get the status of a systemd service (equivalent to 'systemctl status ').""" + svc = SERVICES.get(service) + if not svc: + return f"Unit {service} could not be found." + + if svc.get("status_output"): + return svc["status_output"] + + state = "active (running)" if svc["active"] == "active" else "failed" + return ( + f"● {service} - {svc['description']}\n" + f" Loaded: loaded (/usr/lib/systemd/system/{service}; " + f"{'enabled' if svc['enabled'] else 'disabled'}; preset: disabled)\n" + f" Active: {state}\n" + f" Main PID: {svc['main_pid']}\n" + ) + + +@mcp.tool() +def systemctl_list_failed() -> str: + """List all failed systemd services (equivalent to 'systemctl --failed').""" + failed = [(name, svc) for name, svc in SERVICES.items() if svc["active"] == "failed"] + if not failed: + return "0 loaded units listed." + + lines = [" UNIT LOAD ACTIVE SUB DESCRIPTION"] + for name, svc in failed: + lines.append( + f" {name:<24s} loaded failed failed {svc['description']}" + ) + lines.append(f"\n{len(failed)} loaded units listed.") + return "\n".join(lines) + + +@mcp.tool() +def journalctl(unit: Optional[str] = None, lines: int = 100, priority: Optional[str] = None) -> str: + """Get journal logs, optionally filtered by unit or priority.""" + if unit and unit in JOURNAL_LOGS: + log = JOURNAL_LOGS[unit] + if priority and priority in ("err", "3"): + return "\n".join( + line for line in log.split("\n") + if "Error" in line or "Fatal" in line or "FAILURE" in line or "failed" in line.lower() + ) + return log + + if unit: + return f"-- No entries for unit {unit} --" + + return ( + "-- Journal begins at Sat 2026-02-28 00:00:00 UTC --\n" + "Mar 02 12:00:00 app-server-01 kernel: Linux version 5.14.0-362.el9.x86_64\n" + "Mar 02 12:00:00 app-server-01 systemd[1]: Started system.\n" + ) + + +@mcp.tool() +def getenforce() -> str: + """Get SELinux enforcement mode (equivalent to 'getenforce').""" + return "Enforcing" + + +@mcp.tool() +def ausearch_avc(recent: bool = True, comm: Optional[str] = None) -> str: + """Search for SELinux AVC denial messages (equivalent to 'ausearch -m AVC').""" + denials = [ + { + "timestamp": "Mar 01 18:30:45", + "type": "AVC", + "result": "denied", + "permission": "name_bind", + "scontext": "system_u:system_r:httpd_t:s0", + "tcontext": "system_u:object_r:unreserved_port_t:s0", + "tclass": "tcp_socket", + "comm": "myapp-server", + "port": 9090, + }, + { + "timestamp": "Mar 01 18:30:50", + "type": "AVC", + "result": "denied", + "permission": "name_bind", + "scontext": "system_u:system_r:httpd_t:s0", + "tcontext": "system_u:object_r:unreserved_port_t:s0", + "tclass": "tcp_socket", + "comm": "myapp-server", + "port": 9090, + }, + { + "timestamp": "Mar 01 18:30:56", + "type": "AVC", + "result": "denied", + "permission": "name_bind", + "scontext": "system_u:system_r:httpd_t:s0", + "tcontext": "system_u:object_r:unreserved_port_t:s0", + "tclass": "tcp_socket", + "comm": "myapp-server", + "port": 9090, + }, + ] + + if comm: + denials = [d for d in denials if d["comm"] == comm] + + if not denials: + return "No AVC denials found." + + lines = [] + for d in denials: + lines.append( + f"----\n" + f"time->{d['timestamp']}\n" + f"type=AVC msg=audit: avc: denied {{ {d['permission']} }} for " + f"comm=\"{d['comm']}\" " + f"src={d['port']} " + f"scontext={d['scontext']} " + f"tcontext={d['tcontext']} " + f"tclass={d['tclass']} permissive=0" + ) + return "\n".join(lines) + + +@mcp.tool() +def firewall_cmd_state() -> str: + """Check if firewalld is running (equivalent to 'firewall-cmd --state').""" + return "running" + + +@mcp.tool() +def firewall_cmd_list_all() -> str: + """List all firewall rules for the default zone (equivalent to 'firewall-cmd --list-all').""" + return ( + "public (active)\n" + " target: default\n" + " icmp-block-inversion: no\n" + " interfaces: eth0\n" + " sources: \n" + " services: cockpit dhcpv6-client ssh\n" + " ports: 5432/tcp\n" + " protocols: \n" + " forward: yes\n" + " masquerade: no\n" + " forward-ports: \n" + " source-ports: \n" + " icmp-blocks: \n" + " rich rules: \n" + ) + + +@mcp.tool() +def firewall_cmd_query_port(port: str) -> str: + """Check if a specific port is open in the firewall (e.g. '9090/tcp').""" + open_ports = {"5432/tcp", "22/tcp"} + if port in open_ports: + return "yes" + return "no" + + +@mcp.tool() +def semanage_port_list(port_type: Optional[str] = None) -> str: + """List SELinux port type assignments (equivalent to 'semanage port -l').""" + entries = [ + ("http_port_t", "tcp", "80, 81, 443, 488, 8008, 8009, 8443, 9000"), + ("ssh_port_t", "tcp", "22"), + ("postgresql_port_t", "tcp", "5432"), + ("unreserved_port_t", "tcp", "1024-32767"), + ("unreserved_port_t", "udp", "1024-32767"), + ] + if port_type: + entries = [(t, p, ports) for t, p, ports in entries if t == port_type] + + lines = ["SELinux Port Type Proto Port Number"] + for t, p, ports in entries: + lines.append(f"{t:<26s} {p:<8s} {ports}") + return "\n".join(lines) + + +@mcp.tool() +def system_info() -> str: + """Get basic system information (hostname, OS, kernel, uptime).""" + return json.dumps({ + "hostname": HOST, + "os": f"Red Hat Enterprise Linux {RHEL_VER}", + "kernel": "5.14.0-362.el9.x86_64", + "arch": "x86_64", + "uptime": "15 days, 3:42", + "load_average": "0.45, 0.38, 0.32", + "memory": { + "total": "16384 MB", + "used": "5120 MB", + "free": "8192 MB", + "available": "11264 MB", + }, + "disk": { + "/": {"total": "50G", "used": "18G", "available": "32G", "use_percent": "36%"}, + "/var": {"total": "100G", "used": "45G", "available": "55G", "use_percent": "45%"}, + }, + }, indent=2) + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/evaluation/without_skills/rh-developer__debug-rhel/instruction.md b/evaluation/without_skills/rh-developer__debug-rhel/instruction.md new file mode 100644 index 00000000..ca2ade3a --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-rhel/instruction.md @@ -0,0 +1,12 @@ +# RHEL System Debugging Task + +You are a Red Hat developer. A RHEL-based service is failing to start or accept connections. Investigate the system configuration to identify the issue. + +## Requirements +- Check service status, SELinux, and firewall configuration +- Identify the system-level root cause +- Recommend a fix + +Use available tools to examine the environment. Document your methodology, findings, and recommendations in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-developer__debug-rhel/solution/solve.sh b/evaluation/without_skills/rh-developer__debug-rhel/solution/solve.sh new file mode 100644 index 00000000..350dd5d5 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-rhel/solution/solve.sh @@ -0,0 +1,36 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# RHEL Debug Report + +## Issue: Flask app can't bind to port 8080 + +### Systemd Check +```bash +systemctl status flask-app +``` + +### Journal Logs +```bash +journalctl -u flask-app -n 100 +``` + +### SELinux Check +```bash +getenforce +ausearch -m AVC -ts recent +# Found: denied bind to port 8080 +``` + +### Fix: Add port to SELinux +```bash +sudo semanage port -a -t http_port_t -p tcp 8080 +sudo restorecon -Rv /opt/flask-app +``` + +### Firewall Check +```bash +sudo firewall-cmd --list-all +sudo firewall-cmd --permanent --add-port=8080/tcp +sudo firewall-cmd --reload +``` +REPORT_EOF diff --git a/evaluation/without_skills/rh-developer__debug-rhel/task.toml b/evaluation/without_skills/rh-developer__debug-rhel/task.toml new file mode 100644 index 00000000..32fb504b --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-rhel/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__debug-rhel" +name = "rh-developer RHEL Deployment Debugging Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "debug-rhel", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-developer__debug-rhel/tests/llm_judge.py b/evaluation/without_skills/rh-developer__debug-rhel/tests/llm_judge.py new file mode 100644 index 00000000..e170f4bb --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-rhel/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "ausearch_avc_workflow", + "file": "/root/report.md", + "question": "Does the report use ausearch -m AVC for investigating SELinux denials, rather than generic SELinux commands?", + "reference": "A skilled report uses 'ausearch -m AVC -ts recent' to find recent SELinux AVC denials. An unskilled report checks getenforce or sestatus without examining specific denials." + }, + { + "id": "semanage_port_labeling", + "file": "/root/report.md", + "question": "Does the report use semanage port for labeling nonstandard bind ports in SELinux?", + "reference": "A skilled report uses 'semanage port -a -t http_port_t -p tcp ' for nonstandard ports. An unskilled report suggests disabling SELinux or only uses setsebool." + }, + { + "id": "concrete_rhel_remediation", + "file": "/root/report.md", + "question": "Does the report provide concrete systemctl, firewall-cmd, and semanage/restorecon commands for RHEL troubleshooting?", + "reference": "A skilled report provides specific commands for each layer: systemctl restart for services, firewall-cmd --add-port for networking, semanage+restorecon for SELinux. An unskilled report gives high-level suggestions." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-developer__debug-rhel/tests/test.sh b/evaluation/without_skills/rh-developer__debug-rhel/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-rhel/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-developer__debug-rhel/tests/test_outputs.py b/evaluation/without_skills/rh-developer__debug-rhel/tests/test_outputs.py new file mode 100644 index 00000000..6ba9216b --- /dev/null +++ b/evaluation/without_skills/rh-developer__debug-rhel/tests/test_outputs.py @@ -0,0 +1,97 @@ +""" +Tests for rh-developer__debug-rhel per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_rhel_or_system(self): + content = read_report().lower() + assert "rhel" in content or "system" in content or "service" in content, ( + "report should mention RHEL or system" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_ausearch_avc_command(self): + """Skill teaches ausearch -m AVC -ts recent for recent SELinux denials. + Without skill, agents use generic SELinux checks without ausearch.""" + c = read_report().lower() + assert "ausearch" in c, ( + "should use ausearch for SELinux AVC denial investigation" + ) + + def test_semanage_port_labeling(self): + """Skill teaches semanage port for nonstandard bind port SELinux labeling. + Without skill, agents skip port-level SELinux context management.""" + c = read_report().lower() + assert "semanage port" in c or ("semanage" in c and "port" in c), ( + "should use semanage port for nonstandard port SELinux labeling" + ) + + def test_systemd_journal_workflow(self): + """Skill teaches systemctl status + journalctl -u for service logs.""" + c = read_report().lower() + assert any(t in c for t in ["systemctl", "journalctl"]) and any(t in c for t in [ + "status", "-u", "service", "log" + ]), "should use systemd/journal workflow" + + def test_firewall_cmd(self): + """Skill teaches firewall-cmd for port management.""" + c = read_report().lower() + assert "firewall-cmd" in c or "firewall" in c, ( + "should check firewall configuration" + ) + + def test_concrete_remediation(self): + """Skill teaches concrete remediation commands for RHEL issues.""" + c = read_report().lower() + assert any(t in c for t in ["systemctl restart", "firewall-cmd", "semanage", "restorecon"]) or ( + "```" in read_report() and any(t in c for t in ["sudo", "systemctl"]) + ), "should include concrete RHEL remediation commands" + + def test_permanent_firewall_flag(self): + """Skill teaches using --permanent flag with firewall-cmd to persist rules + across reboots. Without skill, agents use firewall-cmd without --permanent, + creating rules that are lost on reboot.""" + c = read_report() + assert "--permanent" in c, ( + "should use --permanent flag with firewall-cmd for persistent rules" + ) + + def test_http_port_t_selinux_type(self): + """Skill teaches the specific SELinux type http_port_t for web service ports. + Without skill, agents use generic semanage commands without specifying the + correct SELinux type for HTTP ports.""" + c = read_report() + assert "http_port_t" in c, ( + "should reference http_port_t SELinux type for port labeling" + ) + + def test_getenforce_check(self): + """Skill teaches using getenforce to verify SELinux mode (Enforcing/Permissive) + as a first diagnostic step. Without skill, agents jump to specific SELinux + fixes without verifying the enforcement mode.""" + c = read_report().lower() + assert "getenforce" in c, ( + "should use getenforce to check SELinux enforcement mode" + ) diff --git a/evaluation/without_skills/rh-developer__deploy/environment/Dockerfile b/evaluation/without_skills/rh-developer__deploy/environment/Dockerfile new file mode 100644 index 00000000..b01cae66 --- /dev/null +++ b/evaluation/without_skills/rh-developer__deploy/environment/Dockerfile @@ -0,0 +1,63 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-developer__deploy/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__deploy/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/without_skills/rh-developer__deploy/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__deploy/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__deploy/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/without_skills/rh-developer__deploy/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/without_skills/rh-developer__deploy/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__deploy/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/without_skills/rh-developer__deploy/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/without_skills/rh-developer__deploy/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__deploy/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/without_skills/rh-developer__deploy/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__deploy/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__deploy/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/without_skills/rh-developer__deploy/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/without_skills/rh-developer__deploy/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__deploy/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/without_skills/rh-developer__deploy/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__deploy/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__deploy/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/without_skills/rh-developer__deploy/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__deploy/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__deploy/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/without_skills/rh-developer__deploy/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/without_skills/rh-developer__deploy/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__deploy/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/without_skills/rh-developer__deploy/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__deploy/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-developer__deploy/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/without_skills/rh-developer__deploy/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-developer__deploy/instruction.md b/evaluation/without_skills/rh-developer__deploy/instruction.md new file mode 100644 index 00000000..f84c6177 --- /dev/null +++ b/evaluation/without_skills/rh-developer__deploy/instruction.md @@ -0,0 +1,14 @@ +# Application Deployment Task + +You are a Red Hat developer. Your team needs to deploy a web application to OpenShift that will be accessible to external users via HTTPS. + +## Requirements +- Examine the target namespace and available resources on the cluster +- Define the deployment: container image, replica count, resource requests and limits, and health checks +- Configure a Service to expose the application pods internally +- Configure a Route for external HTTPS access with appropriate TLS settings +- Verify the deployment plan addresses image pull access and correct container port mapping + +Document your deployment plan and the complete resource definitions in `/root/report.md`. + +Use MCP tools to examine the cluster. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-developer__deploy/solution/solve.sh b/evaluation/without_skills/rh-developer__deploy/solution/solve.sh new file mode 100644 index 00000000..b8f9ec1b --- /dev/null +++ b/evaluation/without_skills/rh-developer__deploy/solution/solve.sh @@ -0,0 +1,61 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Deployment Plan: customer-portal + +## Deployment +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: customer-portal +spec: + replicas: 1 + selector: + matchLabels: + app: customer-portal + template: + metadata: + labels: + app: customer-portal + spec: + containers: + - name: customer-portal + image: image-registry.openshift-image-registry.svc:5000/myproject/customer-portal:latest + ports: + - containerPort: 3000 +``` + +## Service +```yaml +apiVersion: v1 +kind: Service +metadata: + name: customer-portal +spec: + selector: + app: customer-portal + ports: + - port: 3000 + targetPort: 3000 +``` + +## Route +```yaml +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: customer-portal +spec: + to: + kind: Service + name: customer-portal + port: + targetPort: 3000 + tls: + termination: edge +``` + +### Internal DNS: `http://customer-portal.myproject.svc.cluster.local:3000` + +### On failure: Debug Pod (/debug-pod) or Debug Network (/debug-network) +REPORT_EOF diff --git a/evaluation/without_skills/rh-developer__deploy/task.toml b/evaluation/without_skills/rh-developer__deploy/task.toml new file mode 100644 index 00000000..86e6c127 --- /dev/null +++ b/evaluation/without_skills/rh-developer__deploy/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__deploy" +name = "rh-developer Deployment Planning Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "deploy", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-developer__deploy/tests/llm_judge.py b/evaluation/without_skills/rh-developer__deploy/tests/llm_judge.py new file mode 100644 index 00000000..5ce75615 --- /dev/null +++ b/evaluation/without_skills/rh-developer__deploy/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "security_hardening", + "file": "/root/report.md", + "question": "Does the report include deployment security hardening such as runAsNonRoot, allowPrivilegeEscalation: false, seccompProfile, or insecureEdgeTerminationPolicy: Redirect on the Route?", + "reference": "A skilled report includes security context on the Deployment (runAsNonRoot: true, allowPrivilegeEscalation: false) and configures Route with insecureEdgeTerminationPolicy: Redirect. An unskilled report creates basic Deployment+Service+Route without security hardening." + }, + { + "id": "deployment_service_route", + "file": "/root/report.md", + "question": "Does the report create all three resources (Deployment, Service, Route) with correct selector/port alignment?", + "reference": "A skilled report defines Deployment + Service + Route with matching selectors, targetPort, and containerPort. An unskilled report may miss selector alignment or skip the Route." + }, + { + "id": "tls_and_port_detection", + "file": "/root/report.md", + "question": "Does the report address TLS termination for the Route and port detection based on framework defaults?", + "reference": "A skilled report configures TLS (edge/passthrough) on the Route and detects the application port from framework conventions. An unskilled report hardcodes port 8080 and skips TLS." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-developer__deploy/tests/test.sh b/evaluation/without_skills/rh-developer__deploy/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-developer__deploy/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-developer__deploy/tests/test_outputs.py b/evaluation/without_skills/rh-developer__deploy/tests/test_outputs.py new file mode 100644 index 00000000..01ea8257 --- /dev/null +++ b/evaluation/without_skills/rh-developer__deploy/tests/test_outputs.py @@ -0,0 +1,87 @@ +""" +Tests for rh-developer__deploy per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_deploy(self): + content = read_report().lower() + assert "deploy" in content, "report should mention deployment" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_insecure_redirect_policy(self): + """Skill teaches insecureEdgeTerminationPolicy: Redirect on Route to force + HTTP→HTTPS. Without skill, agents create Routes without redirect policy, + leaving HTTP access open.""" + c = read_report() + assert "insecureEdgeTerminationPolicy" in c or ( + "Redirect" in c and ("http" in c.lower() and "https" in c.lower()) + ), "should configure insecureEdgeTerminationPolicy: Redirect on Route" + + def test_framework_port_detection(self): + """Skill teaches port inference by framework defaults (Node 3000/8080, + Python 5000/8000, Java 8080). Without skill, agents hardcode 8080.""" + c = read_report().lower() + assert any(t in c for t in ["port", "8080", "3000", "5000"]) and any(t in c for t in [ + "detect", "expose", "listen", "framework", "default", "infer" + ]), "should address port detection from framework defaults" + + def test_deployment_service_route_triad(self): + """Skill teaches creating Deployment, Service, Route in sequence.""" + c = read_report().lower() + assert any(t in c for t in ["deployment"]) and "service" in c and any(t in c for t in [ + "route", "external", "https" + ]), "should define Deployment + Service + Route" + + def test_selector_alignment(self): + """Skill teaches Service selector must match Deployment pod labels.""" + c = read_report().lower() + assert any(t in c for t in ["selector", "label", "targetport", "target port"]) or ( + "service" in c and "port" in c and "match" in c + ), "should address selector/port alignment" + + def test_tls_route_config(self): + """Skill teaches Route with TLS termination (edge/passthrough).""" + c = read_report().lower() + assert any(t in c for t in ["tls", "https", "edge", "termination"]), ( + "should address Route TLS for external access" + ) + + def test_hpa_autoscaling(self): + """Skill teaches including HorizontalPodAutoscaler configuration for + production deployments. Without skill, agents set static replica count + without autoscaling.""" + c = read_report() + assert "HorizontalPodAutoscaler" in c or "autoscaling/v2" in c or ( + "hpa" in c.lower() and "autoscal" in c.lower() + ), "should include HorizontalPodAutoscaler for production scaling" + + def test_hsts_security_headers(self): + """Skill teaches HSTS headers or Strict-Transport-Security configuration + on OpenShift Routes. Without skill, agents skip transport security headers.""" + c = read_report() + assert any(t in c for t in [ + "HSTS", "Strict-Transport-Security", "hsts", + "haproxy.router.openshift.io", + ]), "should configure HSTS or transport security headers on Route" diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/Dockerfile b/evaluation/without_skills/rh-developer__detect-project/environment/Dockerfile new file mode 100644 index 00000000..e9a7788a --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/Dockerfile @@ -0,0 +1,64 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs +COPY sample-project /root/project + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__detect-project/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__detect-project/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__detect-project/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__detect-project/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__detect-project/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__detect-project/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__detect-project/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__detect-project/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__detect-project/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-developer__detect-project/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/sample-project/.s2i/environment b/evaluation/without_skills/rh-developer__detect-project/environment/sample-project/.s2i/environment new file mode 100644 index 00000000..a16a265c --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/sample-project/.s2i/environment @@ -0,0 +1 @@ +APP_FILE=app.py diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/sample-project/Dockerfile b/evaluation/without_skills/rh-developer__detect-project/environment/sample-project/Dockerfile new file mode 100644 index 00000000..a7fb87b7 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/sample-project/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.11-slim + +WORKDIR /app +COPY requirements.txt . +RUN pip install -r requirements.txt +COPY . . + +EXPOSE 8080 +CMD ["gunicorn", "-b", "0.0.0.0:8080", "app:app"] diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/sample-project/app.py b/evaluation/without_skills/rh-developer__detect-project/environment/sample-project/app.py new file mode 100644 index 00000000..4761fe8a --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/sample-project/app.py @@ -0,0 +1,12 @@ +from flask import Flask + +app = Flask(__name__) + + +@app.route("/") +def hello(): + return "Hello, World!" + + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=8080) diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/sample-project/requirements.txt b/evaluation/without_skills/rh-developer__detect-project/environment/sample-project/requirements.txt new file mode 100644 index 00000000..cb04ebda --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/sample-project/requirements.txt @@ -0,0 +1,3 @@ +flask +gunicorn +psycopg2-binary diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/sample-project/tests/test_app.py b/evaluation/without_skills/rh-developer__detect-project/environment/sample-project/tests/test_app.py new file mode 100644 index 00000000..5e8fbc93 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/sample-project/tests/test_app.py @@ -0,0 +1,9 @@ +import pytest +from app import app + + +def test_hello(): + with app.test_client() as client: + r = client.get("/") + assert r.status_code == 200 + assert b"Hello" in r.data diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/templates/buildconfig.yaml.template b/evaluation/without_skills/rh-developer__detect-project/environment/templates/buildconfig.yaml.template new file mode 100644 index 00000000..b3294eb2 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/templates/buildconfig.yaml.template @@ -0,0 +1,38 @@ +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: build + app.kubernetes.io/part-of: ${APP_NAME} +spec: + source: + type: Git + git: + uri: ${GIT_URL} + ref: ${GIT_BRANCH} + strategy: + type: Source + sourceStrategy: + from: + kind: DockerImage + name: ${BUILDER_IMAGE} + env: [] + output: + to: + kind: ImageStreamTag + name: ${APP_NAME}:latest + triggers: + - type: ConfigChange + - type: ImageChange + runPolicy: Serial + resources: + limits: + memory: "1Gi" + cpu: "1" + requests: + memory: "512Mi" + cpu: "500m" diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/templates/deployment.yaml.template b/evaluation/without_skills/rh-developer__detect-project/environment/templates/deployment.yaml.template new file mode 100644 index 00000000..eb3b481a --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/templates/deployment.yaml.template @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: application + app.kubernetes.io/part-of: ${APP_NAME} + annotations: + image.openshift.io/triggers: | + [{"from":{"kind":"ImageStreamTag","name":"${APP_NAME}:latest"},"fieldPath":"spec.template.spec.containers[0].image"}] +spec: + replicas: ${REPLICAS} + selector: + matchLabels: + app: ${APP_NAME} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + template: + metadata: + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + spec: + containers: + - name: ${APP_NAME} + image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/${APP_NAME}:latest + ports: + - containerPort: ${CONTAINER_PORT} + protocol: TCP + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + livenessProbe: + httpGet: + path: / + port: ${CONTAINER_PORT} + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: / + port: ${CONTAINER_PORT} + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + env: [] + restartPolicy: Always + terminationGracePeriodSeconds: 30 diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/Chart.yaml.template b/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/Chart.yaml.template new file mode 100644 index 00000000..1aa22dd1 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/Chart.yaml.template @@ -0,0 +1,13 @@ +apiVersion: v2 +name: ${APP_NAME} +description: ${APP_DESCRIPTION} +type: application +version: 0.1.0 +appVersion: "${APP_VERSION}" +keywords: + - ${LANGUAGE} + - ${FRAMEWORK} + - openshift +maintainers: + - name: ${MAINTAINER_NAME} + email: ${MAINTAINER_EMAIL} diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/NOTES.txt.template b/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/NOTES.txt.template new file mode 100644 index 00000000..154e628d --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/NOTES.txt.template @@ -0,0 +1,32 @@ +Congratulations! Your application {{ include "${APP_NAME}.fullname" . }} has been deployed. + +{{- if .Values.route.enabled }} + +Access your application at: +{{- if .Values.route.host }} + https://{{ .Values.route.host }} +{{- else }} + Run: oc get route {{ include "${APP_NAME}.fullname" . }} -o jsonpath='{.spec.host}' +{{- end }} + +{{- else }} + +Your application is available internally at: + {{ include "${APP_NAME}.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }} + +To expose it externally, create a Route or set route.enabled=true. + +{{- end }} + +Useful commands: + # View pods + oc get pods -l app.kubernetes.io/name={{ include "${APP_NAME}.name" . }} + + # View logs + oc logs -l app.kubernetes.io/name={{ include "${APP_NAME}.name" . }} -f + + # Upgrade release + helm upgrade {{ .Release.Name }} ./{{ .Chart.Name }} -f values.yaml + + # Uninstall release + helm uninstall {{ .Release.Name }} diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/_helpers.tpl.template b/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/_helpers.tpl.template new file mode 100644 index 00000000..15873b10 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/_helpers.tpl.template @@ -0,0 +1,60 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "${APP_NAME}.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "${APP_NAME}.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "${APP_NAME}.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "${APP_NAME}.labels" -}} +helm.sh/chart: {{ include "${APP_NAME}.chart" . }} +{{ include "${APP_NAME}.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "${APP_NAME}.selectorLabels" -}} +app.kubernetes.io/name: {{ include "${APP_NAME}.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "${APP_NAME}.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "${APP_NAME}.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/deployment.yaml.template b/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/deployment.yaml.template new file mode 100644 index 00000000..a6cbd868 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/deployment.yaml.template @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "${APP_NAME}.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "${APP_NAME}.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "${APP_NAME}.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/route.yaml.template b/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/route.yaml.template new file mode 100644 index 00000000..e2bab29a --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/route.yaml.template @@ -0,0 +1,24 @@ +{{- if .Values.route.enabled }} +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + {{- if .Values.route.host }} + host: {{ .Values.route.host }} + {{- end }} + to: + kind: Service + name: {{ include "${APP_NAME}.fullname" . }} + weight: 100 + port: + targetPort: http + {{- with .Values.route.tls }} + tls: + termination: {{ .termination }} + insecureEdgeTerminationPolicy: {{ .insecureEdgeTerminationPolicy }} + {{- end }} + wildcardPolicy: None +{{- end }} diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/service.yaml.template b/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/service.yaml.template new file mode 100644 index 00000000..837bc888 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/templates/service.yaml.template @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "${APP_NAME}.selectorLabels" . | nindent 4 }} diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/values.yaml.template b/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/values.yaml.template new file mode 100644 index 00000000..1cca6017 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/templates/helm/values.yaml.template @@ -0,0 +1,67 @@ +# Default values for ${APP_NAME} +replicaCount: 1 + +image: + repository: ${IMAGE_REPOSITORY} + pullPolicy: IfNotPresent + tag: "${IMAGE_TAG}" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: "" + +podAnnotations: {} +podSecurityContext: {} +securityContext: {} + +service: + type: ClusterIP + port: ${CONTAINER_PORT} + +route: + enabled: true + host: "" + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + +resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + +livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + +readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + +nodeSelector: {} +tolerations: [] +affinity: {} + +env: [] +# - name: MY_VAR +# value: "my-value" diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/templates/imagestream.yaml.template b/evaluation/without_skills/rh-developer__detect-project/environment/templates/imagestream.yaml.template new file mode 100644 index 00000000..46572193 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/templates/imagestream.yaml.template @@ -0,0 +1,13 @@ +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: image + app.kubernetes.io/part-of: ${APP_NAME} +spec: + lookupPolicy: + local: false diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/templates/route.yaml.template b/evaluation/without_skills/rh-developer__detect-project/environment/templates/route.yaml.template new file mode 100644 index 00000000..7c53d2e7 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/templates/route.yaml.template @@ -0,0 +1,21 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: route + app.kubernetes.io/part-of: ${APP_NAME} +spec: + to: + kind: Service + name: ${APP_NAME} + weight: 100 + port: + targetPort: http + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + wildcardPolicy: None diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/templates/service.yaml.template b/evaluation/without_skills/rh-developer__detect-project/environment/templates/service.yaml.template new file mode 100644 index 00000000..7e1cf371 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/templates/service.yaml.template @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: service + app.kubernetes.io/part-of: ${APP_NAME} +spec: + selector: + app: ${APP_NAME} + ports: + - name: http + port: ${CONTAINER_PORT} + targetPort: ${CONTAINER_PORT} + protocol: TCP + type: ClusterIP + sessionAffinity: None diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/templates/systemd/systemd-container-rootful.service b/evaluation/without_skills/rh-developer__detect-project/environment/templates/systemd/systemd-container-rootful.service new file mode 100644 index 00000000..c1e8fe8f --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/templates/systemd/systemd-container-rootful.service @@ -0,0 +1,27 @@ +# Rootful Podman container managed by systemd (system service) +# Location: /etc/systemd/system/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${PORT} - Port number (used for both host and container binding) +# ${IMAGE} - Container image reference + +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run --name ${APP_NAME} \ + -p ${PORT}:${PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/templates/systemd/systemd-container-rootless.service b/evaluation/without_skills/rh-developer__detect-project/environment/templates/systemd/systemd-container-rootless.service new file mode 100644 index 00000000..ca9dc371 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/templates/systemd/systemd-container-rootless.service @@ -0,0 +1,27 @@ +# Rootless Podman container managed by systemd (user service) +# Location: ~/.config/systemd/user/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${PORT} - Port number (used for both host and container binding) +# ${IMAGE} - Container image reference + +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run --name ${APP_NAME} \ + -p ${PORT}:${PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/templates/systemd/systemd-native.service b/evaluation/without_skills/rh-developer__detect-project/environment/templates/systemd/systemd-native.service new file mode 100644 index 00000000..c55cfc07 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/environment/templates/systemd/systemd-native.service @@ -0,0 +1,39 @@ +# Native application managed by systemd (system service) +# Location: /etc/systemd/system/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${SERVICE_USER} - User to run the service as +# ${APP_PATH} - Application install path (e.g., /opt/app-name) +# ${PORT} - Application listen port +# ${START_COMMAND} - Application start command +# +# Start command examples by language: +# Node.js: /usr/bin/node ${APP_PATH}/server.js +# Python: /usr/bin/python3 ${APP_PATH}/app.py +# Java: /usr/bin/java -jar ${APP_PATH}/app.jar +# Go: ${APP_PATH}/binary-name + +[Unit] +Description=${APP_NAME} Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=${APP_PATH} +Environment=PORT=${PORT} +ExecStart=${START_COMMAND} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=${APP_PATH} + +[Install] +WantedBy=multi-user.target diff --git a/evaluation/without_skills/rh-developer__detect-project/instruction.md b/evaluation/without_skills/rh-developer__detect-project/instruction.md new file mode 100644 index 00000000..04695ff5 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/instruction.md @@ -0,0 +1,13 @@ +# Project Detection Task + +You are a Red Hat developer. A colleague has handed you a source repository and asked you to figure out what it is and how to deploy it to OpenShift. + +## Requirements +- Examine the project files to identify the programming language, version, and package manager +- Detect the application framework (e.g., Flask, Express, Spring) and build system +- Based on what you find, recommend a deployment strategy: which builder image or base image to use, what build process to follow, and how the application should be started +- Explain your reasoning for the recommended approach + +Document your analysis and deployment recommendation in `/root/report.md`. + +Use available tools to examine the environment. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-developer__detect-project/solution/solve.sh b/evaluation/without_skills/rh-developer__detect-project/solution/solve.sh new file mode 100644 index 00000000..700e7ad4 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/solution/solve.sh @@ -0,0 +1,37 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Project Detection Report + +## Repository: /root/project + +### Detection Methodology +Scanned for indicator files: requirements.txt, package.json, pom.xml, go.mod, Gemfile. +Found: `requirements.txt` → Python project. + +### Detected Type +- **Language**: Python +- **Indicator**: `requirements.txt` found +- **Framework**: Flask (detected from `from flask import Flask` in app.py) +- **Entry Point**: `app.py` with `app = Flask(__name__)` + +### Helm Chart Search +Searched locations: ./Chart.yaml, ./chart/Chart.yaml, ./charts/*/Chart.yaml, ./helm/Chart.yaml, ./deploy/helm/Chart.yaml +Result: No Helm chart found — S2I or Dockerfile strategy recommended. + +### S2I Python Configuration +- **APP_MODULE**: `app:app` (module `app` from `app.py`, WSGI callable `app`) +- **gunicorn** is present in `requirements.txt` — required for the S2I Python builder to serve via APP_MODULE +- S2I Python builder uses gunicorn as the WSGI server when APP_MODULE is set + +### Recommended Builder Image +`registry.access.redhat.com/ubi9/python-39` (UBI base image) + +### Health Checks +- Add `/health` and `/ready` endpoints for OpenShift liveness/readiness probes + +### Recommended Deployment Strategy +1. **Primary**: S2I with `ubi9/python-39` builder image + - Set `APP_MODULE=app:app` in BuildConfig sourceStrategy.env + - Ensure gunicorn is in requirements.txt +2. **Alternative**: Containerize with Dockerfile using UBI base image +REPORT_EOF diff --git a/evaluation/without_skills/rh-developer__detect-project/task.toml b/evaluation/without_skills/rh-developer__detect-project/task.toml new file mode 100644 index 00000000..78be6504 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__detect-project" +name = "rh-developer Project Detection Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "detect-project", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-developer__detect-project/tests/llm_judge.py b/evaluation/without_skills/rh-developer__detect-project/tests/llm_judge.py new file mode 100644 index 00000000..67b69834 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/tests/llm_judge.py @@ -0,0 +1,102 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "s2i_entry_point_sequence", + "file": "/root/report.md", + "question": "Does the report describe the S2I Python builder's entry point detection order — specifically mentioning that the builder checks for files like app.sh before falling back to app.py, and how app.py being the default entry point affects startup?", + "reference": "A skilled report describes the S2I Python startup sequence (check app.sh first, then application.py, then app.py) and explains that since app.py is found, gunicorn will serve it automatically. An unskilled report mentions app.py as the entry point without describing the detection sequence the builder follows." + }, + { + "id": "app_module_gunicorn_link", + "file": "/root/report.md", + "question": "Does the report explain the connection between gunicorn in requirements.txt and APP_MODULE configuration for the S2I Python builder — specifically that gunicorn is required for APP_MODULE to work?", + "reference": "A skilled report connects gunicorn to APP_MODULE, explaining that the S2I Python builder needs gunicorn in requirements.txt to serve the app specified by APP_MODULE (e.g., app:app). An unskilled report mentions gunicorn as a generic web server without connecting it to S2I builder mechanics." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-developer__detect-project/tests/test.sh b/evaluation/without_skills/rh-developer__detect-project/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-developer__detect-project/tests/test_outputs.py b/evaluation/without_skills/rh-developer__detect-project/tests/test_outputs.py new file mode 100644 index 00000000..3da3a2dc --- /dev/null +++ b/evaluation/without_skills/rh-developer__detect-project/tests/test_outputs.py @@ -0,0 +1,79 @@ +""" +Tests for rh-developer__detect-project per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_project_or_language(self): + content = read_report().lower() + assert any(t in content for t in ["project", "language", "framework", "detect"]), ( + "report should mention project detection" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 100, "report should have substantial content" + + +class TestSkillDependent: + def test_s2i_deployment_recommendation(self): + """Skill teaches S2I as preferred deployment for OpenShift.""" + c = read_report().lower() + assert "s2i" in c or "source-to-image" in c or "source to image" in c, ( + "should recommend S2I as deployment strategy for OpenShift" + ) + + def test_app_module_format(self): + """Skill teaches APP_MODULE format 'module:callable' (e.g., app:app) for + S2I Python. Without skill, agents don't know this configuration.""" + c = read_report().lower() + assert "app_module" in c and any(t in c for t in [ + "app:app", "module:", ":app", "module:callable", "wsgi", + ]), "should specify APP_MODULE format (e.g., app:app) for S2I Python" + + def test_gunicorn_s2i_link(self): + """Skill teaches gunicorn is required IN requirements.txt for the S2I + Python builder to use APP_MODULE. Without skill, agents mention gunicorn + generically without connecting it to S2I builder requirements.""" + c = read_report().lower() + assert "gunicorn" in c and ("s2i" in c or "app_module" in c or "builder" in c), ( + "should connect gunicorn to S2I/APP_MODULE (not just as a generic server)" + ) + + def test_ubi_base_image_recommendation(self): + """Skill teaches UBI as the base image for OpenShift.""" + c = read_report().lower() + assert "ubi" in c or "universal base image" in c, ( + "should recommend UBI base image for OpenShift deployment" + ) + + def test_s2i_entry_point_detection(self): + """Skill teaches the S2I Python entry point detection order + (app.sh → application.py → app.py). Without skill, agents don't + describe the builder's startup sequence.""" + c = read_report().lower() + has_sequence = "app.sh" in c + has_default_entry = ("default" in c or "entry point" in c) and "app.py" in c + has_startup = any(t in c for t in [ + "startup logic", "startup sequence", "s2i startup", + "entry point detection", "entry point order", + ]) + assert has_sequence or has_default_entry or has_startup, ( + "should describe S2I Python entry point detection (app.sh/app.py sequence)" + ) diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/Dockerfile b/evaluation/without_skills/rh-developer__helm-deploy/environment/Dockerfile new file mode 100644 index 00000000..8aaec642 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/Dockerfile @@ -0,0 +1,67 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "helm": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-helm-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/mcp-servers/mock-helm-mcp.py b/evaluation/without_skills/rh-developer__helm-deploy/environment/mcp-servers/mock-helm-mcp.py new file mode 100644 index 00000000..8909ad01 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/mcp-servers/mock-helm-mcp.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +""" +Mock Helm MCP Server for rh-developer helm-deploy benchmark task. + +Simulates Helm CLI operations for OpenShift deployment planning. +""" + +from typing import Optional + +from fastmcp import FastMCP + +mcp = FastMCP("helm") + +# Mock data for existing releases +MOCK_RELEASES = [ + { + "name": "api-service", + "namespace": "api-platform", + "revision": 3, + "updated": "2026-02-15T10:30:00Z", + "status": "deployed", + "chart": "api-service-1.2.0", + "app_version": "1.0.0", + }, + { + "name": "web-frontend", + "namespace": "web-frontend", + "revision": 1, + "updated": "2026-02-14T14:20:00Z", + "status": "deployed", + "chart": "web-frontend-0.1.0", + "app_version": "1.0.0", + }, +] + +MOCK_CHART_METADATA = { + "name": "my-app", + "version": "0.1.0", + "appVersion": "1.0.0", + "description": "OpenShift deployment chart for my-app", + "keywords": ["openshift", "deployment"], + "maintainers": [{"name": "Red Hat", "email": "openshift@redhat.com"}], +} + +MOCK_DEFAULT_VALUES = """replicaCount: 1 + +image: + repository: quay.io/example/my-app + tag: latest + pullPolicy: IfNotPresent + +service: + type: ClusterIP + port: 8080 + +route: + enabled: true + host: "" + +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 256Mi +""" + +MOCK_RENDERED_YAML = """--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-app + labels: + app: my-app +spec: + replicas: 1 + selector: + matchLabels: + app: my-app + template: + metadata: + labels: + app: my-app + spec: + containers: + - name: my-app + image: quay.io/example/my-app:latest + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: my-app +spec: + ports: + - port: 8080 + targetPort: 8080 + selector: + app: my-app +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: my-app +spec: + to: + kind: Service + name: my-app + port: + targetPort: 8080 +""" + + +@mcp.tool +def helm_list(namespace: str) -> dict: + """List installed Helm releases in a namespace. + + Args: + namespace: The Kubernetes/OpenShift namespace to list releases from. + """ + releases = [r for r in MOCK_RELEASES if r["namespace"] == namespace] + return { + "releases": releases, + "count": len(releases), + "namespace": namespace, + } + + +@mcp.tool +def helm_show_chart(chart: str) -> dict: + """Show chart metadata (name, version, description). + + Args: + chart: Path to chart directory or chart name (e.g. ./chart or my-chart). + """ + return { + "chart": chart, + "metadata": MOCK_CHART_METADATA, + } + + +@mcp.tool +def helm_show_values(chart: str) -> dict: + """Show default values for a chart. + + Args: + chart: Path to chart directory or chart name. + """ + return { + "chart": chart, + "values": MOCK_DEFAULT_VALUES, + } + + +@mcp.tool +def helm_template( + release_name: str, + chart: str, + namespace: str, + values: Optional[str] = None, +) -> dict: + """Render chart templates to YAML with given values. + + Args: + release_name: Name for the release. + chart: Path to chart directory. + namespace: Target namespace. + values: Optional YAML string of values to override defaults. + """ + return { + "release_name": release_name, + "chart": chart, + "namespace": namespace, + "rendered": MOCK_RENDERED_YAML, + } + + +@mcp.tool +def helm_install_dry_run( + release_name: str, + chart: str, + namespace: str, + values: Optional[str] = None, +) -> dict: + """Simulate helm install (dry-run) to validate before deploying. + + Args: + release_name: Name for the release. + chart: Path to chart directory. + namespace: Target namespace. + values: Optional YAML string of values to override defaults. + """ + return { + "release_name": release_name, + "chart": chart, + "namespace": namespace, + "dry_run": True, + "status": "would_create", + "resources": ["Deployment/my-app", "Service/my-app", "Route/my-app"], + } + + +@mcp.tool +def helm_status(release_name: str, namespace: str) -> dict: + """Get status of an installed Helm release. + + Args: + release_name: Name of the release. + namespace: The namespace where the release is installed. + """ + release = next( + (r for r in MOCK_RELEASES if r["name"] == release_name and r["namespace"] == namespace), + None, + ) + if release: + return { + "release": release_name, + "namespace": namespace, + "status": release, + } + return { + "release": release_name, + "namespace": namespace, + "error": f"Release '{release_name}' not found in namespace '{namespace}'", + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-developer__helm-deploy/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/buildconfig.yaml.template b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/buildconfig.yaml.template new file mode 100644 index 00000000..b3294eb2 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/buildconfig.yaml.template @@ -0,0 +1,38 @@ +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: build + app.kubernetes.io/part-of: ${APP_NAME} +spec: + source: + type: Git + git: + uri: ${GIT_URL} + ref: ${GIT_BRANCH} + strategy: + type: Source + sourceStrategy: + from: + kind: DockerImage + name: ${BUILDER_IMAGE} + env: [] + output: + to: + kind: ImageStreamTag + name: ${APP_NAME}:latest + triggers: + - type: ConfigChange + - type: ImageChange + runPolicy: Serial + resources: + limits: + memory: "1Gi" + cpu: "1" + requests: + memory: "512Mi" + cpu: "500m" diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/deployment.yaml.template b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/deployment.yaml.template new file mode 100644 index 00000000..eb3b481a --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/deployment.yaml.template @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: application + app.kubernetes.io/part-of: ${APP_NAME} + annotations: + image.openshift.io/triggers: | + [{"from":{"kind":"ImageStreamTag","name":"${APP_NAME}:latest"},"fieldPath":"spec.template.spec.containers[0].image"}] +spec: + replicas: ${REPLICAS} + selector: + matchLabels: + app: ${APP_NAME} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + template: + metadata: + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + spec: + containers: + - name: ${APP_NAME} + image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/${APP_NAME}:latest + ports: + - containerPort: ${CONTAINER_PORT} + protocol: TCP + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + livenessProbe: + httpGet: + path: / + port: ${CONTAINER_PORT} + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: / + port: ${CONTAINER_PORT} + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + env: [] + restartPolicy: Always + terminationGracePeriodSeconds: 30 diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/Chart.yaml.template b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/Chart.yaml.template new file mode 100644 index 00000000..1aa22dd1 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/Chart.yaml.template @@ -0,0 +1,13 @@ +apiVersion: v2 +name: ${APP_NAME} +description: ${APP_DESCRIPTION} +type: application +version: 0.1.0 +appVersion: "${APP_VERSION}" +keywords: + - ${LANGUAGE} + - ${FRAMEWORK} + - openshift +maintainers: + - name: ${MAINTAINER_NAME} + email: ${MAINTAINER_EMAIL} diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/NOTES.txt.template b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/NOTES.txt.template new file mode 100644 index 00000000..154e628d --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/NOTES.txt.template @@ -0,0 +1,32 @@ +Congratulations! Your application {{ include "${APP_NAME}.fullname" . }} has been deployed. + +{{- if .Values.route.enabled }} + +Access your application at: +{{- if .Values.route.host }} + https://{{ .Values.route.host }} +{{- else }} + Run: oc get route {{ include "${APP_NAME}.fullname" . }} -o jsonpath='{.spec.host}' +{{- end }} + +{{- else }} + +Your application is available internally at: + {{ include "${APP_NAME}.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }} + +To expose it externally, create a Route or set route.enabled=true. + +{{- end }} + +Useful commands: + # View pods + oc get pods -l app.kubernetes.io/name={{ include "${APP_NAME}.name" . }} + + # View logs + oc logs -l app.kubernetes.io/name={{ include "${APP_NAME}.name" . }} -f + + # Upgrade release + helm upgrade {{ .Release.Name }} ./{{ .Chart.Name }} -f values.yaml + + # Uninstall release + helm uninstall {{ .Release.Name }} diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/_helpers.tpl.template b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/_helpers.tpl.template new file mode 100644 index 00000000..15873b10 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/_helpers.tpl.template @@ -0,0 +1,60 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "${APP_NAME}.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "${APP_NAME}.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "${APP_NAME}.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "${APP_NAME}.labels" -}} +helm.sh/chart: {{ include "${APP_NAME}.chart" . }} +{{ include "${APP_NAME}.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "${APP_NAME}.selectorLabels" -}} +app.kubernetes.io/name: {{ include "${APP_NAME}.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "${APP_NAME}.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "${APP_NAME}.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/deployment.yaml.template b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/deployment.yaml.template new file mode 100644 index 00000000..a6cbd868 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/deployment.yaml.template @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "${APP_NAME}.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "${APP_NAME}.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "${APP_NAME}.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/route.yaml.template b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/route.yaml.template new file mode 100644 index 00000000..e2bab29a --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/route.yaml.template @@ -0,0 +1,24 @@ +{{- if .Values.route.enabled }} +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + {{- if .Values.route.host }} + host: {{ .Values.route.host }} + {{- end }} + to: + kind: Service + name: {{ include "${APP_NAME}.fullname" . }} + weight: 100 + port: + targetPort: http + {{- with .Values.route.tls }} + tls: + termination: {{ .termination }} + insecureEdgeTerminationPolicy: {{ .insecureEdgeTerminationPolicy }} + {{- end }} + wildcardPolicy: None +{{- end }} diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/service.yaml.template b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/service.yaml.template new file mode 100644 index 00000000..837bc888 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/templates/service.yaml.template @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "${APP_NAME}.selectorLabels" . | nindent 4 }} diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/values.yaml.template b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/values.yaml.template new file mode 100644 index 00000000..1cca6017 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/helm/values.yaml.template @@ -0,0 +1,67 @@ +# Default values for ${APP_NAME} +replicaCount: 1 + +image: + repository: ${IMAGE_REPOSITORY} + pullPolicy: IfNotPresent + tag: "${IMAGE_TAG}" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: "" + +podAnnotations: {} +podSecurityContext: {} +securityContext: {} + +service: + type: ClusterIP + port: ${CONTAINER_PORT} + +route: + enabled: true + host: "" + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + +resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + +livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + +readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + +nodeSelector: {} +tolerations: [] +affinity: {} + +env: [] +# - name: MY_VAR +# value: "my-value" diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/imagestream.yaml.template b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/imagestream.yaml.template new file mode 100644 index 00000000..46572193 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/imagestream.yaml.template @@ -0,0 +1,13 @@ +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: image + app.kubernetes.io/part-of: ${APP_NAME} +spec: + lookupPolicy: + local: false diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/route.yaml.template b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/route.yaml.template new file mode 100644 index 00000000..7c53d2e7 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/route.yaml.template @@ -0,0 +1,21 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: route + app.kubernetes.io/part-of: ${APP_NAME} +spec: + to: + kind: Service + name: ${APP_NAME} + weight: 100 + port: + targetPort: http + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + wildcardPolicy: None diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/service.yaml.template b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/service.yaml.template new file mode 100644 index 00000000..7e1cf371 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/service.yaml.template @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: service + app.kubernetes.io/part-of: ${APP_NAME} +spec: + selector: + app: ${APP_NAME} + ports: + - name: http + port: ${CONTAINER_PORT} + targetPort: ${CONTAINER_PORT} + protocol: TCP + type: ClusterIP + sessionAffinity: None diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-container-rootful.service b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-container-rootful.service new file mode 100644 index 00000000..c1e8fe8f --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-container-rootful.service @@ -0,0 +1,27 @@ +# Rootful Podman container managed by systemd (system service) +# Location: /etc/systemd/system/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${PORT} - Port number (used for both host and container binding) +# ${IMAGE} - Container image reference + +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run --name ${APP_NAME} \ + -p ${PORT}:${PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-container-rootless.service b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-container-rootless.service new file mode 100644 index 00000000..ca9dc371 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-container-rootless.service @@ -0,0 +1,27 @@ +# Rootless Podman container managed by systemd (user service) +# Location: ~/.config/systemd/user/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${PORT} - Port number (used for both host and container binding) +# ${IMAGE} - Container image reference + +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run --name ${APP_NAME} \ + -p ${PORT}:${PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-native.service b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-native.service new file mode 100644 index 00000000..c55cfc07 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/environment/templates/systemd/systemd-native.service @@ -0,0 +1,39 @@ +# Native application managed by systemd (system service) +# Location: /etc/systemd/system/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${SERVICE_USER} - User to run the service as +# ${APP_PATH} - Application install path (e.g., /opt/app-name) +# ${PORT} - Application listen port +# ${START_COMMAND} - Application start command +# +# Start command examples by language: +# Node.js: /usr/bin/node ${APP_PATH}/server.js +# Python: /usr/bin/python3 ${APP_PATH}/app.py +# Java: /usr/bin/java -jar ${APP_PATH}/app.jar +# Go: ${APP_PATH}/binary-name + +[Unit] +Description=${APP_NAME} Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=${APP_PATH} +Environment=PORT=${PORT} +ExecStart=${START_COMMAND} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=${APP_PATH} + +[Install] +WantedBy=multi-user.target diff --git a/evaluation/without_skills/rh-developer__helm-deploy/instruction.md b/evaluation/without_skills/rh-developer__helm-deploy/instruction.md new file mode 100644 index 00000000..5ea35a0f --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/instruction.md @@ -0,0 +1,12 @@ +# Helm Deployment Task + +You are a Red Hat developer. Plan the deployment of an application using Helm charts on OpenShift. + +## Requirements +- Evaluate or create a Helm chart structure +- Configure values for the target environment +- Address OpenShift-specific considerations + +Use MCP tools to examine the cluster. Document your methodology, chart configuration, and deployment plan in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-developer__helm-deploy/solution/solve.sh b/evaluation/without_skills/rh-developer__helm-deploy/solution/solve.sh new file mode 100644 index 00000000..caf0f768 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/solution/solve.sh @@ -0,0 +1,31 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Helm Deployment Plan + +## Chart Location +Searched: ./Chart.yaml, ./chart/Chart.yaml, ./charts/*/Chart.yaml, ./helm/Chart.yaml +Found: `./chart/Chart.yaml` + +## Values Override +```yaml +replicaCount: 2 +image: + repository: image-registry.openshift-image-registry.svc:5000/myproject/myapp + tag: latest +service: + port: 8080 +resources: + limits: + memory: 512Mi +``` + +## Deploy Command +```bash +helm install myapp ./chart/ -f values-override.yaml -n myproject +``` + +## Quick Commands +helm status myapp -n myproject +helm history myapp -n myproject +helm rollback myapp 1 -n myproject +REPORT_EOF diff --git a/evaluation/without_skills/rh-developer__helm-deploy/task.toml b/evaluation/without_skills/rh-developer__helm-deploy/task.toml new file mode 100644 index 00000000..89f35c82 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__helm-deploy" +name = "rh-developer Helm Deployment Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "helm-deploy", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-developer__helm-deploy/tests/llm_judge.py b/evaluation/without_skills/rh-developer__helm-deploy/tests/llm_judge.py new file mode 100644 index 00000000..5632c542 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/tests/llm_judge.py @@ -0,0 +1,102 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "openshift_helm_considerations", + "file": "/root/report.md", + "question": "Does the report address OpenShift-specific Helm concerns like Route vs Ingress and SecurityContextConstraints?", + "reference": "A skilled report addresses that OpenShift uses Routes and has SCC requirements that may affect Helm charts designed for vanilla Kubernetes. An unskilled report treats the chart as platform-agnostic." + }, + { + "id": "buildconfig_in_chart", + "file": "/root/report.md", + "question": "Does the report describe including an OpenShift BuildConfig template as part of the Helm chart structure, so that the chart manages the build pipeline alongside the deployment?", + "reference": "A skilled report includes a BuildConfig YAML template inside the Helm chart (e.g., templates/buildconfig.yaml) for S2I builds. An unskilled report assumes pre-built images and does not integrate build pipelines into the chart." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-developer__helm-deploy/tests/test.sh b/evaluation/without_skills/rh-developer__helm-deploy/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-developer__helm-deploy/tests/test_outputs.py b/evaluation/without_skills/rh-developer__helm-deploy/tests/test_outputs.py new file mode 100644 index 00000000..2f4af59c --- /dev/null +++ b/evaluation/without_skills/rh-developer__helm-deploy/tests/test_outputs.py @@ -0,0 +1,61 @@ +""" +Tests for rh-developer__helm-deploy per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: OpenShift-Helm integration (not generic Helm knowledge). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_helm(self): + content = read_report().lower() + assert "helm" in content, "report should mention Helm" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 100, "report should have substantial content" + + +class TestSkillDependent: + def test_values_customization(self): + """Customizing values before deployment.""" + c = read_report().lower() + assert any(t in c for t in ["values", "override", "set", "customize"]) and any(t in c for t in [ + "install", "upgrade", "deploy" + ]), "should address values customization" + + def test_openshift_considerations(self): + """OpenShift-specific Helm considerations (Route, SCC).""" + c = read_report().lower() + assert any(t in c for t in ["openshift", "route", "scc", "security"]), ( + "should address OpenShift-specific Helm concerns" + ) + + def test_buildconfig_integration(self): + """OpenShift BuildConfig integration in Helm charts for S2I builds. + Without skill, agents use static image references.""" + c = read_report() + assert "BuildConfig" in c or "buildconfig" in c.lower() or "build.openshift.io" in c, ( + "should address OpenShift BuildConfig integration in Helm deployment" + ) + + def test_s2i_in_helm_chart(self): + """OpenShift S2I build integration as part of the Helm chart, + so the chart manages both the build and deploy lifecycle.""" + c = read_report().lower() + assert ("s2i" in c or "source-to-image" in c or "source to image" in c) and ( + "helm" in c or "chart" in c or "template" in c + ), "should integrate S2I builds within the Helm chart structure" diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/Dockerfile b/evaluation/without_skills/rh-developer__recommend-image/environment/Dockerfile new file mode 100644 index 00000000..b01cae66 --- /dev/null +++ b/evaluation/without_skills/rh-developer__recommend-image/environment/Dockerfile @@ -0,0 +1,63 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-developer__recommend-image/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/without_skills/rh-developer__recommend-image/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-developer__recommend-image/instruction.md b/evaluation/without_skills/rh-developer__recommend-image/instruction.md new file mode 100644 index 00000000..7d5e0138 --- /dev/null +++ b/evaluation/without_skills/rh-developer__recommend-image/instruction.md @@ -0,0 +1,13 @@ +# Image Recommendation Task + +You are a Red Hat developer. Your team is choosing a container base image for a production Python application. The image must be secure, supported, and appropriately sized. + +## Requirements +- Evaluate the available base images that support the application's language and runtime +- Compare at least two candidate images on: security posture (CVE exposure, update cadence), image size, vendor support lifecycle, and compatibility with the application's dependencies +- Recommend a specific image with clear justification for why it is the best fit +- Note any trade-offs or caveats with the recommendation (e.g., larger size for better compatibility) + +Document your analysis and recommendation in `/root/report.md`. + +Use available tools to examine the environment. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-developer__recommend-image/solution/solve.sh b/evaluation/without_skills/rh-developer__recommend-image/solution/solve.sh new file mode 100644 index 00000000..ccbb9f6c --- /dev/null +++ b/evaluation/without_skills/rh-developer__recommend-image/solution/solve.sh @@ -0,0 +1,18 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Image Recommendations + +## Use Case Assessment +Production: prefer Minimal/Runtime. Development: prefer Full variant. + +## 1. Python 3.11 Flask API +**Image**: `registry.access.redhat.com/ubi9/python-311` +**Variant**: Full (build tools needed for pip install) +**Verify**: `skopeo inspect docker://registry.access.redhat.com/ubi9/python-311` + +## 2. Java 17 Quarkus (pre-built JAR) +**Image**: `registry.access.redhat.com/ubi9/openjdk-17-runtime` +**Variant**: Runtime (no build tools, smaller attack surface, faster startup) +**Rationale**: Pre-built JAR doesn't need compilation tools. Runtime variant is ~60% smaller. Security: reduced attack surface. +**Verify**: `skopeo inspect docker://registry.access.redhat.com/ubi9/openjdk-17-runtime` +REPORT_EOF diff --git a/evaluation/without_skills/rh-developer__recommend-image/task.toml b/evaluation/without_skills/rh-developer__recommend-image/task.toml new file mode 100644 index 00000000..2888fbf5 --- /dev/null +++ b/evaluation/without_skills/rh-developer__recommend-image/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__recommend-image" +name = "rh-developer Image Recommendation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "recommend-image", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-developer__recommend-image/tests/llm_judge.py b/evaluation/without_skills/rh-developer__recommend-image/tests/llm_judge.py new file mode 100644 index 00000000..1d03045e --- /dev/null +++ b/evaluation/without_skills/rh-developer__recommend-image/tests/llm_judge.py @@ -0,0 +1,102 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "remote_image_inspection", + "file": "/root/report.md", + "question": "Does the report discuss a remote image inspection approach (such as skopeo inspect docker://) for querying image metadata without pulling the full image?", + "reference": "A skilled report discusses using skopeo or a similar remote inspection approach to verify image metadata (size, architecture, build date) without pulling. If skopeo is unavailable, the report should still mention it as the recommended tool or note that static reference data was used instead. An unskilled report only considers pulling images locally with podman/docker." + }, + { + "id": "variant_tradeoffs", + "file": "/root/report.md", + "question": "Does the report compare at least two image variant categories (e.g., Full/build-tools vs Minimal/secure vs Runtime/smallest) with explicit trade-offs for each?", + "reference": "A skilled report distinguishes image variant categories and explains trade-offs (size vs tools vs security). An unskilled report recommends one image without comparing alternatives." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-developer__recommend-image/tests/test.sh b/evaluation/without_skills/rh-developer__recommend-image/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-developer__recommend-image/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-developer__recommend-image/tests/test_outputs.py b/evaluation/without_skills/rh-developer__recommend-image/tests/test_outputs.py new file mode 100644 index 00000000..00dfabc3 --- /dev/null +++ b/evaluation/without_skills/rh-developer__recommend-image/tests/test_outputs.py @@ -0,0 +1,66 @@ +""" +Tests for rh-developer__recommend-image per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_image(self): + content = read_report().lower() + assert "image" in content, "report should mention container images" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 100, "report should have substantial content" + + +class TestSkillDependent: + def test_remote_image_inspection_approach(self): + """Skill teaches skopeo inspect docker:// for remote image inspection. + Without skill, agents only consider local podman/docker pull.""" + c = read_report().lower() + assert any(t in c for t in [ + "skopeo", "remote inspect", "registry inspect", + "docker://", "image metadata", "without pulling" + ]), "should discuss remote image inspection approach (e.g., skopeo, registry API)" + + def test_image_variant_categories(self): + """Skill teaches three variant categories: Full (build tools), Minimal + (smaller/secure), Runtime (smallest, no build tools). Without skill, + agents don't distinguish these categories.""" + c = read_report().lower() + variants = ["full", "minimal", "runtime"] + mentioned = sum(1 for v in variants if v in c) + assert mentioned >= 2, ( + "should compare image variant categories (Full, Minimal, Runtime)" + ) + + def test_security_data_awareness(self): + """Skill teaches Red Hat Security Data API for CVE/security status per image. + Without skill, agents skip security posture evaluation.""" + c = read_report().lower() + assert any(t in c for t in ["security data", "cve", "vulnerability", "security api"]) and any(t in c for t in [ + "image", "scan", "check", "posture", "red hat" + ]), "should address security/CVE posture for image selection" + + def test_ubi_registry_awareness(self): + """Skill teaches UBI images from registry.access.redhat.com.""" + c = read_report().lower() + assert any(t in c for t in ["ubi", "red hat", "registry"]) and any(t in c for t in [ + "python", "node", "java", "image" + ]), "should recommend UBI images from Red Hat registry" diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/Dockerfile b/evaluation/without_skills/rh-developer__rhel-deploy/environment/Dockerfile new file mode 100644 index 00000000..f5320118 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/Dockerfile @@ -0,0 +1,67 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + }, \ + "rhel-host": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-rhel-host-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-developer__rhel-deploy/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/mcp-servers/mock-rhel-host-mcp.py b/evaluation/without_skills/rh-developer__rhel-deploy/environment/mcp-servers/mock-rhel-host-mcp.py new file mode 100644 index 00000000..f10dd2f8 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/mcp-servers/mock-rhel-host-mcp.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +""" +Mock RHEL Host MCP Server for rh-developer rhel-deploy benchmark task. + +Simulates a RHEL 9.3 host with Podman 4.9.4 for container deployment planning. +Scenario: Deploy a Flask app container as a systemd service on port 8080. +""" + +from typing import Optional + +from fastmcp import FastMCP + +mcp = FastMCP("rhel-host") + +# Mock state +MOCK_SYSTEM_INFO = { + "os": "Red Hat Enterprise Linux 9.3 (Plow)", + "kernel": "5.14.0-362.18.1.el9_3.x86_64", + "architecture": "x86_64", + "podman_version": "podman version 4.9.4", + "selinux": "Enforcing", + "firewall": "running", +} + +MOCK_OPEN_PORTS = {8080} # Port 8080 opened for Flask app +MOCK_SERVICES = { + "flask-app": { + "name": "flask-app", + "active": "active", + "state": "running", + "enabled": True, + "description": "Flask application container", + }, + "container-flask-app": { + "name": "container-flask-app", + "active": "active", + "state": "running", + "enabled": True, + "description": "Podman container flask-app.service", + }, +} + +MOCK_PODMAN_PS = """CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +a1b2c3d4e5f6 quay.io/ubi9/python-311:latest flask run 2 hours ago Up 2 hours ago 0.0.0.0:8080->8080/tcp flask-app +""" + +MOCK_PODMAN_INSPECT = """[ + { + "Id": "a1b2c3d4e5f6", + "Name": "flask-app", + "State": { + "Status": "running", + "Running": true + }, + "Config": { + "Image": "quay.io/ubi9/python-311:latest", + "Cmd": ["flask", "run", "--host=0.0.0.0", "--port=8080"] + }, + "HostConfig": { + "PortBindings": { + "8080/tcp": [{"HostPort": "8080"}] + } + } + } +] +""" + + +def _match_command(cmd: str) -> Optional[str]: + """Return a command category for pattern matching.""" + cmd_lower = cmd.strip().lower() + if "podman pull" in cmd_lower: + return "podman_pull" + if "podman run" in cmd_lower: + return "podman_run" + if "podman ps" in cmd_lower or cmd_lower == "podman ps": + return "podman_ps" + if "podman inspect" in cmd_lower: + return "podman_inspect" + if "systemctl enable" in cmd_lower: + return "systemctl_enable" + if "systemctl start" in cmd_lower: + return "systemctl_start" + if "systemctl status" in cmd_lower: + return "systemctl_status" + if "firewall-cmd" in cmd_lower: + return "firewall_cmd" + if "semanage fcontext" in cmd_lower: + return "semanage_fcontext" + if "restorecon" in cmd_lower: + return "restorecon" + return None + + +@mcp.tool +def run_command(command: str) -> dict: + """Simulate running a shell command on a RHEL host. + + Supports common deployment patterns: podman, systemctl, firewall-cmd, semanage. + Returns realistic output for supported commands; error for unknown commands. + + Args: + command: The shell command to execute (e.g. 'podman ps', 'systemctl status flask-app'). + """ + kind = _match_command(command) + if kind == "podman_pull": + return { + "command": command, + "exit_code": 0, + "stdout": "Trying to pull quay.io/ubi9/python-311:latest...\nGetting image source signatures\nCopying blob sha256:...\nCopying config sha256:...\nWriting manifest to image destination\nStoring signatures\n", + "stderr": "", + } + if kind == "podman_run": + return { + "command": command, + "exit_code": 0, + "stdout": "a1b2c3d4e5f6", + "stderr": "", + } + if kind == "podman_ps": + return { + "command": command, + "exit_code": 0, + "stdout": MOCK_PODMAN_PS, + "stderr": "", + } + if kind == "podman_inspect": + return { + "command": command, + "exit_code": 0, + "stdout": MOCK_PODMAN_INSPECT, + "stderr": "", + } + if kind == "systemctl_enable": + return { + "command": command, + "exit_code": 0, + "stdout": "", + "stderr": "", + } + if kind == "systemctl_start": + return { + "command": command, + "exit_code": 0, + "stdout": "", + "stderr": "", + } + if kind == "systemctl_status": + return { + "command": command, + "exit_code": 0, + "stdout": """● flask-app.service - Flask application container + Loaded: loaded (/etc/systemd/system/flask-app.service; enabled) + Active: active (running) since Tue 2026-03-17 10:00:00 UTC; 2h ago + Main PID: 1234 (conmon) + Tasks: 8 + Memory: 128.0M + CGroup: /system.slice/flask-app.service +""", + "stderr": "", + } + if kind == "firewall_cmd": + return { + "command": command, + "exit_code": 0, + "stdout": "success\n", + "stderr": "", + } + if kind == "semanage_fcontext": + return { + "command": command, + "exit_code": 0, + "stdout": "", + "stderr": "", + } + if kind == "restorecon": + return { + "command": command, + "exit_code": 0, + "stdout": "", + "stderr": "", + } + return { + "command": command, + "exit_code": 1, + "stdout": "", + "stderr": f"Error: Unknown or unsupported command. Supported: podman pull/run/ps/inspect, systemctl enable/start/status, firewall-cmd, semanage fcontext, restorecon.", + } + + +@mcp.tool +def get_system_info() -> dict: + """Return RHEL version, architecture, and Podman version for the target host.""" + return MOCK_SYSTEM_INFO.copy() + + +@mcp.tool +def check_service(name: str) -> dict: + """Return systemd service status for a given service name. + + Args: + name: Service name (e.g. 'flask-app', 'container-flask-app'). + """ + svc = MOCK_SERVICES.get(name) + if svc: + return {"service": name, "status": svc, "found": True} + return { + "service": name, + "found": False, + "error": f"Service '{name}' not found. Known services: {list(MOCK_SERVICES.keys())}", + } + + +@mcp.tool +def check_port(port: int) -> dict: + """Return whether a port is open in the firewall. + + Args: + port: Port number to check (e.g. 8080). + """ + open_port = port in MOCK_OPEN_PORTS + return { + "port": port, + "open": open_port, + "message": f"Port {port} is {'open' if open_port else 'closed'} in firewall.", + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/buildconfig.yaml.template b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/buildconfig.yaml.template new file mode 100644 index 00000000..b3294eb2 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/buildconfig.yaml.template @@ -0,0 +1,38 @@ +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: build + app.kubernetes.io/part-of: ${APP_NAME} +spec: + source: + type: Git + git: + uri: ${GIT_URL} + ref: ${GIT_BRANCH} + strategy: + type: Source + sourceStrategy: + from: + kind: DockerImage + name: ${BUILDER_IMAGE} + env: [] + output: + to: + kind: ImageStreamTag + name: ${APP_NAME}:latest + triggers: + - type: ConfigChange + - type: ImageChange + runPolicy: Serial + resources: + limits: + memory: "1Gi" + cpu: "1" + requests: + memory: "512Mi" + cpu: "500m" diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/deployment.yaml.template b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/deployment.yaml.template new file mode 100644 index 00000000..eb3b481a --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/deployment.yaml.template @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: application + app.kubernetes.io/part-of: ${APP_NAME} + annotations: + image.openshift.io/triggers: | + [{"from":{"kind":"ImageStreamTag","name":"${APP_NAME}:latest"},"fieldPath":"spec.template.spec.containers[0].image"}] +spec: + replicas: ${REPLICAS} + selector: + matchLabels: + app: ${APP_NAME} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + template: + metadata: + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + spec: + containers: + - name: ${APP_NAME} + image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/${APP_NAME}:latest + ports: + - containerPort: ${CONTAINER_PORT} + protocol: TCP + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + livenessProbe: + httpGet: + path: / + port: ${CONTAINER_PORT} + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: / + port: ${CONTAINER_PORT} + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + env: [] + restartPolicy: Always + terminationGracePeriodSeconds: 30 diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/Chart.yaml.template b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/Chart.yaml.template new file mode 100644 index 00000000..1aa22dd1 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/Chart.yaml.template @@ -0,0 +1,13 @@ +apiVersion: v2 +name: ${APP_NAME} +description: ${APP_DESCRIPTION} +type: application +version: 0.1.0 +appVersion: "${APP_VERSION}" +keywords: + - ${LANGUAGE} + - ${FRAMEWORK} + - openshift +maintainers: + - name: ${MAINTAINER_NAME} + email: ${MAINTAINER_EMAIL} diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/NOTES.txt.template b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/NOTES.txt.template new file mode 100644 index 00000000..154e628d --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/NOTES.txt.template @@ -0,0 +1,32 @@ +Congratulations! Your application {{ include "${APP_NAME}.fullname" . }} has been deployed. + +{{- if .Values.route.enabled }} + +Access your application at: +{{- if .Values.route.host }} + https://{{ .Values.route.host }} +{{- else }} + Run: oc get route {{ include "${APP_NAME}.fullname" . }} -o jsonpath='{.spec.host}' +{{- end }} + +{{- else }} + +Your application is available internally at: + {{ include "${APP_NAME}.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }} + +To expose it externally, create a Route or set route.enabled=true. + +{{- end }} + +Useful commands: + # View pods + oc get pods -l app.kubernetes.io/name={{ include "${APP_NAME}.name" . }} + + # View logs + oc logs -l app.kubernetes.io/name={{ include "${APP_NAME}.name" . }} -f + + # Upgrade release + helm upgrade {{ .Release.Name }} ./{{ .Chart.Name }} -f values.yaml + + # Uninstall release + helm uninstall {{ .Release.Name }} diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/_helpers.tpl.template b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/_helpers.tpl.template new file mode 100644 index 00000000..15873b10 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/_helpers.tpl.template @@ -0,0 +1,60 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "${APP_NAME}.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "${APP_NAME}.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "${APP_NAME}.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "${APP_NAME}.labels" -}} +helm.sh/chart: {{ include "${APP_NAME}.chart" . }} +{{ include "${APP_NAME}.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "${APP_NAME}.selectorLabels" -}} +app.kubernetes.io/name: {{ include "${APP_NAME}.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "${APP_NAME}.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "${APP_NAME}.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/deployment.yaml.template b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/deployment.yaml.template new file mode 100644 index 00000000..a6cbd868 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/deployment.yaml.template @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "${APP_NAME}.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "${APP_NAME}.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "${APP_NAME}.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/route.yaml.template b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/route.yaml.template new file mode 100644 index 00000000..e2bab29a --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/route.yaml.template @@ -0,0 +1,24 @@ +{{- if .Values.route.enabled }} +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + {{- if .Values.route.host }} + host: {{ .Values.route.host }} + {{- end }} + to: + kind: Service + name: {{ include "${APP_NAME}.fullname" . }} + weight: 100 + port: + targetPort: http + {{- with .Values.route.tls }} + tls: + termination: {{ .termination }} + insecureEdgeTerminationPolicy: {{ .insecureEdgeTerminationPolicy }} + {{- end }} + wildcardPolicy: None +{{- end }} diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/service.yaml.template b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/service.yaml.template new file mode 100644 index 00000000..837bc888 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/templates/service.yaml.template @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "${APP_NAME}.fullname" . }} + labels: + {{- include "${APP_NAME}.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "${APP_NAME}.selectorLabels" . | nindent 4 }} diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/values.yaml.template b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/values.yaml.template new file mode 100644 index 00000000..1cca6017 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/helm/values.yaml.template @@ -0,0 +1,67 @@ +# Default values for ${APP_NAME} +replicaCount: 1 + +image: + repository: ${IMAGE_REPOSITORY} + pullPolicy: IfNotPresent + tag: "${IMAGE_TAG}" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: "" + +podAnnotations: {} +podSecurityContext: {} +securityContext: {} + +service: + type: ClusterIP + port: ${CONTAINER_PORT} + +route: + enabled: true + host: "" + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + +resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + +livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + +readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + +nodeSelector: {} +tolerations: [] +affinity: {} + +env: [] +# - name: MY_VAR +# value: "my-value" diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/imagestream.yaml.template b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/imagestream.yaml.template new file mode 100644 index 00000000..46572193 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/imagestream.yaml.template @@ -0,0 +1,13 @@ +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: image + app.kubernetes.io/part-of: ${APP_NAME} +spec: + lookupPolicy: + local: false diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/route.yaml.template b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/route.yaml.template new file mode 100644 index 00000000..7c53d2e7 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/route.yaml.template @@ -0,0 +1,21 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: route + app.kubernetes.io/part-of: ${APP_NAME} +spec: + to: + kind: Service + name: ${APP_NAME} + weight: 100 + port: + targetPort: http + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + wildcardPolicy: None diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/service.yaml.template b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/service.yaml.template new file mode 100644 index 00000000..7e1cf371 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/service.yaml.template @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: ${APP_NAME} + namespace: ${NAMESPACE} + labels: + app: ${APP_NAME} + app.kubernetes.io/name: ${APP_NAME} + app.kubernetes.io/component: service + app.kubernetes.io/part-of: ${APP_NAME} +spec: + selector: + app: ${APP_NAME} + ports: + - name: http + port: ${CONTAINER_PORT} + targetPort: ${CONTAINER_PORT} + protocol: TCP + type: ClusterIP + sessionAffinity: None diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-container-rootful.service b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-container-rootful.service new file mode 100644 index 00000000..c1e8fe8f --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-container-rootful.service @@ -0,0 +1,27 @@ +# Rootful Podman container managed by systemd (system service) +# Location: /etc/systemd/system/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${PORT} - Port number (used for both host and container binding) +# ${IMAGE} - Container image reference + +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run --name ${APP_NAME} \ + -p ${PORT}:${PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-container-rootless.service b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-container-rootless.service new file mode 100644 index 00000000..ca9dc371 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-container-rootless.service @@ -0,0 +1,27 @@ +# Rootless Podman container managed by systemd (user service) +# Location: ~/.config/systemd/user/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${PORT} - Port number (used for both host and container binding) +# ${IMAGE} - Container image reference + +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run --name ${APP_NAME} \ + -p ${PORT}:${PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-native.service b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-native.service new file mode 100644 index 00000000..c55cfc07 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/environment/templates/systemd/systemd-native.service @@ -0,0 +1,39 @@ +# Native application managed by systemd (system service) +# Location: /etc/systemd/system/${APP_NAME}.service +# +# Variables to replace: +# ${APP_NAME} - Application name +# ${SERVICE_USER} - User to run the service as +# ${APP_PATH} - Application install path (e.g., /opt/app-name) +# ${PORT} - Application listen port +# ${START_COMMAND} - Application start command +# +# Start command examples by language: +# Node.js: /usr/bin/node ${APP_PATH}/server.js +# Python: /usr/bin/python3 ${APP_PATH}/app.py +# Java: /usr/bin/java -jar ${APP_PATH}/app.jar +# Go: ${APP_PATH}/binary-name + +[Unit] +Description=${APP_NAME} Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=${APP_PATH} +Environment=PORT=${PORT} +ExecStart=${START_COMMAND} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=${APP_PATH} + +[Install] +WantedBy=multi-user.target diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/instruction.md b/evaluation/without_skills/rh-developer__rhel-deploy/instruction.md new file mode 100644 index 00000000..b7c3a70e --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/instruction.md @@ -0,0 +1,12 @@ +# RHEL Deployment Task + +You are a Red Hat developer. Plan the deployment of a containerized application on RHEL using Podman and systemd. + +## Requirements +- Configure the container to run as a systemd service +- Address security hardening (SELinux, privilege restrictions) +- Include volume mounts and networking configuration + +Use available tools to examine the environment. Document your methodology, configuration, and deployment plan in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/solution/solve.sh b/evaluation/without_skills/rh-developer__rhel-deploy/solution/solve.sh new file mode 100644 index 00000000..cf537860 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/solution/solve.sh @@ -0,0 +1,43 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# RHEL Deployment Plan + +## Rootless Podman Setup +```bash +sudo useradd -m appuser +sudo loginctl enable-linger appuser +``` + +## Container Run +```bash +podman run -d --name flask-app -p 8080:5000 -v /opt/app-data:/data:z flask-app:latest +``` + +## Systemd Service +Path: `~/.config/systemd/user/flask-app.service` +```ini +[Unit] +Description=Flask App Container +[Service] +ExecStart=/usr/bin/podman run --rm --name flask-app -p 8080:5000 -v /opt/app-data:/data:Z flask-app:latest +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +[Install] +WantedBy=default.target +``` + +## Firewall +```bash +sudo firewall-cmd --permanent --add-port=8080/tcp +sudo firewall-cmd --reload +``` + +## SELinux +```bash +sudo semanage port -a -t http_port_t -p tcp 8080 +sudo semanage fcontext -a -t container_file_t '/opt/app-data(/.*)?' +sudo restorecon -Rv /opt/app-data +``` +REPORT_EOF diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/task.toml b/evaluation/without_skills/rh-developer__rhel-deploy/task.toml new file mode 100644 index 00000000..0ac61da9 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__rhel-deploy" +name = "rh-developer RHEL Deployment Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "rhel-deploy", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/tests/llm_judge.py b/evaluation/without_skills/rh-developer__rhel-deploy/tests/llm_judge.py new file mode 100644 index 00000000..5d7ba0df --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "selinux_volume_labels", + "file": "/root/report.md", + "question": "Does the report explain SELinux volume labels :z (shared, multi-container) and :Z (private) for Podman bind mounts?", + "reference": "A skilled report uses :z or :Z suffixes on volume mounts and explains the difference. An unskilled report skips SELinux mount context." + }, + { + "id": "rootless_systemd", + "file": "/root/report.md", + "question": "Does the report address rootless systemd service configuration (~/.config/systemd/user/) and loginctl enable-linger?", + "reference": "A skilled report shows the rootless systemd path and explains enable-linger for services to survive logout. An unskilled report only shows rootful /etc/systemd/system/ paths." + }, + { + "id": "semanage_fcontext_restorecon", + "file": "/root/report.md", + "question": "Does the report use semanage fcontext + restorecon for setting SELinux file contexts on application directories?", + "reference": "A skilled report uses 'semanage fcontext -a -t bin_t' plus 'restorecon -Rv' for app files. An unskilled report skips file-level SELinux context management." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/tests/test.sh b/evaluation/without_skills/rh-developer__rhel-deploy/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/tests/test_outputs.py b/evaluation/without_skills/rh-developer__rhel-deploy/tests/test_outputs.py new file mode 100644 index 00000000..b4a1c092 --- /dev/null +++ b/evaluation/without_skills/rh-developer__rhel-deploy/tests/test_outputs.py @@ -0,0 +1,98 @@ +""" +Tests for rh-developer__rhel-deploy per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_rhel_or_podman(self): + content = read_report().lower() + assert "rhel" in content or "podman" in content, "report should mention RHEL or Podman" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_selinux_volume_labels(self): + """Skill teaches SELinux volume labels: :z = shared (relabeled for multi-container), + :Z = private. Without skill, agents skip SELinux mount context.""" + c = read_report() + assert ":z" in c or ":Z" in c or "selinux" in c.lower(), ( + "should address SELinux volume labels (:z shared, :Z private)" + ) + + def test_rootless_systemd_path(self): + """Skill teaches rootless systemd service location ~/.config/systemd/user/ + vs /etc/systemd/system/ for rootful. Without skill, agents only know rootful.""" + c = read_report() + assert ".config/systemd/user" in c or "rootless" in c.lower(), ( + "should address rootless systemd path (~/.config/systemd/user/)" + ) + + def test_enable_linger(self): + """Skill teaches loginctl enable-linger required for rootless user services + to survive logout. Without skill, agents miss this requirement.""" + c = read_report().lower() + assert "enable-linger" in c or "loginctl" in c or "linger" in c, ( + "should mention loginctl enable-linger for rootless services" + ) + + def test_semanage_fcontext(self): + """Skill teaches semanage fcontext + restorecon for setting SELinux context + on application files. Without skill, agents skip file context management.""" + c = read_report().lower() + assert ("semanage fcontext" in c or "semanage" in c) and ( + "restorecon" in c or "fcontext" in c + ), "should use semanage fcontext + restorecon for file SELinux context" + + def test_firewall_port(self): + """Skill teaches firewall-cmd for opening application ports.""" + c = read_report().lower() + assert "firewall-cmd" in c or ("firewall" in c and "port" in c), ( + "should address firewall port configuration" + ) + + def test_systemd_hardening_directives(self): + """Docs teach systemd hardening directives: NoNewPrivileges=true, + ProtectSystem=strict, ReadWritePaths. Without docs, agents create basic + unit files without security hardening.""" + c = read_report() + assert any(t in c for t in [ + "NoNewPrivileges", "ProtectSystem", "ReadWritePaths", + "PrivateTmp", "ProtectHome", + ]) or "hardening" in c.lower(), ( + "should include systemd hardening directives (NoNewPrivileges, ProtectSystem)" + ) + + def test_container_security_practices(self): + """Skill teaches defence-in-depth for containers: dropping capabilities, + resource limits, read-only root, security options. Without skill, + agents deploy containers with default security settings.""" + c = read_report().lower() + practices = sum(1 for t in [ + "cap-drop", "cap_drop", "capability", + "--read-only", "read-only root", + "resource limit", "memory", "cpus", + "no-new-privileges", "security-opt", + ] if t in c) + assert practices >= 2, ( + "should address at least 2 container security practices " + "(capability dropping, resource limits, read-only root, security options)" + ) diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/Dockerfile b/evaluation/without_skills/rh-developer__s2i-build/environment/Dockerfile new file mode 100644 index 00000000..b01cae66 --- /dev/null +++ b/evaluation/without_skills/rh-developer__s2i-build/environment/Dockerfile @@ -0,0 +1,63 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-developer__s2i-build/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/without_skills/rh-developer__s2i-build/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-developer__s2i-build/instruction.md b/evaluation/without_skills/rh-developer__s2i-build/instruction.md new file mode 100644 index 00000000..107967b9 --- /dev/null +++ b/evaluation/without_skills/rh-developer__s2i-build/instruction.md @@ -0,0 +1,12 @@ +# S2I Build Configuration Task + +You are a Red Hat developer. Configure a Source-to-Image (S2I) build for a Python web application. + +## Requirements +- Select the appropriate builder image +- Configure the build process and entry point +- Address application startup configuration + +Use MCP tools to examine the cluster. Document your methodology, configuration, and build plan in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-developer__s2i-build/solution/solve.sh b/evaluation/without_skills/rh-developer__s2i-build/solution/solve.sh new file mode 100644 index 00000000..a25acec6 --- /dev/null +++ b/evaluation/without_skills/rh-developer__s2i-build/solution/solve.sh @@ -0,0 +1,60 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# S2I Build Configuration + +## Problem +Python Flask app uses `main.py` as entry point, not the default `app.py`. + +## Solution +1. Create ImageStream for output image +2. Create BuildConfig with `APP_MODULE=main:app` in `sourceStrategy.env` +3. Ensure `gunicorn` is in `requirements.txt` + +### ImageStream +```yaml +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: + name: flask-app + labels: + app: flask-app +spec: + lookupPolicy: + local: false +``` + +### BuildConfig +```yaml +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: flask-app +spec: + source: + type: Git + git: + uri: https://github.com/example/flask-app + strategy: + type: Source + sourceStrategy: + from: + kind: ImageStreamTag + name: python:3.11-ubi9 + namespace: openshift + env: + - name: APP_MODULE + value: "main:app" + output: + to: + kind: ImageStreamTag + name: flask-app:latest +``` + +### S2I Build Phases +- **Assemble**: Install dependencies from requirements.txt (including gunicorn), compile assets. Customizable via `.s2i/bin/assemble`. +- **Run**: Start the application using gunicorn with APP_MODULE. Customizable via `.s2i/bin/run`. + +### Why APP_MODULE is needed +S2I Python startup sequence: app.sh → gunicorn+APP_MODULE → app.py → ERROR +Since entry is main.py (not app.py), gunicorn must be installed and APP_MODULE must point to main:app. +REPORT_EOF diff --git a/evaluation/without_skills/rh-developer__s2i-build/task.toml b/evaluation/without_skills/rh-developer__s2i-build/task.toml new file mode 100644 index 00000000..8dedc143 --- /dev/null +++ b/evaluation/without_skills/rh-developer__s2i-build/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__s2i-build" +name = "rh-developer S2I Build Configuration Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "s2i-build", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-developer__s2i-build/tests/llm_judge.py b/evaluation/without_skills/rh-developer__s2i-build/tests/llm_judge.py new file mode 100644 index 00000000..5fbc562a --- /dev/null +++ b/evaluation/without_skills/rh-developer__s2i-build/tests/llm_judge.py @@ -0,0 +1,114 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "app_module_in_buildconfig", + "file": "/root/report.md", + "question": "Does the report specify that APP_MODULE should be set in the BuildConfig's sourceStrategy.env section (not as a generic environment variable), using the module:callable format (e.g., app:app or main:app)?", + "reference": "A skilled report places APP_MODULE in sourceStrategy.env of the BuildConfig YAML, using the module:callable format. An unskilled report mentions APP_MODULE generically without specifying its placement in sourceStrategy.env." + }, + { + "id": "s2i_build_phases", + "file": "/root/report.md", + "question": "Does the report explain S2I build phases (assemble for dependency installation and compilation, run for application startup) and how they can be customized via .s2i/bin/ scripts?", + "reference": "A skilled report explains the assemble and run phases and mentions .s2i/bin/assemble or .s2i/bin/run for customization. An unskilled report treats S2I as a monolithic process." + }, + { + "id": "gunicorn_dependency", + "file": "/root/report.md", + "question": "Does the report explicitly state that gunicorn must be in requirements.txt specifically BECAUSE the S2I Python builder uses gunicorn to serve the application specified by APP_MODULE?", + "reference": "A skilled report identifies gunicorn as a required dependency for Python S2I with APP_MODULE. An unskilled report doesn't link gunicorn to the entry point mechanism." + }, + { + "id": "imagestream_as_separate_resource", + "file": "/root/report.md", + "question": "Does the report include a standalone ImageStream YAML manifest (with apiVersion: image.openshift.io/v1 and kind: ImageStream) as a separate resource definition, rather than only referencing ImageStreamTag within the BuildConfig output section?", + "reference": "A skilled report defines the ImageStream as its own YAML resource with apiVersion: image.openshift.io/v1, kind: ImageStream, and lookupPolicy configuration, created as a prerequisite before the BuildConfig. An unskilled report only references ImageStreamTag as an output target in the BuildConfig but does not show the ImageStream resource definition." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-developer__s2i-build/tests/test.sh b/evaluation/without_skills/rh-developer__s2i-build/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-developer__s2i-build/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-developer__s2i-build/tests/test_outputs.py b/evaluation/without_skills/rh-developer__s2i-build/tests/test_outputs.py new file mode 100644 index 00000000..ec2af10d --- /dev/null +++ b/evaluation/without_skills/rh-developer__s2i-build/tests/test_outputs.py @@ -0,0 +1,84 @@ +""" +Tests for rh-developer__s2i-build per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_s2i(self): + content = read_report().lower() + assert "s2i" in content or "source-to-image" in content, ( + "report should mention S2I" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_app_module_format(self): + """Skill teaches APP_MODULE env var format module:app (e.g. main:app) for + non-default Python entry points. Without skill, agents don't know this format.""" + c = read_report() + assert "APP_MODULE" in c or "app_module" in c.lower(), ( + "should reference APP_MODULE env var for Python S2I entry point" + ) + + def test_module_colon_app_syntax(self): + """Skill teaches the module:app syntax (e.g., main:app, wsgi:application). + Without skill, agents don't know the colon-separated format.""" + c = read_report() + assert any(t in c for t in ["main:app", "wsgi:app", "module:app", ":app", ":application"]) or ( + "APP_MODULE" in c and ":" in c + ), "should show module:app format for APP_MODULE" + + def test_s2i_build_phases(self): + """Skill teaches S2I build phases: assemble (install deps, compile) and + run (start app). Without skill, agents treat S2I as a black box.""" + c = read_report().lower() + assert ("assemble" in c and ("run" in c or "start" in c)) or ( + "build phase" in c or "build step" in c or "build process" in c + ), "should explain S2I build phases (assemble and run)" + + def test_buildconfig_imagestream(self): + """Skill teaches creating ImageStream + BuildConfig with source/builder/output.""" + c = read_report().lower() + assert any(t in c for t in ["buildconfig", "imagestream", "build config"]) and any(t in c for t in [ + "source", "builder", "output" + ]), "should define BuildConfig/ImageStream" + + def test_gunicorn_requirement(self): + """Skill teaches gunicorn must be in requirements.txt for APP_MODULE.""" + c = read_report().lower() + assert "gunicorn" in c and any(t in c for t in [ + "requirements", "pip", "install", "wsgi", "app_module" + ]), "should address gunicorn requirement for S2I Python" + + def test_standalone_imagestream_yaml(self): + """Skill teaches creating ImageStream as a separate resource with + image.openshift.io/v1 API group and lookupPolicy. Without skill, + agents reference ImageStreamTag in BuildConfig but don't define + the ImageStream resource itself.""" + c = read_report() + has_is_api = "image.openshift.io" in c + has_lookup = "lookupPolicy" in c + assert has_is_api or has_lookup, ( + "should define ImageStream resource with image.openshift.io API" + ) + diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/Dockerfile b/evaluation/without_skills/rh-developer__validate-environment/environment/Dockerfile new file mode 100644 index 00000000..b01cae66 --- /dev/null +++ b/evaluation/without_skills/rh-developer__validate-environment/environment/Dockerfile @@ -0,0 +1,63 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-openshift-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/builder-images.md new file mode 100644 index 00000000..6561c5ca --- /dev/null +++ b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/builder-images.md @@ -0,0 +1,308 @@ +--- +title: S2I Builder Image Reference +category: containers +sources: + - title: Red Hat Container Catalog + url: https://catalog.redhat.com/software/containers/search + sections: UBI images, S2I builders + date_accessed: 2026-02-08 + - title: OpenShift Source-to-Image (S2I) + url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html + sections: S2I builder images, Language detection + date_accessed: 2026-02-08 + - title: Red Hat Universal Base Images + url: https://developers.redhat.com/products/rhel/ubi + sections: UBI9 images, Language runtimes + date_accessed: 2026-02-08 +--- + +# S2I Builder Image Reference + +Use this reference when recommending S2I builder images to users. + +> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. + +For use-case-aware image selection, use the `/recommend-image` skill. + +--- + +## Dynamic Lookup and Verification + +**This reference may be outdated.** Always verify image availability before recommending. + +### Verify with Skopeo (Recommended) + +```bash +# Check if an image exists and get metadata +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Get specific fields +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# List all available tags +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 +``` + +**If skopeo is not installed**, prompt the user: +``` +Install with: sudo dnf install skopeo (Fedora/RHEL) + sudo apt install skopeo (Ubuntu/Debian) + brew install skopeo (macOS) +``` + +### Check Security Status (Red Hat Security Data API) + +Query CVE information (no authentication required): + +```bash +# Check for critical CVEs affecting UBI9 +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' + +# Get CVE details +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' +``` + +### Verify with Red Hat Catalog API (Alternative) + +```bash +# Search for available Node.js images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' + +# Search for available Python images +curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' +``` + +--- + +## Project Detection and Version Mapping + +### Extract Version from Project Files + +Before recommending an image, check the project's version requirements: + +| Project File | How to Extract Version | +|--------------|------------------------| +| `package.json` | `.engines.node` field | +| `requirements.txt` | `python_requires` or comments | +| `pyproject.toml` | `[project].requires-python` | +| `pom.xml` | `` or `` | +| `go.mod` | `go` directive (e.g., `go 1.21`) | +| `*.csproj` | `` (e.g., `net8.0`) | + +### Detect Language from Files + +| Indicator File(s) | Language | Framework | Version Source | +|-------------------|----------|-----------|----------------| +| `package.json` | Node.js | - | `.engines.node` | +| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | +| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | +| `pom.xml` | Java | Maven | `` or `` | +| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | +| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | +| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | +| `requirements.txt` | Python | - | `python_requires` or shebang | +| `Pipfile` | Python | Pipenv | `[requires].python_version` | +| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | +| `go.mod` | Go | - | `go` directive line | +| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | +| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | +| `composer.json` | PHP | - | `require.php` field | +| `Cargo.toml` | Rust | - | Custom (no official S2I) | + +### Map Version to Image + +**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) + +| Language | Version Mapping | Image Pattern | +|----------|-----------------|---------------| +| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | +| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | +| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | +| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | +| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | +| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | +| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | + +### Verify and Fallback + +1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` +2. **If version not found**: Use nearest available LTS version +3. **If no version in project**: Use current LTS (check catalog API) + +--- + +## Red Hat UBI-based Images + +### Node.js + +| Version | Full Image | Minimal Image | Use Case | +|---------|------------|---------------|----------| +| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | +| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | +| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | + +**Choose minimal for:** Production, security-focused, smaller image size +**Choose full for:** Development, native module compilation + +### Python + +| Version | Image | Notes | +|---------|-------|-------| +| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | +| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | +| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | + +### Java / OpenJDK + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | +| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | +| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | + +**Choose runtime for:** Production with pre-built JARs, smallest footprint +**Choose build for:** S2I builds, Maven/Gradle compilation needed + +### Go + +| Version | Image | Notes | +|---------|-------|-------| +| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | +| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | + +### Ruby + +| Version | Image | Notes | +|---------|-------|-------| +| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | +| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | + +### .NET + +| Version | Build Image | Runtime Image | Notes | +|---------|-------------|---------------|-------| +| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | +| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | +| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | + +**Choose runtime for:** Production with pre-built assemblies +**Choose build for:** S2I builds, dotnet build/publish needed + +### PHP + +| Version | Image | Notes | +|---------|-------|-------| +| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | +| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | + +### Perl + +| Version | Image | Notes | +|---------|-------|-------| +| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | + +--- + +## Image Variants and Use-Case Selection + +### Quick Use-Case Matrix + +| Use Case | Variant | Priority | Example | +|----------|---------|----------|---------| +| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | +| Development | Full | Tools, Debug | `nodejs-20` | +| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | +| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | + +### Image Variants + +| Variant | Description | Has Build Tools | Size | +|---------|-------------|-----------------|------| +| Full | Complete development environment | Yes | Largest | +| Minimal | Essential packages only | Limited | Medium | +| Runtime | Runtime only, no build tools | No | Smallest | + +**Availability by language:** + +| Language | Full | Minimal | Runtime | +|----------|------|---------|---------| +| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | +| Python | `python-{ver}` | - | - | +| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | +| Go | `go-toolset:{ver}` | - | (produces static binary) | +| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | +| Ruby | `ruby-{ver}` | - | - | +| PHP | `php-{ver}` | - | - | + +### When to Recommend Each Variant + +**Full variant:** +- User needs to compile native extensions +- Development/debugging environment +- CI/CD build stages + +**Minimal variant:** +- Production deployments +- Security-focused environments +- When size matters but some tools needed + +**Runtime variant:** +- Pre-compiled applications (JARs, .NET assemblies) +- Maximum security posture +- Smallest possible footprint + +--- + +## OpenShift Built-in ImageStreams + +These are often pre-configured in OpenShift clusters under the `openshift` namespace: + +| ImageStream | Usage | +|-------------|-------| +| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | +| `python:3.11-ubi9` | Python 3.11 on UBI 9 | +| `openjdk-17-ubi8` | Java 17 on UBI 8 | +| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | +| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | + +When using OpenShift ImageStreams, reference them as: +```yaml +from: + kind: ImageStreamTag + namespace: openshift + name: nodejs:20-ubi9 +``` + +--- + +## Framework-Specific Recommendations + +### Quarkus (Java) +- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` + +### Spring Boot (Java) +- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` +- Ensure `spring-boot-maven-plugin` is configured for packaging + +### Next.js / React (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-20` +- Ensure build outputs to `build/` or `.next/` + +### Django / Flask (Python) +- Use: `registry.access.redhat.com/ubi9/python-311` +- Ensure `requirements.txt` or `Pipfile` exists at root + +### Express.js (Node.js) +- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher +- Ensure `npm start` script is defined in `package.json` + +--- + +## Python S2I Entry Point Requirements + +**Quick reference:** +- Default entry point: `app.py` (works without configuration) +- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable +- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/debugging-patterns.md new file mode 100644 index 00000000..2863d559 --- /dev/null +++ b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/debugging-patterns.md @@ -0,0 +1,478 @@ +--- +title: Debugging Patterns +category: references +sources: + - title: Kubernetes Debugging Pods + url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ + sections: Debugging Pods, Common Errors + date_accessed: 2026-02-16 + - title: OpenShift Troubleshooting Guide + url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html + sections: Pod issues, Build issues + date_accessed: 2026-02-16 + - title: OpenShift Pipelines Troubleshooting + url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html + sections: Troubleshooting, PipelineRun status, TaskRun status + date_accessed: 2026-02-25 + - title: Podman Troubleshooting + url: https://github.com/containers/podman/blob/main/troubleshooting.md + sections: Common Issues + date_accessed: 2026-02-16 +--- + +# Debugging Patterns + +This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. + +## Exit Code Reference + +### Container/Process Exit Codes + +| Exit Code | Signal | Meaning | Common Cause | +|-----------|--------|---------|--------------| +| 0 | - | Success | Normal termination | +| 1 | - | General error | Application error, unhandled exception | +| 2 | - | Misuse of shell | Invalid arguments, syntax error | +| 126 | - | Permission denied | Cannot execute command | +| 127 | - | Command not found | Binary/script missing in PATH | +| 128 | - | Invalid exit argument | Exit called with non-integer | +| 128+N | Signal N | Killed by signal | See signal table below | +| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | +| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | +| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | + +### Signal Reference (128+N) + +| Signal | Number | Exit Code | Typical Cause | +|--------|--------|-----------|---------------| +| SIGHUP | 1 | 129 | Terminal closed | +| SIGINT | 2 | 130 | Ctrl+C | +| SIGQUIT | 3 | 131 | Ctrl+\ | +| SIGKILL | 9 | 137 | OOM, forced termination | +| SIGSEGV | 11 | 139 | Segmentation fault | +| SIGTERM | 15 | 143 | Graceful stop request | + +## Pod Failure Patterns + +### CrashLoopBackOff + +**Symptom:** Pod repeatedly crashes and restarts + +**Diagnosis Flow:** +``` +CrashLoopBackOff +├─ Check exit code +│ ├─ 0 → Application exits normally (missing loop/server?) +│ ├─ 1 → Application error (check logs) +│ ├─ 127 → Command not found (check entrypoint) +│ └─ 137 → OOM killed (check memory limits) +├─ Check logs (current + previous) +│ ├─ Import errors → Missing dependencies +│ ├─ Connection errors → External service down +│ └─ Config errors → Missing env vars/secrets +└─ Check events + └─ FailedMount → Missing secrets/configmaps +``` + +**Common Causes:** +1. Application crashes on startup (dependency errors) +2. Memory limit too low (OOMKilled) +3. Missing environment variables or secrets +4. Database/service connection failures +5. Health probe failing immediately + +### ImagePullBackOff + +**Symptom:** Cannot pull container image + +**Diagnosis Flow:** +``` +ImagePullBackOff +├─ Check event message +│ ├─ "unauthorized" → Registry authentication +│ │ └─ Check imagePullSecrets +│ ├─ "not found" → Wrong image name/tag +│ │ └─ Verify image exists in registry +│ ├─ "timeout" → Network/registry issue +│ │ └─ Check cluster network egress +│ └─ "manifest unknown" → Tag doesn't exist +│ └─ Verify tag in registry +└─ Check image reference + ├─ Missing registry prefix? + ├─ Typo in image name? + └─ Tag exists? +``` + +**Common Causes:** +1. Private registry without imagePullSecret +2. Image tag doesn't exist +3. Registry URL typo +4. Network policy blocking egress +5. Registry rate limiting + +### Pending Pod + +**Symptom:** Pod stuck in Pending state + +**Diagnosis Flow:** +``` +Pending +├─ Check events +│ ├─ "FailedScheduling" +│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests +│ │ ├─ "node selector" → No matching nodes +│ │ ├─ "taints" → Need tolerations +│ │ └─ "PVC not bound" → Storage issue +│ └─ No events → Check resourceQuota +└─ Check node status + └─ All nodes NotReady? → Node issue +``` + +**Common Causes:** +1. Insufficient cluster resources +2. Node selector doesn't match any nodes +3. PersistentVolumeClaim not bound +4. Resource quota exceeded +5. Affinity/anti-affinity rules too strict + +### OOMKilled + +**Symptom:** Container terminated with exit code 137 + +**Diagnosis Flow:** +``` +OOMKilled (exit 137) +├─ Check container state +│ └─ OOMKilled: true → Memory exhaustion confirmed +├─ Compare memory usage vs limit +│ ├─ Limit too low → Increase memory limit +│ └─ Memory leak → Profile application +└─ Check for: + ├─ Java → Heap size (-Xmx) exceeds limit + ├─ Node.js → --max-old-space-size too high + └─ Python → Large data structures in memory +``` + +**Common Causes:** +1. Memory limit set too low for application +2. Memory leak in application +3. Java heap size exceeds container limit +4. Processing large files/datasets in memory + +## Build Failure Patterns + +### S2I Build Phases + +| Phase | What Happens | Common Failures | +|-------|--------------|-----------------| +| **fetch-source** | Clone git repository | Auth failure, repo not found | +| **pull-builder** | Pull S2I builder image | Image not found, auth | +| **assemble** | Run S2I assemble script | Dependency install, build errors | +| **commit** | Create image layer | Disk space | +| **push** | Push to internal registry | Auth, quota | + +### Assemble Phase Failures + +**Node.js:** +``` +npm ERR! 404 Not Found +└─ Package doesn't exist in registry + → Check package.json for typos + +npm ERR! code ERESOLVE +└─ Dependency conflict + → Run npm install --legacy-peer-deps + +npm ERR! code ENOENT +└─ File not found + → Check paths in package.json +``` + +**Python:** +``` +ERROR: Could not find a version that satisfies the requirement +└─ Package not found + → Check requirements.txt spelling + +ModuleNotFoundError: No module named 'X' +└─ APP_MODULE misconfigured + → See docs/python-s2i-entrypoints.md + +gunicorn: command not found +└─ gunicorn not in requirements + → Add gunicorn to requirements.txt +``` + +**Java:** +``` +[ERROR] Failed to execute goal +└─ Maven/Gradle build failure + → Check pom.xml or build.gradle + +java.lang.OutOfMemoryError: Java heap space +└─ Build needs more memory + → Add MAVEN_OPTS=-Xmx512m +``` + +## Pipeline/Tekton Failure Patterns + +### PipelineRun Failure Decision Tree + +``` +PipelineRun Failed +├─ Check PipelineRun status conditions +│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline +│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace +│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation +│ └─ "Failed" → Check which TaskRun failed (see below) +├─ Check failed TaskRun +│ ├─ Step failure (non-zero exit) +│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) +│ │ ├─ build step → Compilation/dependency error +│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) +│ │ └─ test step → Test failures +│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) +│ ├─ Workspace issue → PVC not bound or permission denied +│ └─ Step image pull failure → ImagePullBackOff on step container +└─ Pipeline stuck (Running too long) + ├─ TaskRun pending → Pod can't be scheduled + ├─ Step running indefinitely → Check logs for hang/deadlock + └─ Custom task waiting → Check custom task controller +``` + +### TaskRun Failure Analysis + +``` +TaskRun Failed +├─ Pod not created → Check ServiceAccount exists, resource quotas +├─ Pod pending → Scheduling issue (see Pod Failure Patterns) +├─ Pod terminated → Check step statuses +│ ├─ Exit 1 → Script/application error (check step logs) +│ ├─ Exit 125-127 → Entrypoint/command issue in step image +│ └─ Exit 137 → OOM killed (increase step resources) +└─ Workspace binding failure + ├─ PVC not found → Create PVC or fix workspace binding + ├─ RWO blocks parallel tasks → Use RWX or separate workspaces + └─ Permission denied → Check fsGroup, runAsUser in pod security context +``` + +### Common Tekton Error Messages + +| Error Message | Fix | +|--------------|-----| +| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | +| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | +| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | +| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | +| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | +| `missing required parameter "X"` | Add parameter value to PipelineRun spec | +| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | +| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | + +## Network Troubleshooting + +### Service Has No Endpoints + +**Diagnosis Flow:** +``` +No endpoints +├─ Check service selector +│ └─ Compare with pod labels +│ ├─ Labels don't match → Fix selector or pod labels +│ └─ Labels match → Check pod readiness +├─ Check pod status +│ ├─ Pods not running → Debug pods first +│ └─ Pods running but not ready → Check readiness probe +└─ Check readiness probe + ├─ HTTP probe failing → Application not listening + └─ TCP probe failing → Wrong port +``` + +### Route Returning 503 + +**Diagnosis Flow:** +``` +503 Service Unavailable +├─ Check endpoints +│ └─ No endpoints → Pods not ready +├─ Check backend pods +│ ├─ All pods failing readiness → Application issue +│ └─ Some pods ready → Load balancer issue +└─ Check route configuration + └─ Wrong service or port → Fix route spec +``` + +### Connection Refused + +**Diagnosis Flow:** +``` +Connection refused +├─ Is service created? → oc get svc +├─ Does service have endpoints? → oc get endpoints +├─ Is pod running? → oc get pods +├─ Is application listening? → Check container port +└─ Is port correct? → Compare service port vs container port +``` + +## RHEL System Patterns + +### systemd Service Failures + +| Exit Code | Meaning | Common Fix | +|-----------|---------|------------| +| 1 | General error | Check application logs | +| 126 | Permission | Check ExecStart permissions | +| 127 | Not found | Check binary path in ExecStart | +| 203 | EXEC | Wrong architecture or format | +| 217 | USER | Service user doesn't exist | + +### SELinux Denial Patterns + +| Denial Type | Example | Typical Fix | +|-------------|---------|-------------| +| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | +| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | +| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | +| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | + +See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. + +## Troubleshooting Decision Tree + +### Application Not Accessible + +``` +Cannot access application +├─ Internal (from cluster)? +│ ├─ Yes, works internally → Route/Ingress issue +│ │ ├─ Check route admitted +│ │ ├─ Check route host/path +│ │ └─ Check TLS configuration +│ └─ No, fails internally too → Service/Pod issue +│ ├─ Check service endpoints +│ ├─ Check pod status +│ └─ Check pod readiness +└─ Neither works? + └─ Debug pod first (/debug-pod) +``` + +### Build Keeps Failing + +``` +Build failures +├─ Which phase? +│ ├─ fetch-source → Git access issue +│ │ ├─ Check source secret +│ │ └─ Verify git URL +│ ├─ pull-builder → Builder image issue +│ │ ├─ Check image reference +│ │ └─ Import ImageStream +│ ├─ assemble → Build script issue +│ │ ├─ Check dependencies +│ │ └─ Check language-specific config +│ └─ push → Registry issue +│ └─ Check push secret +└─ Same failure pattern? + └─ Compare with last successful build +``` + +### Pipeline Keeps Failing + +``` +Pipeline failures +├─ Same task always fails? +│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision +│ ├─ build step → Check source code, Containerfile path, build context +│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL +├─ Different tasks fail? +│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas +│ └─ Workspace contention → Use RWX PVC or separate workspaces +├─ Pipeline hangs? +│ ├─ TaskRun pending → Pod can't be scheduled +│ └─ Step running indefinitely → Check step logs +└─ Pipeline never triggers? + ├─ EventListener pod not running → Check EL deployment/logs + ├─ Webhook misconfigured → Verify webhook URL and secret + └─ TriggerBinding wrong → Check CEL expression param extraction +``` + +## Quick Reference Commands + +### OpenShift Debugging + +```bash +# Pod status and events +oc describe pod [pod-name] + +# Pod logs (current) +oc logs [pod-name] + +# Pod logs (previous container) +oc logs [pod-name] --previous + +# All events in namespace +oc get events --sort-by='.lastTimestamp' + +# Check endpoints +oc get endpoints [service-name] + +# Build logs +oc logs build/[build-name] +``` + +### Pipeline/Tekton Debugging + +```bash +# List PipelineRuns (oldest first) +oc get pipelinerun --sort-by='.metadata.creationTimestamp' + +# Get PipelineRun details +oc get pipelinerun [name] -o yaml + +# List TaskRuns for a PipelineRun +oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] + +# Get TaskRun pod logs for a specific step +oc logs [taskrun-name]-pod -c step-[step-name] + +# Get events for pipeline resources +oc get events --field-selector involvedObject.kind=PipelineRun + +# Describe EventListener +oc get eventlistener [name] -o yaml +``` + +### RHEL Debugging + +```bash +# Service status +systemctl status [service] + +# Journal logs +journalctl -u [service] -n 100 + +# SELinux denials +ausearch -m AVC -ts recent + +# Firewall rules +firewall-cmd --list-all + +# SELinux context +ls -lZ [path] +``` + +### Container Debugging + +```bash +# List all containers +podman ps -a + +# Container inspect +podman inspect [container] + +# Container logs +podman logs [container] + +# Run interactively for debugging +podman run -it --entrypoint /bin/sh [image] +``` diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/dynamic-validation.md new file mode 100644 index 00000000..a027f0ce --- /dev/null +++ b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/dynamic-validation.md @@ -0,0 +1,259 @@ +--- +title: Dynamic Image Validation Reference +category: containers +sources: + - title: Skopeo Documentation + url: https://github.com/containers/skopeo + sections: Inspecting images, Copying images + date_accessed: 2026-02-08 + - title: Red Hat Security Data API + url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 + sections: CVE queries, Product filtering + date_accessed: 2026-02-08 +--- + +# Dynamic Image Validation Reference + +This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. + +## Skopeo Commands + +Skopeo inspects container images without downloading them, providing real-time metadata. + +### Prerequisites + +**Check if skopeo is installed:** +```bash +which skopeo +# or +skopeo --version +``` + +**Installation:** +| OS | Command | +|----|---------| +| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | +| Ubuntu/Debian | `sudo apt install skopeo` | +| macOS (Homebrew) | `brew install skopeo` | + +### Basic Inspection + +```bash +# Inspect an image (full JSON output) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# The docker:// transport is OCI-standard and works with all registries +# (Docker Hub, Red Hat, Quay, Podman registries, etc.) +``` + +### Extracting Specific Fields + +```bash +# Get creation date +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' + +# Get architecture +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' + +# Get all labels +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' + +# Get specific label (e.g., version) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' + +# Get layer count +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' +``` + +### Listing Available Tags + +```bash +# List all tags for an image +skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Output includes all available versions/tags +``` + +### Image Transport Options + +```bash +# Remote registry (most common) +skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 + +# Local Podman storage +skopeo inspect containers-storage:localhost/myimage:latest + +# OCI layout directory +skopeo inspect oci:/path/to/oci-layout:tag + +# Docker archive +skopeo inspect docker-archive:/path/to/image.tar +``` + +### Useful Metadata Fields + +| Field | Description | Use Case | +|-------|-------------|----------| +| `Created` | Image build timestamp | Freshness indicator | +| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | +| `Os` | Operating system | Should be "linux" for UBI | +| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | +| `Layers` | Layer digests | Calculate approximate size | +| `Digest` | Immutable image hash | Pin exact version | + +### Error Handling + +**Image not found:** +``` +Error: Error reading manifest: ... 404 Not Found +``` +→ Image does not exist at specified tag + +**Authentication required:** +``` +Error: Error reading manifest: unauthorized +``` +→ Private registry, need `skopeo login` first + +**Network error:** +``` +Error: Error initializing source: pinging container registry +``` +→ Network connectivity issue + +--- + +## Red Hat Security Data API + +The Security Data API provides CVE information without authentication. + +### Base Endpoint + +``` +https://access.redhat.com/hydra/rest/securitydata/ +``` + +### Query CVEs + +```bash +# Get all CVEs for UBI 9 (may return many results) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" + +# Filter by severity (critical, important, moderate, low) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" + +# Filter by date (CVEs after a specific date) +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" + +# Count critical CVEs +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' +``` + +### Product Names for Queries + +| Image Base | Product Name (URL-encoded) | +|------------|---------------------------| +| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | +| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | +| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | +| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | + +### Response Fields + +Each CVE object contains: + +| Field | Description | +|-------|-------------| +| `CVE` | CVE identifier (e.g., CVE-2024-1234) | +| `severity` | critical, important, moderate, low | +| `public_date` | When CVE was disclosed | +| `advisories` | Related Red Hat advisories | +| `bugzilla` | Bugzilla tracking URL | +| `affected_packages` | Packages affected by CVE | + +### Parsing Examples + +```bash +# Get CVE IDs and severities +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' + +# Get most recent CVE date +curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' + +# Check if any critical CVEs exist +CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') +if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "Warning: $CRITICAL_COUNT critical CVEs found" +fi +``` + +--- + +## Validation Workflow + +### Complete Validation Sequence + +``` +1. Check if skopeo is installed + ├── Yes → Continue to step 2 + └── No → Prompt user to install, offer to continue with static data + +2. For each candidate image: + a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] + b. If fails → Remove from candidates, try next + c. If succeeds → Extract: Created, Architecture, Labels + +3. Query Security Data API for UBI base: + a. Run: curl CVE query for critical severity + b. Parse count of critical CVEs + c. If count > 0 → Add warning to recommendation + +4. Compile results: + - Image metadata (from skopeo) + - Security status (from API) + - Static scoring data (from reference tables) + +5. Present recommendation with sources indicated +``` + +### Fallback Behavior + +| Scenario | Action | +|----------|--------| +| Skopeo not installed | Prompt installation, offer static-only mode | +| Skopeo command fails | Note "unable to verify", use static data | +| Security API unavailable | Note "security not verified", proceed | +| Image not found | Remove from candidates, suggest alternatives | +| Network offline | Use static data only, note limitations | + +--- + +## Integration with Recommendation Output + +### When Dynamic Data Available + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | 147 MB | Skopeo | +| Built | 2026-01-28 | Skopeo | +| Architecture | amd64, arm64 | Skopeo | + +**Security Status:** No critical CVEs +- Last checked: 2026-02-03 +- Source: Red Hat Security Data API +``` + +### When Dynamic Data Unavailable + +```markdown +| Property | Value | Source | +|----------|-------|--------| +| Size | ~150 MB (estimate) | Static | +| Built | Unknown | - | +| Architecture | Assumed amd64 | Static | + +**Security Status:** Not verified (warning) +- Skopeo not installed - install for accurate metadata +- Run: `sudo dnf install skopeo` +``` diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/human-in-the-loop.md new file mode 100644 index 00000000..696fccf1 --- /dev/null +++ b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/human-in-the-loop.md @@ -0,0 +1,98 @@ +# Human-in-the-Loop Requirements + +This document defines mandatory checkpoint behavior for all rh-developer skills. + +## Critical Requirements + +**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: + +1. **Wait for user confirmation** before executing any actions +2. **Do NOT proceed** to the next step until the user explicitly approves +3. **Present options clearly** (yes/no/modify) and wait for response +4. **Never auto-execute** resource creation, builds, or deployments +5. **Never skip configuration questions** even if user seems to know what they want + +If the user says "no" or wants modifications, address their concerns before proceeding. + +## Anti-Patterns to Avoid + +**CRITICAL - DO NOT DO THIS:** + +| Anti-Pattern | Why It's Wrong | +|--------------|----------------| +| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | +| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | +| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | +| User is in a hurry → Rush through phases | Speed causes mistakes in production | + +## When User Provides Multiple Answers + +If user says: "yes do helm deployment to test-app namespace" + +**DO NOT** skip phases. Instead: + +1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." +2. Continue: "Let me confirm the configuration details..." +3. Still ask: Environment type, config approach, resources, etc. +4. Get explicit confirmation for each phase + +**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** + +## Standard Checkpoint Language + +Use this exact pattern after EVERY step/phase: + +```markdown +**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. + +- If user says "yes" → Proceed to next phase +- If user says "no" → Ask what they would like to change +- If user says "modify" → Update configuration and show again for confirmation +- If user gives multiple answers at once → Still confirm each remaining checkpoint individually +``` + +## Mandatory Configuration Questions + +Before ANY resource creation, these questions should be asked: + +| Question | Why It Matters | +|----------|----------------| +| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | +| Runtime vs build-time config | Affects flexibility and rebuild frequency | +| Resource limits | Prevents OOM, ensures fair scheduling | +| Replicas | Affects availability and cost | + +## Include in Your Skill + +Add this section after Prerequisites in your SKILL.md: + +```markdown +## Critical: Human-in-the-Loop Requirements + +See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. + +**Key Rules:** +1. WAIT for explicit user confirmation at each phase +2. Never skip configuration questions, even if user specifies strategy upfront +3. Strategy choice ≠ Configuration approval +``` + +## Phase Execution Rules + +**MANDATORY:** Execute phases in order. Each phase MUST: + +1. Display the phase information to the user +2. Ask the specific question for that phase +3. Wait for user response +4. Only then proceed to next phase + +**Even if user provides information for multiple phases at once:** +- Acknowledge what they said +- But still display each phase's confirmation prompt +- Get explicit "yes" for each phase before executing + +Example: +- User: "yes do helm to test-app namespace" +- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." +- [Still show Configuration Review phase] +- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/image-selection-criteria.md new file mode 100644 index 00000000..184b7f5e --- /dev/null +++ b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/image-selection-criteria.md @@ -0,0 +1,221 @@ +--- +title: Image Selection Criteria Reference +category: containers +sources: + - title: Red Hat Container Best Practices + url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification + sections: Image sizing, Security considerations + date_accessed: 2026-02-08 + - title: OpenShift Image Guidelines + url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html + sections: Image creation, Optimization + date_accessed: 2026-02-08 +--- + +# Image Selection Criteria Reference + +This document provides detailed criteria for selecting the optimal container image based on use case requirements. + +## Scoring Matrix + +Use this matrix to score image options based on user requirements. + +### Criteria Weights by Environment + +| Criteria | Production | Development | Edge/IoT | Serverless | +|----------|------------|-------------|----------|------------| +| Image Size | 3 | 1 | 5 | 4 | +| Security Posture | 5 | 2 | 4 | 3 | +| Build Tools | 1 | 5 | 1 | 1 | +| Startup Time | 3 | 1 | 3 | 5 | +| LTS Status | 5 | 2 | 4 | 3 | +| Debug Tools | 1 | 5 | 1 | 1 | + +**Scale:** 1 (low importance) to 5 (high importance) + +### Image Variant Scores + +| Variant | Size | Security | Build Tools | Startup | Debug | +|---------|------|----------|-------------|---------|-------| +| Full | 2 | 2 | 5 | 2 | 5 | +| Minimal | 4 | 4 | 2 | 4 | 2 | +| Runtime | 5 | 5 | 1 | 5 | 1 | + +**Scale:** 1 (poor) to 5 (excellent) + +## Image Size Reference + +Approximate compressed image sizes: + +### Node.js +| Image | Size | +|-------|------| +| `ubi9/nodejs-20` | ~250MB | +| `ubi9/nodejs-20-minimal` | ~150MB | + +### Python +| Image | Size | +|-------|------| +| `ubi9/python-311` | ~280MB | + +### Java +| Image | Size | +|-------|------| +| `ubi9/openjdk-17` | ~400MB | +| `ubi9/openjdk-17-runtime` | ~200MB | + +### Go +| Image | Size | +|-------|------| +| `ubi9/go-toolset:1.21` | ~500MB | +| Final binary | ~10-50MB | + +### .NET +| Image | Size | +|-------|------| +| `ubi9/dotnet-80` | ~350MB | +| `ubi9/dotnet-80-runtime` | ~150MB | + +## LTS Support Timeline + +### Node.js +| Version | Status | End of Life | +|---------|--------|-------------| +| 18 LTS | Active | April 2025 | +| 20 LTS | Active | April 2026 | +| 22 LTS | Active | April 2027 | + +### Python +| Version | Status | End of Life | +|---------|--------|-------------| +| 3.9 | Security | October 2025 | +| 3.11 | Active | October 2027 | +| 3.12 | Active | October 2028 | + +### Java (OpenJDK) +| Version | Status | Extended Support | +|---------|--------|------------------| +| 11 LTS | Active | Red Hat until 2027 | +| 17 LTS | Active | Red Hat until 2029 | +| 21 LTS | Active | Red Hat until 2031 | + +### .NET +| Version | Status | End of Life | +|---------|--------|-------------| +| 6.0 LTS | Active | November 2024 | +| 8.0 LTS | Active | November 2026 | + +## Security Considerations + +### Minimal Images - When to Use +- Fewer installed packages = smaller attack surface +- Recommended for production workloads +- May lack debugging tools when issues occur + +### Full Images - When to Use +- Include development tools (gcc, make, etc.) +- Needed for native extensions (Python C extensions, Node native modules) +- Better for development and debugging + +### Runtime Images - When to Use +- No build tools at all +- Smallest possible footprint +- Requires pre-compiled application (JAR, static binary) + +## Framework-Specific Considerations + +### Quarkus (Java) +**For JVM mode:** +- Use `ubi9/openjdk-21` for build +- Use `ubi9/openjdk-21-runtime` for production + +**For Native mode:** +- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` +- Run: `quay.io/quarkus/quarkus-micro-image:2.0` +- Dramatically faster startup (~50ms vs ~2s) + +### Spring Boot (Java) +**Standard:** +- Build and run: `ubi9/openjdk-17` + +**Optimized production:** +- Build with layered JAR: `spring-boot-maven-plugin` with layers +- Run on: `ubi9/openjdk-17-runtime` + +### Next.js (Node.js) +**Development:** +- Use `ubi9/nodejs-20` + +**Production (multi-stage recommended):** +1. Build stage: `ubi9/nodejs-20` +2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output + +### Django/Flask (Python) +- Always use full image (may need compilation for dependencies) +- `ubi9/python-311` recommended +- Consider `gunicorn` for production + +## Decision Tree + +``` +START + | + v +Is this production? + | + +-- YES --> Need native compilation? + | | + | +-- YES --> Use FULL variant + | | + | +-- NO --> Is app pre-compiled? + | | + | +-- YES --> Use RUNTIME variant + | | + | +-- NO --> Use MINIMAL variant + | + +-- NO (Development) --> Use FULL variant +``` + +## Multi-Stage Build Recommendations + +For optimal production images, consider multi-stage builds: + +### Node.js Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder +COPY . . +RUN npm ci && npm run build + +# Production stage +FROM registry.access.redhat.com/ubi9/nodejs-20-minimal +COPY --from=builder /app/dist /app +CMD ["node", "/app/index.js"] +``` + +### Java Example +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder +COPY . . +RUN mvn package -DskipTests + +# Production stage +FROM registry.access.redhat.com/ubi9/openjdk-21-runtime +COPY --from=builder /app/target/*.jar /app/app.jar +CMD ["java", "-jar", "/app/app.jar"] +``` + +### Go Example +Go produces static binaries, so minimal base is ideal: +```dockerfile +# Build stage +FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder +COPY . . +RUN go build -o /app/server + +# Production stage +FROM registry.access.redhat.com/ubi9/ubi-micro +COPY --from=builder /app/server /server +CMD ["/server"] +``` diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/prerequisites.md new file mode 100644 index 00000000..d81a9b5c --- /dev/null +++ b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/prerequisites.md @@ -0,0 +1,212 @@ +--- +title: Prerequisites +category: setup +sources: + - title: OpenShift CLI (oc) Installation + url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html + sections: Installing the CLI, Logging in + date_accessed: 2026-02-08 + - title: Helm Installation Guide + url: https://helm.sh/docs/intro/install/ + sections: From script, From package managers + date_accessed: 2026-02-08 + - title: Podman Installation + url: https://podman.io/docs/installation + sections: Linux, macOS, Windows + date_accessed: 2026-02-08 + - title: Skopeo Installation + url: https://github.com/containers/skopeo/blob/main/install.md + sections: Distribution packages, Building from source + date_accessed: 2026-02-08 +--- + +# Prerequisites + +This document lists all tools required by the rh-developer agentic collection. + +## Required Tools by Skill + +| Skill | Required Tools | Optional Tools | +|-------|----------------|----------------| +| `/detect-project` | `git` | - | +| `/s2i-build` | `oc` | `git` | +| `/deploy` | `oc` | - | +| `/helm-deploy` | `oc`, `helm` | - | +| `/containerize-deploy` | `oc` | `git`, `helm` | +| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | +| `/recommend-image` | - | `skopeo`, `curl`, `jq` | +| `/debug-pod` | `oc` | - | +| `/debug-build` | `oc` | - | +| `/debug-network` | `oc` | - | +| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | +| `/debug-container` | `podman` or `docker` | - | + +## Tool Reference + +### OpenShift CLI (oc) + +**Required for:** Cluster operations, S2I builds, deployments + +```bash +# Check installation +oc version + +# Installation +# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ +# Or via package manager: +sudo dnf install openshift-clients # Fedora/RHEL +brew install openshift-cli # macOS +``` + +### Helm + +**Required for:** Helm chart deployments + +```bash +# Check installation +helm version + +# Installation +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Or via package manager: +sudo dnf install helm # Fedora/RHEL +brew install helm # macOS +``` + +### Podman + +**Required for:** Container builds, RHEL container deployments + +```bash +# Check installation +podman --version + +# Installation +sudo dnf install podman # Fedora/RHEL/CentOS +sudo apt install podman # Ubuntu/Debian +brew install podman # macOS +``` + +### Docker (alternative to Podman) + +**Required for:** Container builds (if Podman not available) + +```bash +# Check installation +docker --version + +# Installation +# See: https://docs.docker.com/engine/install/ +``` + +### Skopeo + +**Required for:** Image inspection, tag verification + +```bash +# Check installation +skopeo --version + +# Installation +sudo dnf install skopeo # Fedora/RHEL/CentOS +sudo apt install skopeo # Ubuntu/Debian +brew install skopeo # macOS +``` + +### Git + +**Required for:** Repository cloning + +```bash +# Check installation +git --version + +# Installation +sudo dnf install git # Fedora/RHEL/CentOS +sudo apt install git # Ubuntu/Debian +brew install git # macOS (or Xcode Command Line Tools) +``` + +### SSH + +**Required for:** RHEL remote deployments + +```bash +# Check installation +ssh -V + +# Usually pre-installed on Linux/macOS +# Windows: Use OpenSSH or WSL +``` + +### curl and jq + +**Required for:** API calls and JSON parsing + +```bash +# Check installation +curl --version +jq --version + +# Installation +sudo dnf install curl jq # Fedora/RHEL/CentOS +sudo apt install curl jq # Ubuntu/Debian +brew install curl jq # macOS +``` + +## Cluster Requirements + +### OpenShift Cluster Access + +For S2I builds and deployments, you need: + +1. **Logged in to cluster:** + ```bash + oc login + # or + oc login --token= --server= + ``` + +2. **Namespace with edit permissions:** + ```bash + # Verify access + oc auth can-i create deployments + oc auth can-i create buildconfigs + ``` + +3. **Image registry accessible:** + ```bash + # Verify internal registry + oc get route -n openshift-image-registry + ``` + +### RHEL/Fedora Host Access + +For RHEL deployments, you need: + +1. **SSH access to target host:** + ```bash + ssh user@target-host + ``` + +2. **sudo privileges on target** (for systemd services) + +3. **Firewall ports open** (for application access) + +## Quick Validation + +Run these commands to check your environment: + +```bash +# Core tools +which oc helm podman git ssh curl jq skopeo + +# Cluster connection (if using OpenShift) +oc whoami +oc project + +# Container runtime +podman info || docker info +``` + +Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/python-s2i-entrypoints.md new file mode 100644 index 00000000..bb29398e --- /dev/null +++ b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/python-s2i-entrypoints.md @@ -0,0 +1,70 @@ +--- +title: Python S2I Entry Point Requirements +category: containers +sources: + - title: UBI Python S2I Builder + url: https://github.com/sclorg/s2i-python-container + sections: Run script logic, APP_MODULE configuration + date_accessed: 2026-02-08 + - title: Red Hat Python S2I Documentation + url: https://catalog.redhat.com/software/containers/ubi9/python-311 + sections: Environment variables, Startup behavior + date_accessed: 2026-02-08 +--- + +# Python S2I Entry Point Requirements + +The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. + +## How the S2I Python Run Script Works + +The S2I Python builder uses this startup logic (in order): + +1. If `app.sh` exists → Execute it directly +2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn +3. If `app.py` exists → Run with Python directly +4. Otherwise → **ERROR: No start command found** + +## Entry Point Configuration Matrix + +| Entry Point File | gunicorn in requirements | Configuration Needed | Result | +|------------------|--------------------------|----------------------|--------| +| `app.py` | No | None | Works (Python direct) | +| `app.py` | Yes | None (optional APP_MODULE) | Works | +| `main.py` | **No** | - | **FAILS** | +| `main.py` | Yes | `APP_MODULE=main:app` | Works | +| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | +| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | + +## APP_MODULE Format + +- **Format:** `[python_module]:[flask_app_variable]` +- **Example:** `main:app` → imports `app` from `main.py` +- **Requires:** `gunicorn` in `requirements.txt` + +### Common Patterns + +| File | Typical APP_MODULE | +|------|-------------------| +| `main.py` with `app = Flask(__name__)` | `main:app` | +| `main.py` with `application = Flask(__name__)` | `main:application` | +| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | +| `src/app.py` with `app` | `src.app:app` | + +## Alternative: APP_FILE + +- Set `APP_FILE=main.py` to run with Python directly (development mode) +- **Not recommended for production** (no WSGI server, no worker management) +- Use only if gunicorn is not an option + +## Critical Warning + +**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** +- The S2I build will succeed (dependencies install) +- The container will **fail to start** with "No start command found" +- This is a **runtime failure**, not a build failure + +**Always verify:** +1. Entry point file name +2. `gunicorn` in requirements.txt +3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/rhel-deployment.md new file mode 100644 index 00000000..06eda277 --- /dev/null +++ b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/rhel-deployment.md @@ -0,0 +1,580 @@ +--- +title: RHEL Deployment Reference +category: deployment +sources: + - title: RHEL System Administrator's Guide - systemd + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings + sections: Managing services, Unit files + date_accessed: 2026-02-08 + - title: RHEL SELinux Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux + sections: Contexts, Port labeling + date_accessed: 2026-02-08 + - title: RHEL Firewall Configuration + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters + sections: firewalld, Opening ports + date_accessed: 2026-02-08 +--- + +# RHEL Deployment Reference + +Reference material for deploying applications to standalone RHEL systems. + +## Table of Contents + +1. [RHEL Version Compatibility](#rhel-version-compatibility) +2. [Systemd Unit Templates](#systemd-unit-templates) +3. [SELinux Configuration](#selinux-configuration) +4. [Firewall Commands](#firewall-commands) +5. [SSH Connection Patterns](#ssh-connection-patterns) +6. [Runtime Package Mapping](#runtime-package-mapping) + +--- + +## RHEL Version Compatibility + +| Distribution | Version | Podman | Recommended | +|--------------|---------|--------|-------------| +| RHEL | 8.x | 4.0+ | Production ready | +| RHEL | 9.x | 4.4+ | **Recommended** | +| CentOS Stream | 8 | 4.0+ | Development | +| CentOS Stream | 9 | 4.4+ | Development | +| Rocky Linux | 8.x | 4.0+ | Production ready | +| Rocky Linux | 9.x | 4.4+ | Production ready | +| AlmaLinux | 8.x | 4.0+ | Production ready | +| AlmaLinux | 9.x | 4.4+ | Production ready | +| Fedora | 38+ | 4.6+ | Latest features | + +### Version Detection Commands + +```bash +# Get RHEL/CentOS version +cat /etc/redhat-release + +# Get detailed OS info +cat /etc/os-release + +# Check architecture +uname -m + +# Check kernel version +uname -r +``` + +--- + +## Systemd Unit Templates + +### Podman Container Service (Rootful) + +```ini +[Unit] +Description=${APP_NAME} Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +# Pre-start: ensure clean state +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} + +# Main container run +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} + +# Stop container gracefully +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Podman Container Service (Rootless) + +```ini +[Unit] +Description=${APP_NAME} Container (Rootless) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +TimeoutStartSec=300 +TimeoutStopSec=70 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=default.target +``` + +**Rootless setup commands:** +```bash +# Create user systemd directory +mkdir -p ~/.config/systemd/user + +# Place unit file +cp ${APP_NAME}.service ~/.config/systemd/user/ + +# Reload and enable +systemctl --user daemon-reload +systemctl --user enable --now ${APP_NAME} + +# Keep services running after logout +loginctl enable-linger ${USER} +``` + +### Podman Container with Volumes + +```ini +[Unit] +Description=${APP_NAME} Container with Persistent Data +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=5 + +ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} +ExecStartPre=-/usr/bin/podman rm ${APP_NAME} +ExecStart=/usr/bin/podman run \ + --name ${APP_NAME} \ + -p ${HOST_PORT}:${CONTAINER_PORT} \ + -v /var/lib/${APP_NAME}/data:/app/data:z \ + -e DATABASE_URL=${DATABASE_URL} \ + --rm \ + ${IMAGE} +ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Node.js Application + +```ini +[Unit] +Description=${APP_NAME} Node.js Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=NODE_ENV=production +Environment=PORT=${PORT} +ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Python Application + +```ini +[Unit] +Description=${APP_NAME} Python Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PYTHONUNBUFFERED=1 +Environment=PORT=${PORT} +ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Java Application + +```ini +[Unit] +Description=${APP_NAME} Java Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=JAVA_OPTS=-Xmx512m +ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} +Restart=always +RestartSec=5 +SuccessExitStatus=143 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/opt/${APP_NAME} + +[Install] +WantedBy=multi-user.target +``` + +### Native Go Application + +```ini +[Unit] +Description=${APP_NAME} Go Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=/opt/${APP_NAME} +Environment=PORT=${PORT} +ExecStart=/opt/${APP_NAME}/${BINARY_NAME} +Restart=always +RestartSec=5 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +--- + +## SELinux Configuration + +### Common SELinux Contexts + +| Context Type | Use Case | +|--------------|----------| +| `container_t` | Standard Podman container processes | +| `container_file_t` | Container data files | +| `bin_t` | Executable binaries | +| `httpd_sys_content_t` | Web application content (read-only) | +| `httpd_sys_rw_content_t` | Web application content (read-write) | +| `var_lib_t` | Application data in /var/lib | + +### Volume Label Options for Podman + +| Option | Description | Use Case | +|--------|-------------|----------| +| `:z` | Shared volume label | Volume accessed by multiple containers | +| `:Z` | Private volume label | Volume accessed by single container only | + +Example: +```bash +podman run -v /data/shared:/app/shared:z myimage # Shared +podman run -v /data/private:/app/data:Z myimage # Private +``` + +### SELinux Commands + +```bash +# Check current SELinux mode +getenforce + +# View file context +ls -Z /path/to/file + +# Set context for application directory +sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" +sudo restorecon -Rv /opt/myapp + +# Set context for web content +sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" +sudo restorecon -Rv /opt/myapp/public + +# Allow non-standard port for HTTP +sudo semanage port -a -t http_port_t -p tcp 8080 + +# View port contexts +sudo semanage port -l | grep http + +# Check for SELinux denials +sudo ausearch -m AVC -ts recent + +# Generate policy from denials (troubleshooting) +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy +sudo semodule -i mypolicy.pp + +# Temporarily set permissive (for debugging only) +sudo setenforce 0 +``` + +### Common SELinux Booleans + +```bash +# Allow HTTP to connect to network (for proxy/API calls) +sudo setsebool -P httpd_can_network_connect 1 + +# Allow HTTP to connect to databases +sudo setsebool -P httpd_can_network_connect_db 1 + +# List all HTTP-related booleans +getsebool -a | grep httpd +``` + +--- + +## Firewall Commands + +### Basic Port Management + +```bash +# Check firewall status +sudo firewall-cmd --state + +# List all open ports +sudo firewall-cmd --list-ports + +# List all services +sudo firewall-cmd --list-services + +# Open port permanently +sudo firewall-cmd --permanent --add-port=8080/tcp + +# Open port temporarily (until reload) +sudo firewall-cmd --add-port=8080/tcp + +# Reload firewall to apply permanent changes +sudo firewall-cmd --reload + +# Remove port +sudo firewall-cmd --permanent --remove-port=8080/tcp +sudo firewall-cmd --reload +``` + +### Service-Based Management + +```bash +# Add HTTP service +sudo firewall-cmd --permanent --add-service=http + +# Add HTTPS service +sudo firewall-cmd --permanent --add-service=https + +# Remove service +sudo firewall-cmd --permanent --remove-service=http + +# Apply changes +sudo firewall-cmd --reload +``` + +### Zone Management + +```bash +# List zones +sudo firewall-cmd --get-zones + +# Get active zone +sudo firewall-cmd --get-active-zones + +# Add port to specific zone +sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp + +# Set default zone +sudo firewall-cmd --set-default-zone=public +``` + +### Rich Rules (Advanced) + +```bash +# Allow specific IP to access port +sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' + +# Rate limiting +sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' + +# Apply changes +sudo firewall-cmd --reload +``` + +--- + +## SSH Connection Patterns + +### Test Connection + +```bash +# Basic connection test +ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" + +# Verbose output for debugging +ssh -v user@host + +# Test with specific key +ssh -i ~/.ssh/mykey user@host "echo 'OK'" +``` + +### Execute Remote Commands + +```bash +# Single command +ssh user@host "command" + +# Multiple commands +ssh user@host "cmd1 && cmd2 && cmd3" + +# With sudo +ssh user@host "sudo command" + +# Preserve environment +ssh user@host 'bash -l -c "command"' +``` + +### File Transfer + +```bash +# Copy file to remote +scp local_file user@host:/remote/path/ + +# Copy directory recursively +scp -r local_dir user@host:/remote/path/ + +# Using rsync (preferred for large transfers) +rsync -avz --progress local_dir/ user@host:/remote/path/ + +# Exclude patterns +rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ +``` + +### SSH Config for Convenience + +``` +# ~/.ssh/config +Host myrhel + HostName 192.168.1.100 + User deploy + Port 22 + IdentityFile ~/.ssh/id_rsa + StrictHostKeyChecking accept-new +``` + +Usage: `ssh myrhel "command"` + +--- + +## Runtime Package Mapping + +### Node.js + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | +| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | + +### Python + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.8 | `dnf install -y python38 python38-pip` | N/A | +| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | +| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | +| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | + +### Java + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | +| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | +| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | + +### Go + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | + +### Ruby + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | +| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | + +### PHP + +| Version | RHEL 8 | RHEL 9 | +|---------|--------|--------| +| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | +| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | +| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | + +### Module Stream Commands + +```bash +# List available streams for a module +dnf module list nodejs + +# Enable specific stream +sudo dnf module enable nodejs:20 + +# Reset module (to switch streams) +sudo dnf module reset nodejs + +# Install from enabled stream +sudo dnf install -y nodejs npm +``` + +--- + +## Service User Creation + +For running applications as non-root: + +```bash +# Create system user for the application +sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp + +# Set ownership +sudo chown -R myapp:myapp /opt/myapp + +# Allow user to bind to privileged port (if needed) +sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary +``` diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/selinux-troubleshooting.md new file mode 100644 index 00000000..9942375c --- /dev/null +++ b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/selinux-troubleshooting.md @@ -0,0 +1,387 @@ +--- +title: SELinux Troubleshooting +category: references +sources: + - title: Red Hat SELinux User's and Administrator's Guide + url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index + sections: Troubleshooting, Managing confined services + date_accessed: 2026-02-16 + - title: SELinux Project Wiki + url: https://selinuxproject.org/page/Main_Page + sections: Troubleshooting + date_accessed: 2026-02-16 + - title: Fedora SELinux Guide + url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ + sections: Troubleshooting + date_accessed: 2026-02-16 +--- + +# SELinux Troubleshooting + +This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. + +## Understanding SELinux + +### SELinux Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | +| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | +| **Disabled** | SELinux is completely disabled | Not recommended | + +```bash +# Check current mode +getenforce + +# Temporarily switch to permissive (until reboot) +sudo setenforce 0 + +# Switch back to enforcing +sudo setenforce 1 +``` + +### SELinux Contexts + +Every file, process, and port has an SELinux context: + +``` +user:role:type:level +``` + +Example: `system_u:object_r:httpd_sys_content_t:s0` + +- **user**: SELinux user (system_u, user_u, etc.) +- **role**: Role (object_r for files) +- **type**: Type label (most important for troubleshooting) +- **level**: MLS/MCS level (usually s0) + +```bash +# View file context +ls -lZ /path/to/file + +# View process context +ps auxZ | grep [process] + +# View port context +semanage port -l | grep [port] +``` + +## Finding SELinux Denials + +### Using ausearch + +```bash +# Recent denials (last 10 minutes) +sudo ausearch -m AVC -ts recent + +# Denials from today +sudo ausearch -m AVC -ts today + +# Denials for specific process +sudo ausearch -m AVC -c [command-name] + +# Denials involving specific file +sudo ausearch -m AVC -f /path/to/file +``` + +### Using journalctl + +```bash +# SELinux messages in journal +sudo journalctl -t setroubleshoot + +# AVC messages +sudo journalctl | grep "avc: denied" +``` + +### Using sealert + +```bash +# Install setroubleshoot (if not installed) +sudo dnf install setroubleshoot-server + +# Analyze all denials +sudo sealert -a /var/log/audit/audit.log + +# Interactive analysis +sudo sealert -b +``` + +## Reading AVC Denials + +Example AVC denial: + +``` +type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 +``` + +**Breakdown:** +| Field | Value | Meaning | +|-------|-------|---------| +| `denied { bind }` | bind | Denied action (bind to socket) | +| `pid=1234` | 1234 | Process ID | +| `comm="httpd"` | httpd | Command name | +| `src=8080` | 8080 | Port number | +| `scontext=...httpd_t...` | httpd_t | Source type (process) | +| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | +| `tclass=tcp_socket` | tcp_socket | Object class | + +**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). + +## Common Denial Types and Fixes + +### Port Binding Denials + +**Symptom:** Application cannot bind to non-standard port + +**Example denial:** +``` +avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t +``` + +**Fix:** +```bash +# Add port to allowed type +sudo semanage port -a -t http_port_t -p tcp 8080 + +# Verify +sudo semanage port -l | grep 8080 +``` + +**Common port types:** +| Port Type | Typical Ports | Used By | +|-----------|---------------|---------| +| `http_port_t` | 80, 443, 8080 | Web servers | +| `postgresql_port_t` | 5432 | PostgreSQL | +| `mysqld_port_t` | 3306 | MySQL/MariaDB | +| `redis_port_t` | 6379 | Redis | +| `mongod_port_t` | 27017 | MongoDB | + +### File Access Denials + +**Symptom:** Application cannot read/write files + +**Example denial:** +``` +avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t +``` + +**Fix - Change file context:** +```bash +# Set file context pattern +sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" + +# Apply the context +sudo restorecon -Rv /srv/myapp + +# Verify +ls -lZ /srv/myapp +``` + +**Common file types:** +| File Type | Access | Use Case | +|-----------|--------|----------| +| `httpd_sys_content_t` | Read | Web content | +| `httpd_sys_rw_content_t` | Read/Write | Web app data | +| `container_file_t` | Container access | Podman volumes | +| `var_log_t` | Log files | Application logs | + +### Network Connection Denials + +**Symptom:** Application cannot connect to external services + +**Example denial:** +``` +avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t +``` + +**Fix - Enable boolean:** +```bash +# Allow httpd to connect to network +sudo setsebool -P httpd_can_network_connect on + +# Or specifically to databases +sudo setsebool -P httpd_can_network_connect_db on + +# List all httpd booleans +sudo getsebool -a | grep httpd +``` + +**Common booleans:** +| Boolean | Purpose | +|---------|---------| +| `httpd_can_network_connect` | Allow outbound network connections | +| `httpd_can_network_connect_db` | Allow database connections | +| `httpd_can_sendmail` | Allow sending email | +| `httpd_use_nfs` | Allow NFS access | +| `container_manage_cgroup` | Allow container cgroup management | + +## Container-Specific Issues + +### Podman Volume Mounts + +When mounting host directories into containers, SELinux may block access. + +**Solutions:** + +1. **Shared label (:z)** - Multiple containers can access + ```bash + podman run -v /host/path:/container/path:z [image] + ``` + +2. **Private label (:Z)** - Only this container can access + ```bash + podman run -v /host/path:/container/path:Z [image] + ``` + +3. **Manual relabeling:** + ```bash + sudo semanage fcontext -a -t container_file_t "/data(/.*)?" + sudo restorecon -Rv /data + ``` + +### Container Booleans + +```bash +# Enable container to manage cgroups (for systemd in container) +sudo setsebool -P container_manage_cgroup on + +# Allow containers to connect to any port +sudo setsebool -P container_connect_any on + +# List all container booleans +sudo getsebool -a | grep container +``` + +## Troubleshooting Workflow + +### Step 1: Confirm SELinux is the Issue + +```bash +# Temporarily disable SELinux +sudo setenforce 0 + +# Test if application works +[test application] + +# Re-enable SELinux +sudo setenforce 1 +``` + +If application works with SELinux permissive, SELinux is blocking. + +### Step 2: Find the Denial + +```bash +# Get recent denials +sudo ausearch -m AVC -ts recent + +# Or use sealert for analysis +sudo sealert -a /var/log/audit/audit.log +``` + +### Step 3: Determine Fix Type + +| Denial Type | Fix Approach | +|-------------|--------------| +| Port binding | `semanage port` | +| File access | `semanage fcontext` + `restorecon` | +| Network connection | `setsebool` | +| Process capability | Custom policy or boolean | + +### Step 4: Apply Fix + +```bash +# For port: +sudo semanage port -a -t [type] -p [tcp/udp] [port] + +# For file: +sudo semanage fcontext -a -t [type] "[path](/.*)?" +sudo restorecon -Rv [path] + +# For boolean: +sudo setsebool -P [boolean] on +``` + +### Step 5: Verify + +```bash +# Test application +[restart and test] + +# Check for new denials +sudo ausearch -m AVC -ts recent +``` + +## Generating Custom Policies + +If no existing type or boolean works, generate a custom policy: + +```bash +# Generate policy from recent denials +sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy + +# Review the policy +cat mypolicy.te + +# Install the policy +sudo semodule -i mypolicy.pp +``` + +**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. + +## Quick Reference + +### Common Commands + +```bash +# SELinux status +getenforce +sestatus + +# File context +ls -lZ [path] +restorecon -Rv [path] + +# Process context +ps auxZ | grep [process] + +# Port context +semanage port -l | grep [port] +semanage port -a -t [type] -p tcp [port] + +# Booleans +getsebool -a | grep [keyword] +setsebool -P [boolean] on + +# File context rules +semanage fcontext -l | grep [path] +semanage fcontext -a -t [type] "[path](/.*)?" + +# Audit logs +ausearch -m AVC -ts recent +sealert -a /var/log/audit/audit.log +``` + +### Common Types for Web Applications + +| Resource | Type | +|----------|------| +| Web content (read-only) | `httpd_sys_content_t` | +| Web content (read-write) | `httpd_sys_rw_content_t` | +| Web scripts | `httpd_sys_script_exec_t` | +| Application logs | `httpd_log_t` | +| HTTP ports | `http_port_t` | +| Container files | `container_file_t` | + +### Common Booleans for Applications + +| Application | Boolean | Purpose | +|-------------|---------|---------| +| Web server | `httpd_can_network_connect` | Outbound connections | +| Web server | `httpd_can_network_connect_db` | Database connections | +| Web server | `httpd_unified` | Unified handling | +| Container | `container_manage_cgroup` | cgroup management | +| Container | `container_connect_any` | Connect to any port | +| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/mcp-servers/mock-openshift-mcp.py b/evaluation/without_skills/rh-developer__validate-environment/environment/mcp-servers/mock-openshift-mcp.py new file mode 100644 index 00000000..dadb59fb --- /dev/null +++ b/evaluation/without_skills/rh-developer__validate-environment/environment/mcp-servers/mock-openshift-mcp.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for rh-developer benchmark task. + +Simulates an OpenShift cluster with 3 namespaces, each containing a broken +deployment that requires different debugging skills to diagnose: + + 1. api-platform / api-service (Python FastAPI) + - S2I build succeeded, pod crashes at runtime + - Entry point is main.py (not app.py), no gunicorn installed + - Requires python-s2i-entrypoints.md knowledge + + 2. web-frontend / web-frontend (Node.js React) + - Pod in CrashLoopBackOff, exit code 137 (OOMKilled) + - Container memory limit 64Mi is too low for Node.js + - Requires debugging-patterns.md exit code knowledge + + 3. order-system / order-service (Java Quarkus) + - Pod running, Route returns 503 + - Service selector mismatch: app=order-svc vs pod label app=order-service + - Tekton PipelineRun failed, logs in step-build container + - Requires debug-network + debug-pipeline knowledge + +Also provides application source metadata for image recommendation. +""" + +from typing import Optional +from fastmcp import FastMCP + +mcp = FastMCP("openshift") + + +# --------------------------------------------------------------------------- +# Namespace / Project data +# --------------------------------------------------------------------------- + +NAMESPACES = [ + {"name": "api-platform", "status": "Active", "labels": {"app-type": "backend"}}, + {"name": "web-frontend", "status": "Active", "labels": {"app-type": "frontend"}}, + {"name": "order-system", "status": "Active", "labels": {"app-type": "backend"}}, +] + + +# --------------------------------------------------------------------------- +# Deployment data +# --------------------------------------------------------------------------- + +DEPLOYMENTS = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "containers": [ + { + "name": "api-service", + "image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + "env": [ + {"name": "APP_SCRIPT", "value": ""}, + {"name": "APP_FILE", "value": "main.py"}, + ], + } + ], + "labels": {"app": "api-service", "deployment": "api-service"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "replicas": 1, + "available_replicas": 0, + "ready_replicas": 0, + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "containers": [ + { + "name": "web-frontend", + "image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "strategy": "RollingUpdate", + "status": "Available=False (0/1 replicas ready)", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "replicas": 1, + "available_replicas": 1, + "ready_replicas": 1, + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "containers": [ + { + "name": "order-service", + "image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + } + ], + "labels": {"app": "order-service", "deployment": "order-service"}, + "strategy": "RollingUpdate", + "status": "Available=True (1/1 replicas ready)", + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod data +# --------------------------------------------------------------------------- + +PODS = { + "api-platform": [ + { + "name": "api-service-7b8f9d4c5-x2k9m", + "namespace": "api-platform", + "status": "CrashLoopBackOff", + "restart_count": 5, + "labels": {"app": "api-service", "deployment": "api-service"}, + "containers": [ + { + "name": "api-service", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 1, + "reason": "Error", + "message": "Application exited with error", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}, + }, + } + ], + }, + ], + "web-frontend": [ + { + "name": "web-frontend-6c5d8b7a9-p4n2j", + "namespace": "web-frontend", + "status": "CrashLoopBackOff", + "restart_count": 8, + "labels": {"app": "web-frontend", "deployment": "web-frontend"}, + "containers": [ + { + "name": "web-frontend", + "state": "Waiting", + "reason": "CrashLoopBackOff", + "last_state": { + "terminated": { + "exit_code": 137, + "reason": "OOMKilled", + "message": "Container exceeded memory limit", + } + }, + "ready": False, + "resources": { + "requests": {"cpu": "50m", "memory": "32Mi"}, + "limits": {"cpu": "200m", "memory": "64Mi"}, + }, + } + ], + }, + ], + "order-system": [ + { + "name": "order-service-5a4b3c2d1-h7j6k", + "namespace": "order-system", + "status": "Running", + "restart_count": 0, + "labels": {"app": "order-service", "deployment": "order-service"}, + "containers": [ + { + "name": "order-service", + "state": "Running", + "ready": True, + "ports": [{"containerPort": 8080}], + "resources": { + "requests": {"cpu": "200m", "memory": "512Mi"}, + "limits": {"cpu": "1", "memory": "1Gi"}, + }, + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Pod logs +# --------------------------------------------------------------------------- + +POD_LOGS = { + "api-service-7b8f9d4c5-x2k9m": ( + "---> Running application from script (app.sh) ...\n" + "sh: app.sh: No such file or directory\n" + "---> Trying to run with gunicorn ...\n" + "Traceback (most recent call last):\n" + " File \"/opt/app-root/bin/gunicorn\", line 5, in \n" + " from gunicorn.app.wsgiapp import run\n" + "ModuleNotFoundError: No module named 'gunicorn'\n" + "---> Trying to run app.py ...\n" + "Error: Could not find '/opt/app-root/src/app.py'\n" + "---> Failed to find any valid entry point.\n" + " Set the APP_MODULE environment variable to specify your application callable.\n" + " Expected one of: app.sh, gunicorn with APP_MODULE, or app.py\n" + ), + "web-frontend-6c5d8b7a9-p4n2j": ( + "> react-app@1.0.0 start\n" + "> node server.js\n" + "\n" + "Server starting on port 3000...\n" + "Loading configuration...\n" + "Initializing middleware...\n" + "Killed\n" + ), + "order-service-5a4b3c2d1-h7j6k": ( + "__ ____ __ _____ ___ __ ____ ______ \n" + " --/ __ \\/ / / / _ | / _ \\/ //_/ / / / __/ \n" + " -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\\ \\ \n" + "--\\___\\_\\____/_/ |_/_/|_/_/|_|\\____/___/ \n" + "2026-02-15 10:30:15,234 INFO [io.quarkus] Quarkus 3.8.1 on JVM started in 2.345s.\n" + "2026-02-15 10:30:15,236 INFO [io.quarkus] Profile prod activated.\n" + "2026-02-15 10:30:15,237 INFO [io.quarkus] Installed features: [cdi, rest, smallrye-health]\n" + "2026-02-15 10:30:15,238 INFO [io.quarkus] Listening on: http://0.0.0.0:8080\n" + ), +} + + +# --------------------------------------------------------------------------- +# Build data +# --------------------------------------------------------------------------- + +BUILDS = { + "api-platform": [ + { + "name": "api-service-1", + "namespace": "api-platform", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/api-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/python:3.11-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest", + "duration": "2m15s", + }, + ], + "web-frontend": [ + { + "name": "web-frontend-1", + "namespace": "web-frontend", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/web-frontend.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:20-ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest", + "duration": "3m42s", + }, + ], + "order-system": [ + { + "name": "order-service-1", + "namespace": "order-system", + "status": "Complete", + "source_type": "Git", + "source_uri": "https://github.com/example/order-service.git", + "strategy": "Source", + "builder_image": "image-registry.openshift-image-registry.svc:5000/openshift/openjdk-17:ubi9", + "output_image": "image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest", + "duration": "4m08s", + }, + ], +} + +BUILD_LOGS = { + "api-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/api-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image python:3.11-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from requirements.txt ...\n" + "Collecting fastapi==0.109.0\n" + "Collecting uvicorn==0.27.0\n" + "Collecting pydantic==2.5.3\n" + "Successfully installed fastapi-0.109.0 uvicorn-0.27.0 pydantic-2.5.3\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/api-platform/api-service:latest\n" + "Push successful\n" + ), + "web-frontend-1": ( + "===> STEP 1: Fetching source from https://github.com/example/web-frontend.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image nodejs:20-ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Installing dependencies from package.json ...\n" + "---> Running build script: npm run build ...\n" + "---> Build complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/web-frontend/web-frontend:latest\n" + "Push successful\n" + ), + "order-service-1": ( + "===> STEP 1: Fetching source from https://github.com/example/order-service.git\n" + "Cloning into '/tmp/src'...\n" + "===> STEP 2: Pulling builder image openjdk-17:ubi9\n" + "===> STEP 3: Running assemble script\n" + "---> Installing application source ...\n" + "---> Building with Maven ...\n" + "[INFO] BUILD SUCCESS\n" + "---> Assemble script complete.\n" + "===> STEP 4: Committing image\n" + "===> STEP 5: Pushing image to image-registry.openshift-image-registry.svc:5000/order-system/order-service:latest\n" + "Push successful\n" + ), +} + + +# --------------------------------------------------------------------------- +# Service data +# --------------------------------------------------------------------------- + +SERVICES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "type": "ClusterIP", + "cluster_ip": "172.30.45.112", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "api-service"}, + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "type": "ClusterIP", + "cluster_ip": "172.30.89.201", + "ports": [{"port": 3000, "target_port": 3000, "protocol": "TCP"}], + "selector": {"app": "web-frontend"}, + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "type": "ClusterIP", + "cluster_ip": "172.30.67.55", + "ports": [{"port": 8080, "target_port": 8080, "protocol": "TCP"}], + "selector": {"app": "order-svc"}, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Route data +# --------------------------------------------------------------------------- + +ROUTES = { + "api-platform": [ + { + "name": "api-service", + "namespace": "api-platform", + "host": "api-service-api-platform.apps.cluster.example.com", + "path": "/", + "service": "api-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "web-frontend": [ + { + "name": "web-frontend", + "namespace": "web-frontend", + "host": "web-frontend-web-frontend.apps.cluster.example.com", + "path": "/", + "service": "web-frontend", + "port": 3000, + "tls_termination": "edge", + "status": "Admitted", + }, + ], + "order-system": [ + { + "name": "order-service", + "namespace": "order-system", + "host": "order-service-order-system.apps.cluster.example.com", + "path": "/", + "service": "order-service", + "port": 8080, + "tls_termination": "edge", + "status": "Admitted", + "conditions": [ + { + "type": "Admitted", + "status": "True", + "message": "Route admitted but backend returns 503 Service Unavailable", + } + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +EVENTS = { + "api-platform": [ + {"type": "Normal", "reason": "Created", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Created container api-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Started container api-service"}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/api-service-7b8f9d4c5-x2k9m", + "message": "Back-off restarting failed container api-service"}, + ], + "web-frontend": [ + {"type": "Normal", "reason": "Created", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Created container web-frontend"}, + {"type": "Normal", "reason": "Started", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Started container web-frontend"}, + {"type": "Warning", "reason": "OOMKilled", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Container web-frontend was OOMKilled (exit code 137). Memory limit: 64Mi."}, + {"type": "Warning", "reason": "BackOff", "object": "Pod/web-frontend-6c5d8b7a9-p4n2j", + "message": "Back-off restarting failed container web-frontend"}, + ], + "order-system": [ + {"type": "Normal", "reason": "Created", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Created container order-service"}, + {"type": "Normal", "reason": "Started", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Started container order-service"}, + {"type": "Normal", "reason": "Scheduled", "object": "Pod/order-service-5a4b3c2d1-h7j6k", + "message": "Successfully assigned order-system/order-service-5a4b3c2d1-h7j6k to worker-2"}, + {"type": "Warning", "reason": "FailedPipelineRun", "object": "PipelineRun/order-service-deploy-run-7x2k", + "message": "PipelineRun failed at task 'integration-test'. Check step-build and step-test containers for logs."}, + ], +} + + +# --------------------------------------------------------------------------- +# Tekton pipeline data +# --------------------------------------------------------------------------- + +PIPELINE_RUNS = { + "order-system": [ + { + "name": "order-service-deploy-run-7x2k", + "namespace": "order-system", + "pipeline": "order-service-deploy", + "status": "Failed", + "start_time": "2026-02-15T09:15:00Z", + "completion_time": "2026-02-15T09:22:30Z", + "task_runs": [ + { + "name": "order-service-deploy-run-7x2k-build", + "task": "build", + "status": "Succeeded", + "steps": [ + {"name": "step-git-clone", "status": "Completed", "exit_code": 0}, + {"name": "step-build", "status": "Completed", "exit_code": 0}, + {"name": "step-push", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-deploy", + "task": "deploy", + "status": "Succeeded", + "steps": [ + {"name": "step-deploy", "status": "Completed", "exit_code": 0}, + ], + }, + { + "name": "order-service-deploy-run-7x2k-integration-test", + "task": "integration-test", + "status": "Failed", + "steps": [ + {"name": "step-test", "status": "Failed", "exit_code": 1, + "log": ( + "Running integration tests against order-service...\n" + "GET https://order-service-order-system.apps.cluster.example.com/api/health\n" + "Response: 503 Service Unavailable\n" + "FAIL: Health check returned 503, expected 200\n" + "Hint: Service endpoint is unreachable. Verify service routing.\n" + )}, + ], + }, + ], + }, + ], +} + + +# --------------------------------------------------------------------------- +# Application source metadata (for image recommendation) +# --------------------------------------------------------------------------- + +APP_SOURCES = { + "inventory-api": { + "name": "inventory-api", + "language": "Python", + "version": "3.11", + "framework": "Flask", + "entry_point": "app.py", + "dependencies": ["flask==3.0.0", "sqlalchemy==2.0.25", "gunicorn==21.2.0", "psycopg2-binary==2.9.9"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/inventory-api.git", + }, + "customer-portal": { + "name": "customer-portal", + "language": "Node.js", + "version": "20", + "framework": "React (Next.js)", + "entry_point": "server.js", + "dependencies": ["next@14.1.0", "react@18.2.0", "express@4.18.2"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/customer-portal.git", + }, + "payment-processor": { + "name": "payment-processor", + "language": "Java", + "version": "17", + "framework": "Quarkus", + "entry_point": "src/main/java/com/example/Application.java", + "build_tool": "Maven", + "dependencies": ["quarkus-rest", "quarkus-hibernate-orm-panache", "quarkus-jdbc-postgresql"], + "target": "production", + "has_dockerfile": False, + "has_tests": True, + "repo": "https://github.com/example/payment-processor.git", + "notes": "Quarkus application. Consider native compilation for production.", + }, +} + + +# --------------------------------------------------------------------------- +# MCP Tools +# --------------------------------------------------------------------------- + +@mcp.tool +def list_projects() -> dict: + """List all OpenShift projects (namespaces) in the cluster. + + Returns project names, status, and labels. + """ + return {"projects": NAMESPACES, "count": len(NAMESPACES)} + + +@mcp.tool +def get_deployments(namespace: str) -> dict: + """Get deployments in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + deps = DEPLOYMENTS.get(namespace, []) + return {"deployments": deps, "count": len(deps), "namespace": namespace} + + +@mcp.tool +def get_pods(namespace: str) -> dict: + """Get pods in a namespace with their status and container details. + + Args: + namespace: The OpenShift namespace/project name. + """ + pods = PODS.get(namespace, []) + return {"pods": pods, "count": len(pods), "namespace": namespace} + + +@mcp.tool +def pod_logs(pod_name: str, namespace: str, previous: bool = False) -> dict: + """Get logs from a pod. + + Args: + pod_name: Name of the pod. + namespace: The OpenShift namespace/project name. + previous: If True, get logs from the previous terminated container. + """ + logs = POD_LOGS.get(pod_name, f"No logs available for pod {pod_name}") + return {"pod": pod_name, "namespace": namespace, "logs": logs, "previous": previous} + + +@mcp.tool +def get_builds(namespace: str) -> dict: + """Get builds in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + builds = BUILDS.get(namespace, []) + return {"builds": builds, "count": len(builds), "namespace": namespace} + + +@mcp.tool +def get_build_log(build_name: str, namespace: str) -> dict: + """Get the log output from a build. + + Args: + build_name: Name of the build (e.g. 'api-service-1'). + namespace: The OpenShift namespace/project name. + """ + log = BUILD_LOGS.get(build_name, f"No build log found for {build_name}") + return {"build": build_name, "namespace": namespace, "log": log} + + +@mcp.tool +def get_services(namespace: str) -> dict: + """Get services in a namespace with their selectors and ports. + + Args: + namespace: The OpenShift namespace/project name. + """ + svcs = SERVICES.get(namespace, []) + return {"services": svcs, "count": len(svcs), "namespace": namespace} + + +@mcp.tool +def get_routes(namespace: str) -> dict: + """Get routes in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + routes = ROUTES.get(namespace, []) + return {"routes": routes, "count": len(routes), "namespace": namespace} + + +@mcp.tool +def get_events(namespace: str) -> dict: + """Get events in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + events = EVENTS.get(namespace, []) + return {"events": events, "count": len(events), "namespace": namespace} + + +@mcp.tool +def get_pipeline_runs(namespace: str) -> dict: + """Get Tekton PipelineRuns in a namespace. + + Args: + namespace: The OpenShift namespace/project name. + """ + runs = PIPELINE_RUNS.get(namespace, []) + return {"pipeline_runs": runs, "count": len(runs), "namespace": namespace} + + +@mcp.tool +def get_app_source_info(app_name: str) -> dict: + """Get detected source information for an application project. + + Returns language, framework, version, dependencies, and deployment target. + + Args: + app_name: Application name (e.g. 'inventory-api', 'customer-portal', 'payment-processor'). + """ + if app_name in APP_SOURCES: + return APP_SOURCES[app_name] + return {"error": f"Application '{app_name}' not found. Available: {list(APP_SOURCES.keys())}"} + + +@mcp.tool +def list_available_apps() -> dict: + """List all application projects available for analysis. + + Returns names and basic metadata for applications that need + image recommendations or deployment planning. + """ + apps = [] + for name, info in APP_SOURCES.items(): + apps.append({ + "name": name, + "language": info["language"], + "version": info["version"], + "framework": info["framework"], + "target": info["target"], + }) + return {"applications": apps, "count": len(apps)} + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-developer__validate-environment/instruction.md b/evaluation/without_skills/rh-developer__validate-environment/instruction.md new file mode 100644 index 00000000..b9024f98 --- /dev/null +++ b/evaluation/without_skills/rh-developer__validate-environment/instruction.md @@ -0,0 +1,13 @@ +# Environment Validation Task + +You are a Red Hat developer. Before deploying a new application, you need to confirm the OpenShift environment is ready and properly configured. + +## Requirements +- Verify cluster connectivity: confirm you can reach the API server and authenticate successfully +- Check namespace readiness: does the target namespace exist, and do you have permissions to create deployments, services, and routes in it? +- Verify resource availability: are there sufficient CPU and memory quotas remaining for a new deployment? +- Produce a readiness checklist with pass/fail status for each check and an overall go/no-go recommendation + +Document your validation results and readiness assessment in `/root/report.md`. + +Use MCP tools to examine the cluster. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-developer__validate-environment/solution/solve.sh b/evaluation/without_skills/rh-developer__validate-environment/solution/solve.sh new file mode 100644 index 00000000..3cb34892 --- /dev/null +++ b/evaluation/without_skills/rh-developer__validate-environment/solution/solve.sh @@ -0,0 +1,36 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Environment Validation Report + +## Validation Scope: All +(Options: All, OpenShift, RHEL/Containers, Minimal) + +### Tool Availability +| Tool | Status | Version | +|------|--------|---------| +| git | OK | 2.43.0 | +| curl | OK | 8.5.0 | +| jq | OK | 1.7.1 | +| oc | OK | 4.14.0 | +| helm | OK | 3.14.0 | +| podman | OK | 4.9.0 | +| skopeo | OK | 1.14.0 | +| ssh | OK | OpenSSH 9.6 | + +Status indicators: OK (working), MISSING (not in PATH), WARN (optional missing). + +### OpenShift Permissions (oc auth can-i) +| Resource | Action | Status | +|----------|--------|--------| +| deployments | create | OK | +| buildconfigs | create | OK | +| imagestreams | create | OK | + +### Connectivity +- Cluster: Connected (`oc whoami` → admin) +- Podman info: `podman info --format '{{.Host.OS}} {{.Host.Arch}}'` → linux amd64 + +### Ready for +/detect-project, /s2i-build, /deploy, /helm-deploy, /containerize-deploy + +REPORT_EOF diff --git a/evaluation/without_skills/rh-developer__validate-environment/task.toml b/evaluation/without_skills/rh-developer__validate-environment/task.toml new file mode 100644 index 00000000..10df7267 --- /dev/null +++ b/evaluation/without_skills/rh-developer__validate-environment/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-developer__validate-environment" +name = "rh-developer Environment Validation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-developer", "validate-environment", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-developer__validate-environment/tests/llm_judge.py b/evaluation/without_skills/rh-developer__validate-environment/tests/llm_judge.py new file mode 100644 index 00000000..3545ef59 --- /dev/null +++ b/evaluation/without_skills/rh-developer__validate-environment/tests/llm_judge.py @@ -0,0 +1,108 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + { + "id": "skopeo_validation", + "file": "/root/report.md", + "question": "Does the report validate skopeo as a required tool in the environment?", + "reference": "A skilled report checks that skopeo is installed and available, as it is needed for image recommendation workflows. An unskilled report skips skopeo validation." + }, + { + "id": "rbac_permission_checks", + "file": "/root/report.md", + "question": "Does the report verify that the user has permissions to create deployments, buildconfigs, or imagestreams in the target namespace?", + "reference": "A skilled report checks create permissions for deployments, buildconfigs, and imagestreams — either via 'oc auth can-i' commands or via MCP/API queries that verify the same permissions. An unskilled report only checks identity (oc whoami) without verifying specific resource permissions." + }, + { + "id": "structured_validation_report", + "file": "/root/report.md", + "question": "Does the report present environment validation results in a structured pass/fail format with remediation for failures?", + "reference": "A skilled report presents each check as pass/fail with an overall go/no-go assessment and remediation steps. An unskilled report lists tools without structured evaluation." + } +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-developer__validate-environment/tests/test.sh b/evaluation/without_skills/rh-developer__validate-environment/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-developer__validate-environment/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-developer__validate-environment/tests/test_outputs.py b/evaluation/without_skills/rh-developer__validate-environment/tests/test_outputs.py new file mode 100644 index 00000000..8f62b808 --- /dev/null +++ b/evaluation/without_skills/rh-developer__validate-environment/tests/test_outputs.py @@ -0,0 +1,67 @@ +""" +Tests for rh-developer__validate-environment per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: methodology checks that require skill knowledge. +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_environment(self): + content = read_report().lower() + assert any(t in content for t in ["environment", "cluster", "ready", "validation"]), ( + "report should mention environment validation" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 100, "report should have substantial content" + + +class TestSkillDependent: + def test_skopeo_as_required_tool(self): + """Skill teaches skopeo is a required dependency for image recommendation flows. + Without skill, agents skip skopeo in environment validation.""" + c = read_report().lower() + assert "skopeo" in c, ( + "should validate skopeo as a required tool" + ) + + def test_oc_auth_can_i_checks(self): + """Skill teaches oc auth can-i create deployments/buildconfigs/imagestreams + for permission checks. Without skill, agents only check oc whoami.""" + c = read_report().lower() + has_permission_method = ("auth can-i" in c or "can-i" in c or "permission" in c) + has_resource_type = any(t in c for t in [ + "deployment", "buildconfig", "imagestream", "create" + ]) + assert has_permission_method and has_resource_type, ( + "should verify create permissions for deployments/buildconfigs/imagestreams" + ) + + def test_tool_version_checks(self): + """Skill teaches checking version/availability of oc, helm, podman, git.""" + c = read_report().lower() + tools = ["oc", "helm", "podman", "git", "skopeo"] + mentioned = sum(1 for t in tools if t in c) + assert mentioned >= 3, "should validate multiple CLI tools" + + def test_structured_pass_fail(self): + """Skill teaches presenting results as pass/fail per check.""" + c = read_report().lower() + assert any(t in c for t in ["pass", "fail", "missing", "go", "no-go", "available"]) and any(t in c for t in [ + "tool", "check", "oc", "helm", "result" + ]), "should provide structured pass/fail validation report" diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/Dockerfile b/evaluation/without_skills/rh-sre__cve-impact/environment/Dockerfile new file mode 100644 index 00000000..5a2cfdee --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/Dockerfile @@ -0,0 +1,43 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..6c122770 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- Pass Rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/without_skills/rh-sre__cve-impact/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..fe5d549c --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-sre__cve-impact/instruction.md b/evaluation/without_skills/rh-sre__cve-impact/instruction.md new file mode 100644 index 00000000..00b38e1d --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/instruction.md @@ -0,0 +1,14 @@ +# CVE Impact Analysis Task + +You are a Red Hat SRE. A critical vulnerability has been announced, and management needs to know how many of your systems are affected before deciding on emergency patching. + +## Requirements +- Query your fleet to identify all systems affected by the CVE +- Break down the impact by environment (production vs staging vs development) and by RHEL version +- Report total affected system count, and flag any high-criticality systems (e.g., customer-facing, compliance-regulated) +- If results span multiple pages, ensure you capture the complete picture +- Provide a risk summary: severity, exposure scope, and recommended urgency level + +Document your methodology, impact analysis, and risk assessment in `/root/report.md`. + +Use MCP tools to query vulnerability data. If reference documentation or skills are available in this environment, consult them before beginning work. Complete the entire analysis autonomously — do not stop to ask for user confirmation or input at any checkpoint. Use reasonable defaults (e.g., fetch all available data) and proceed through every step to produce the final report. diff --git a/evaluation/without_skills/rh-sre__cve-impact/solution/solve.sh b/evaluation/without_skills/rh-sre__cve-impact/solution/solve.sh new file mode 100644 index 00000000..fbbfb891 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/solution/solve.sh @@ -0,0 +1,15 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# CVE Impact Analysis + +## CVE-2024-12345 +- Severity: Critical (CVSS 9.8) +- Affected systems: 6 +- Patched: 2 +- Vulnerable: 4 + +## Pagination +Used limit=100 per page, system_uuid for system-level queries. First page only often returns 0 remediatable CVEs—systems may have 1700+ CVEs (~18 API calls). Recommend "all pages" for remediatable queries. + +## Data parsed using cve-response-parser.py +REPORT_EOF diff --git a/evaluation/without_skills/rh-sre__cve-impact/task.toml b/evaluation/without_skills/rh-sre__cve-impact/task.toml new file mode 100644 index 00000000..1ef53278 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__cve-impact" +name = "rh-sre CVE Impact Analysis Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "cve-impact", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-sre__cve-impact/tests/llm_judge.py b/evaluation/without_skills/rh-sre__cve-impact/tests/llm_judge.py new file mode 100644 index 00000000..91bf4254 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/tests/llm_judge.py @@ -0,0 +1,94 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "comprehensive_cve_coverage", "file": "/root/report.md", "question": "Does the report analyze multiple CVEs from the fleet inventory (at least 3 distinct CVE IDs) with per-CVE detail, rather than focusing only on a single CVE?", "reference": "A skilled report comprehensively assesses all CVEs affecting the fleet (CVE-2024-12345, 54321, 98765, 11111, 22222) with individual analysis. An unskilled report often focuses only on the primary CVE-2024-12345."}, + {"id": "multi_environment_breakdown", "file": "/root/report.md", "question": "Does the report break down CVE impact across at least 3 distinct environment tiers (e.g., production, staging, development, QA, legacy) with per-environment system counts or status?", "reference": "A skilled report categorizes affected systems by environment tier (production, staging, dev, QA, legacy) with counts per environment. An unskilled report provides aggregate totals without environment-level detail."}, + {"id": "risk_assessment", "file": "/root/report.md", "question": "Does the report provide a risk assessment that considers multiple factors such as CVSS score, affected system count, and environment criticality?", "reference": "A skilled report includes a multi-factor risk assessment. An unskilled report gives generic severity ratings without combining multiple factors."}, + {"id": "operational_priority_ranking", "file": "/root/report.md", "question": "Does the report assign explicit operational priority tiers (like P0/P1/P2 or Priority 1/2/3) to CVEs with associated SLA timeframes (e.g., within 24 hours, within 7 days), going beyond just severity labels?", "reference": "A skilled report maps CVE severity to operational priority tiers (P0=immediate/24h, P1=7 days, P2=30 days) with concrete remediation deadlines. An unskilled report uses only vendor severity labels (Critical/Important/Moderate) without operational priority mapping."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-sre__cve-impact/tests/test.sh b/evaluation/without_skills/rh-sre__cve-impact/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-sre__cve-impact/tests/test_outputs.py b/evaluation/without_skills/rh-sre__cve-impact/tests/test_outputs.py new file mode 100644 index 00000000..d5edc006 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-impact/tests/test_outputs.py @@ -0,0 +1,92 @@ +""" +Tests for rh-sre__cve-impact per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_cve(self): + content = read_report().lower() + assert "cve" in content, "report should mention CVEs" + + def test_mentions_impact(self): + content = read_report().lower() + assert any(t in content for t in ["impact", "affected", "system", "fleet"]), ( + "report should discuss impact" + ) + + +class TestSkillDependent: + def test_full_cve_coverage(self): + """Skill teaches comprehensive fleet-wide CVE assessment across all CVEs. + Without skill, agents often focus only on the primary CVE.""" + c = read_report() + cve_ids = ["CVE-2024-12345", "CVE-2024-54321", "CVE-2024-98765", + "CVE-2024-11111", "CVE-2024-22222"] + found = sum(1 for cve in cve_ids if cve in c) + assert found >= 3, ( + f"should analyze multiple CVEs from fleet (found {found}/5); " + "skill teaches comprehensive multi-CVE assessment" + ) + + def test_prioritized_remediation_order(self): + """Skill teaches prioritizing CVEs with explicit priority ranking + (P0/P1/P2 or similar ordered tiers). Without skill, agents list by + severity without operational priority ranking.""" + c = read_report() + has_priority = any(t in c for t in [ + "P0", "P1", "P2", "Priority 0", "Priority 1", "Priority 2", + ]) or any(t in c.lower() for t in [ + "priority order", "remediation priority", "remediation order", + "triage priority", "priority ranking", "prioritized order", + ]) + assert has_priority, ( + "should assign explicit priority ranking (P0/P1/P2 or equivalent) to CVEs" + ) + + def test_multi_environment_breakdown(self): + """Skill teaches breaking down impact by environment (prod/staging/dev/QA/legacy). + Without skill, agents report aggregate counts without per-environment detail.""" + c = read_report().lower() + envs = ["production", "staging", "development", "qa", "legacy", "dev"] + found = sum(1 for e in envs if e in c) + assert found >= 3, ( + f"should break down impact across multiple environments (found {found}); " + "skill teaches per-environment categorization" + ) + + def test_risk_assessment_structure(self): + """Skill: Risk assessment with CVSS, affected count, environment criticality.""" + c = read_report().lower() + has_risk = any(t in c for t in ["risk", "priority", "urgency", "criticality"]) + has_factors = any(t in c for t in ["cvss", "affect", "severity", "count", "staging", "criticality"]) + assert has_risk and has_factors, ( + "should provide risk assessment with multiple factors (skill: Step 5)" + ) + + def test_classification_methodology(self): + """Skill teaches using classification criteria/methodology for CVE interpretation. + Without skill, agents classify severity ad-hoc.""" + c = read_report().lower() + assert any(t in c for t in [ + "classification", "methodology", "criteria", + "vulnerability-logic", "cvss-scoring", + "scoring framework", "risk framework", + ]) or ("consult" in c and "reference" in c), ( + "should reference classification methodology for CVE interpretation" + ) diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/Dockerfile b/evaluation/without_skills/rh-sre__cve-validation/environment/Dockerfile new file mode 100644 index 00000000..5a2cfdee --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/Dockerfile @@ -0,0 +1,43 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..6c122770 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- Pass Rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/without_skills/rh-sre__cve-validation/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..fe5d549c --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-sre__cve-validation/instruction.md b/evaluation/without_skills/rh-sre__cve-validation/instruction.md new file mode 100644 index 00000000..27325f5c --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/instruction.md @@ -0,0 +1,12 @@ +# CVE Validation Task + +You are a Red Hat SRE. Validate a set of CVEs to determine which are real, applicable, and remediable on your fleet. + +## Requirements +- Validate CVE identifiers and severity +- Determine which CVEs have available fixes or advisories +- Classify CVEs by remediation status + +Use MCP tools to query vulnerability data. Document your methodology, validation results, and classification in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. Complete the entire analysis autonomously — do not stop after preliminary steps like MCP validation. Proceed through CVE querying, validation, classification, and report generation without waiting for user input. diff --git a/evaluation/without_skills/rh-sre__cve-validation/solution/solve.sh b/evaluation/without_skills/rh-sre__cve-validation/solution/solve.sh new file mode 100644 index 00000000..f4350508 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/solution/solve.sh @@ -0,0 +1,14 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# CVE Validation Report + +## CVE-2024-12345 +- Format: Valid (^CVE-\d{4}-\d{4,7}$) +- Advisory available: Yes (advisory_available, advisories_list) +- Do NOT use rules[] for remediation decision +- Remediation status: automated_remediation_available +- Validation status: valid +- Severity: Critical (Red Hat) +- Affected packages: httpd 2.4.37-1.el8 → 2.4.37-2.el8 +- Priority: P0 (24 hours) +REPORT_EOF diff --git a/evaluation/without_skills/rh-sre__cve-validation/task.toml b/evaluation/without_skills/rh-sre__cve-validation/task.toml new file mode 100644 index 00000000..98d08db5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__cve-validation" +name = "rh-sre CVE Validation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "cve-validation", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-sre__cve-validation/tests/llm_judge.py b/evaluation/without_skills/rh-sre__cve-validation/tests/llm_judge.py new file mode 100644 index 00000000..f0df9c9c --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "advisory_not_rules", "file": "/root/report.md", "question": "Does the report use advisory_available or advisories_list (not rules[]) to determine remediation availability?", "reference": "A skilled report checks advisory_available/advisories_list for remediation status. An unskilled report incorrectly uses rules[] which is the Advisor engine."}, + {"id": "format_validation", "file": "/root/report.md", "question": "Does the report validate CVE format and accept 4-7 digit sequence numbers?", "reference": "A skilled report accepts CVE IDs with 4-7 digit sequences. An unskilled report may reject valid CVEs with non-5-digit sequences."}, + {"id": "structured_output", "file": "/root/report.md", "question": "Does the report output validation_status and remediation availability in a structured format?", "reference": "A skilled report presents clear validation_status and automated_remediation_available fields."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-sre__cve-validation/tests/test.sh b/evaluation/without_skills/rh-sre__cve-validation/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-sre__cve-validation/tests/test_outputs.py b/evaluation/without_skills/rh-sre__cve-validation/tests/test_outputs.py new file mode 100644 index 00000000..21b9262c --- /dev/null +++ b/evaluation/without_skills/rh-sre__cve-validation/tests/test_outputs.py @@ -0,0 +1,81 @@ +""" +Tests for rh-sre__cve-validation per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_cve(self): + content = read_report().lower() + assert "cve" in content, "report should mention CVEs" + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_format_then_api_validation(self): + """Skill: Validate format (regex) first; if valid, ALWAYS call get_cve—do not reject on year/sequence.""" + c = read_report().lower() + has_format = any(t in c for t in ["regex", "pattern", "cve-", "cve-format", "year/sequence"]) + has_api_call = any(t in c for t in ["get_cve", "call", "api", "retrieve", "fetch"]) + assert has_format or has_api_call, ( + "should validate format then call get_cve (skill: do NOT reject on year/sequence before API)" + ) + + def test_advisory_available_not_rules(self): + """Skill teaches remediation determined by advisory_available/advisories_list/remediation field, NOT by rules[].""" + c = read_report().lower() + assert any(t in c for t in ["advisory_available", "advisories_list"]), ( + "should use advisory_available or advisories_list for remediation (skill: rules[] is wrong)" + ) + + def test_cve_regex_acceptance(self): + """Skill teaches CVE sequence is 4-7 digits (not always 5).""" + c = read_report().lower() + assert any(t in c for t in ["4,7", "4-7", "4-7 digit", "4 to 7", "regex"]), ( + "should accept CVE sequence 4-7 digits (skill: not always 5 digits)" + ) + + def test_validation_status_output(self): + """Skill: Return validation_status and remediation_status.automated_remediation_available.""" + c = read_report().lower() + has_status = any(t in c for t in ["validation_status", "valid", "invalid", "not_remediable"]) + has_remediation_flag = any(t in c for t in ["automated_remediation", "automated", "manual", "remediat"]) + assert has_status and has_remediation_flag, ( + "should output validation_status and remediation availability" + ) + + def test_affected_packages_with_versions(self): + """Skill: Identify affected packages with current and fixed versions.""" + c = read_report().lower() + has_packages = any(t in c for t in ["package", "affected", "component"]) + has_versions = any(t in c for t in ["version", "fixed", "current", "el8", "el9"]) + assert has_packages and has_versions, ( + "should identify packages with version info (skill: for playbook-generator)" + ) + + def test_remediation_field_value(self): + """Docs teach remediation==2 means automated remediation available. + Without docs, agents don't know the numeric remediation field semantics.""" + c = read_report().lower() + assert any(t in c for t in [ + "remediation==2", "remediation=2", "remediation field", "remediation value", + "automated remediation", + ]), "should interpret remediation field value (2=automated)" diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/Dockerfile b/evaluation/without_skills/rh-sre__execution-summary/environment/Dockerfile new file mode 100644 index 00000000..5a2cfdee --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/Dockerfile @@ -0,0 +1,43 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..6c122770 --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- Pass Rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/without_skills/rh-sre__execution-summary/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..fe5d549c --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-sre__execution-summary/instruction.md b/evaluation/without_skills/rh-sre__execution-summary/instruction.md new file mode 100644 index 00000000..5521bb63 --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/instruction.md @@ -0,0 +1,15 @@ +# Execution Summary Task + +You are a Red Hat SRE. Your team just completed an emergency remediation of a critical CVE across your managed fleet. Management needs a structured post-incident execution summary. + +## Scenario +A critical kernel vulnerability was announced. Your team used automation tools to identify affected systems, generate remediation playbooks, execute patching, and verify the fix. Now you need to document what was done. + +## Requirements +- Use MCP tools to query the current state of the fleet, identify which systems were affected, and gather evidence of remediation actions taken +- Produce an execution summary that includes: what was done, which tools and automation were used, the sequence of steps, results and verification outcomes, and any remaining gaps +- Structure the summary so it can be reviewed by management and used for future incident response improvement + +Document the full execution summary, including your methodology and the tools used, in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-sre__execution-summary/solution/solve.sh b/evaluation/without_skills/rh-sre__execution-summary/solution/solve.sh new file mode 100644 index 00000000..68891309 --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/solution/solve.sh @@ -0,0 +1,13 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Execution Summary + +**** EXECUTION SUMMARY START **** +Agents: None +Skills: rh-sre:fleet-inventory,rh-sre:cve-impact +Tools: lightspeed-mcp:get_host_details,lightspeed-mcp:get_cves +Docs: docs/references/cvss-scoring.md,docs/insights/vulnerability-logic.md +**** EXECUTION SUMMARY END **** + +This summary shows all agents, skills, tools, and documentation used during the workflow. +REPORT_EOF diff --git a/evaluation/without_skills/rh-sre__execution-summary/task.toml b/evaluation/without_skills/rh-sre__execution-summary/task.toml new file mode 100644 index 00000000..a983e99f --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__execution-summary" +name = "rh-sre Execution Summary Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "execution-summary", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-sre__execution-summary/tests/llm_judge.py b/evaluation/without_skills/rh-sre__execution-summary/tests/llm_judge.py new file mode 100644 index 00000000..c426d912 --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "tool_purpose_attribution", "file": "/root/report.md", "question": "Does the report pair each tool or MCP call with an explicit purpose statement explaining why it was invoked, rather than just listing tools used or describing actions narratively?", "reference": "A skilled report explicitly pairs each tool (e.g., get_host_details, get_cves) with a 'Purpose:' or 'used to' statement. An unskilled report lists tools in a table or mentions them inline without structured per-tool purpose mapping."}, + {"id": "tool_purpose_categorization", "file": "/root/report.md", "question": "Does the report organize MCP tools into named functional categories (e.g., 'Vulnerability Assessment Tools', 'Fleet Management Tools', 'Remediation Tools') rather than listing all tools in a single flat list?", "reference": "A skilled report groups tools by their functional purpose into labeled sub-sections (e.g., 'Assessment Tools: get_cves, get_cve', 'Fleet Tools: get_host_details'). An unskilled report lists all tools sequentially without categorizing them by function."}, + {"id": "methodology_phases", "file": "/root/report.md", "question": "Does the report organize the execution into at least two named methodology phases (e.g., 'Data Collection', 'Evidence Gathering', 'Triage', 'Verification') as distinct sections or headings?", "reference": "A skilled report structures execution into named phases as headings or sections. An unskilled report presents a flat narrative or simple numbered list without phase-level organization."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-sre__execution-summary/tests/test.sh b/evaluation/without_skills/rh-sre__execution-summary/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-sre__execution-summary/tests/test_outputs.py b/evaluation/without_skills/rh-sre__execution-summary/tests/test_outputs.py new file mode 100644 index 00000000..6cd1228a --- /dev/null +++ b/evaluation/without_skills/rh-sre__execution-summary/tests/test_outputs.py @@ -0,0 +1,55 @@ +""" +Tests for rh-sre__execution-summary per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: skill-specific patterns (not generic report quality). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['summary', 'execution', 'remediation']), ( + "report should mention execution summary or remediation" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 300, "execution summary should be substantial" + + +class TestSkillDependent: + def test_methodology_phases(self): + """Structuring execution into methodology phases + (data collection, evidence gathering, etc.).""" + c = read_report().lower() + phase_terms = [ + "data collection", "evidence gathering", "discovery", + "triage", "assessment", "verification", + "phase 1", "phase 2", "step 1", "step 2", + ] + found = sum(1 for t in phase_terms if t in c) + assert found >= 2, ( + f"should organize execution into methodology phases (found {found})" + ) + + def test_docs_from_consulted(self): + """Extract docs from 'I consulted' statements; path from docs/ or skills/ onwards.""" + c = read_report().lower() + has_docs = any(t in c for t in ["docs/", "skills/", "consult", "documentation"]) + assert has_docs, ( + "should list documentation consulted" + ) diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/Dockerfile b/evaluation/without_skills/rh-sre__fleet-inventory/environment/Dockerfile new file mode 100644 index 00000000..5a2cfdee --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/Dockerfile @@ -0,0 +1,43 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..6c122770 --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- Pass Rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/without_skills/rh-sre__fleet-inventory/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..fe5d549c --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/instruction.md b/evaluation/without_skills/rh-sre__fleet-inventory/instruction.md new file mode 100644 index 00000000..3074bb9c --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/instruction.md @@ -0,0 +1,17 @@ +# Fleet Inventory Task + +You are a Red Hat SRE. Your manager has asked for a current snapshot of all RHEL systems in your managed fleet ahead of an upcoming compliance audit. + +## Scenario +The compliance team needs to know exactly what systems you manage, their RHEL versions, patch levels, and any outstanding vulnerability exposure. They need this by end of day. + +## Requirements +- Query the fleet to enumerate all managed RHEL systems +- For each system, report: hostname, RHEL version, last check-in date, and patch status +- Identify which systems have outstanding CVEs, grouped by severity +- Flag any systems that are stale (not checking in) or running unsupported RHEL versions +- Summarize the fleet's overall health and compliance readiness + +Document your methodology, findings, and fleet assessment in `/root/report.md`. + +Use MCP tools to query data. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/solution/solve.sh b/evaluation/without_skills/rh-sre__fleet-inventory/solution/solve.sh new file mode 100644 index 00000000..dc994408 --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/solution/solve.sh @@ -0,0 +1,25 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Fleet Inventory Report + +## Systems Summary +| Hostname | RHEL | Environment | Status | Last Seen | +|----------|------|-------------|--------|-----------| +| web-01 | 9.3 | Production | Active | 2024-01-15 | +| db-01 | 9.3 | Production | Active | 2024-01-15 | +| dev-01 | 8.9 | Development | Stale | 2024-01-01 | + +## Data Source +Queried via `get_host_details` with pagination. Key fields: rhel_version, tags, stale, last_seen. + +## CVE-Affected Systems +Use `get_cve_systems` with cve_id (CVE-YYYY-NNNNN). Check remediation_available flag. + +## Status Interpretation +- **Vulnerable**: CVE affects system, patch not applied → suggest /remediation +- **Patched**: Previously affected, now remediated → no action +- **Not Affected**: Exclude from affected count + +## Next Steps +For CVE remediation, transition to /remediation skill. +REPORT_EOF diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/task.toml b/evaluation/without_skills/rh-sre__fleet-inventory/task.toml new file mode 100644 index 00000000..cff6fe66 --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__fleet-inventory" +name = "rh-sre Fleet Inventory Query Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "fleet-inventory", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/tests/llm_judge.py b/evaluation/without_skills/rh-sre__fleet-inventory/tests/llm_judge.py new file mode 100644 index 00000000..977611c9 --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/tests/llm_judge.py @@ -0,0 +1,92 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "system_id_for_remediation", "file": "/root/report.md", "question": "Does the report track individual system identifiers (system_uuid, system_id, or host UUID) and link them to specific remediation follow-up actions, rather than just listing hostnames?", "reference": "A skilled report captures system UUIDs or identifiers to enable programmatic remediation API calls. An unskilled report lists hostnames or display names without machine-usable identifiers for follow-up."}, + {"id": "classification_methodology", "file": "/root/report.md", "question": "Does the report reference a classification methodology, classification criteria, or vulnerability classification framework for interpreting CVE status, rather than using ad-hoc severity labeling?", "reference": "A skilled report consults or references CVE classification criteria or methodology documentation before interpreting vulnerability data. An unskilled report classifies CVEs based on general knowledge without referencing established criteria."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/tests/test.sh b/evaluation/without_skills/rh-sre__fleet-inventory/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/tests/test_outputs.py b/evaluation/without_skills/rh-sre__fleet-inventory/tests/test_outputs.py new file mode 100644 index 00000000..f8c232d0 --- /dev/null +++ b/evaluation/without_skills/rh-sre__fleet-inventory/tests/test_outputs.py @@ -0,0 +1,67 @@ +""" +Tests for rh-sre__fleet-inventory per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['system', 'host', 'fleet', 'inventory']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_system_identifier_tracking(self): + """Skill teaches tracking system identifiers for follow-up actions. + Without skill, agents list systems without identifiers for remediation.""" + c = read_report().lower() + assert any(t in c for t in [ + "system id", "system_id", "system_uuid", "uuid", "identifier", + ]) and any(t in c for t in [ + "remediat", "follow-up", "subsequent", "action", "track", + ]), ( + "should track system identifiers for follow-up remediation actions" + ) + + def test_remediation_transition_offer(self): + """Skill: Offer transition to a remediation workflow for CVE remediation.""" + c = read_report().lower() + assert any(t in c for t in [ + "next step", "remediate", "playbook", + "remediation workflow", "remediation action", + ]), "should offer next steps for remediation" + + def test_classification_criteria_reference(self): + """Skill/docs teach consulting classification criteria or reference + documentation before interpreting vulnerability data. Without skill, + agents classify CVEs based on general knowledge alone.""" + c = read_report().lower() + assert any(t in c for t in [ + "classification criteria", "classification methodology", + "vulnerability classification", "cve classification", + ]) or ( + "classification" in c and any(t in c for t in [ + "consult", "reference", "methodology", "criteria", + ]) + ), "should reference CVE classification criteria or methodology" diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/Dockerfile b/evaluation/without_skills/rh-sre__job-template-creator/environment/Dockerfile new file mode 100644 index 00000000..51ce02e5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/Dockerfile @@ -0,0 +1,47 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + }, \ + "aap-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-aap-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..6c122770 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- Pass Rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/mcp-servers/mock-aap-mcp.py b/evaluation/without_skills/rh-sre__job-template-creator/environment/mcp-servers/mock-aap-mcp.py new file mode 100644 index 00000000..d8ae4fd5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/mcp-servers/mock-aap-mcp.py @@ -0,0 +1,1048 @@ +#!/usr/bin/env python3 +""" +Mock AAP (Ansible Automation Platform) MCP Server + +Simulates the AAP MCP gateway for per-skill evaluation tasks. Implements +the full set of tools used by rh-sre skills: + - job_templates_list / job_templates_retrieve + - projects_list + - job_templates_launch_retrieve + - jobs_retrieve / jobs_stdout_retrieve + - jobs_job_events_list / jobs_job_host_summaries_list + - jobs_relaunch_retrieve + - inventories_list / hosts_list + +Data mirrors a realistic AAP deployment: + - 6 job templates (3 remediation, 1 compliance, 1 patching, 1 reporting) + - 3 projects (remediation, compliance, reporting) + - 3 inventories (production 30 hosts, staging 15 hosts, all-managed 63 hosts) + - 12 recent jobs with varied statuses + +Follows the same mock-server pattern as mock-lightspeed-mcp.py. +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +mcp = FastMCP("aap-mcp") + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +def _ts(delta: timedelta) -> str: + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +# --------------------------------------------------------------------------- +# Mock data: Projects +# --------------------------------------------------------------------------- + +MOCK_PROJECTS = [ + { + "id": 6, + "type": "project", + "name": "Remediation Playbooks", + "description": "CVE and security remediation playbooks managed via Git", + "scm_type": "git", + "scm_url": "https://github.com/org/remediation-playbooks.git", + "scm_branch": "main", + "scm_revision": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + "status": "successful", + "last_job_run": _ts(timedelta(hours=2)), + "last_update_failed": False, + "created": _ts(timedelta(days=90)), + "modified": _ts(timedelta(hours=2)), + }, + { + "id": 7, + "type": "project", + "name": "Compliance Checks", + "description": "STIG and CIS compliance scanning playbooks", + "scm_type": "git", + "scm_url": "https://github.com/org/compliance-playbooks.git", + "scm_branch": "main", + "scm_revision": "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3", + "status": "successful", + "last_job_run": _ts(timedelta(days=1)), + "last_update_failed": False, + "created": _ts(timedelta(days=120)), + "modified": _ts(timedelta(days=1)), + }, + { + "id": 8, + "type": "project", + "name": "Fleet Reporting", + "description": "System inventory and health reporting playbooks", + "scm_type": "git", + "scm_url": "https://github.com/org/fleet-reports.git", + "scm_branch": "main", + "scm_revision": "c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "status": "successful", + "last_job_run": _ts(timedelta(days=3)), + "last_update_failed": False, + "created": _ts(timedelta(days=180)), + "modified": _ts(timedelta(days=3)), + }, +] + +# --------------------------------------------------------------------------- +# Mock data: Inventories & Hosts +# --------------------------------------------------------------------------- + +MOCK_INVENTORIES = [ + { + "id": 1, + "type": "inventory", + "name": "Production Systems", + "description": "All production RHEL systems across data centers", + "total_hosts": 30, + "has_active_failures": False, + "hosts_with_active_failures": 0, + "total_groups": 5, + "groups_with_active_failures": 0, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=365)), + "modified": _ts(timedelta(days=1)), + }, + { + "id": 2, + "type": "inventory", + "name": "Staging Systems", + "description": "Pre-production staging environment", + "total_hosts": 15, + "has_active_failures": False, + "hosts_with_active_failures": 0, + "total_groups": 3, + "groups_with_active_failures": 0, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=300)), + "modified": _ts(timedelta(days=7)), + }, + { + "id": 3, + "type": "inventory", + "name": "All Managed Systems", + "description": "Complete fleet: production, staging, development, QA, legacy", + "total_hosts": 63, + "has_active_failures": True, + "hosts_with_active_failures": 2, + "total_groups": 8, + "groups_with_active_failures": 1, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=365)), + "modified": _ts(timedelta(hours=6)), + }, +] + + +def _generate_hosts(inventory_id: int) -> list[dict]: + """Generate realistic hosts for an inventory.""" + hosts: list[dict] = [] + if inventory_id == 1: + roles = ["web", "db", "app", "lb", "monitoring", "cache"] + for i, role in enumerate(roles): + for j in range(5 if role in ("web", "app") else 4 if role == "db" else 3 if role == "monitoring" else 2): + hosts.append({ + "id": len(hosts) + 1, + "type": "host", + "name": f"{role}-{j+1:02d}.prod.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "production", "role": "{role}"}}', + }) + if len(hosts) >= 30: + break + if len(hosts) >= 30: + break + elif inventory_id == 2: + for i in range(15): + role = ["web", "db", "app"][i % 3] + hosts.append({ + "id": 100 + i, + "type": "host", + "name": f"{role}-{i+1:02d}.staging.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "staging", "role": "{role}"}}', + }) + elif inventory_id == 3: + for i in range(30): + hosts.append({ + "id": 200 + i, + "type": "host", + "name": f"host-{i+1:02d}.prod.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": i in (45, 58), + "variables": f'{{"rhel_version": "9.3", "environment": "production"}}', + }) + for i in range(15): + hosts.append({ + "id": 230 + i, + "type": "host", + "name": f"host-{i+1:02d}.staging.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "staging"}}', + }) + for i in range(10): + hosts.append({ + "id": 245 + i, + "type": "host", + "name": f"dev-{i+1:02d}.dev.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "8.9", "environment": "development"}}', + }) + for i in range(5): + hosts.append({ + "id": 255 + i, + "type": "host", + "name": f"qa-{i+1:02d}.qa.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.2", "environment": "qa"}}', + }) + for i in range(3): + hosts.append({ + "id": 260 + i, + "type": "host", + "name": f"legacy-{i+1:02d}.corp.example.com", + "inventory": inventory_id, + "enabled": i < 2, + "has_active_failures": i == 2, + "variables": f'{{"rhel_version": "7.9", "environment": "legacy"}}', + }) + return hosts + + +# --------------------------------------------------------------------------- +# Mock data: Job Templates +# --------------------------------------------------------------------------- + +MOCK_JOB_TEMPLATES = [ + { + "id": 10, + "type": "job_template", + "name": "CVE Remediation - Kernel Update", + "description": "Kernel update with boom snapshot for rollback safety", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": True, + "job_type": "check", + "verbosity": 1, + "timeout": 3600, + "forks": 5, + "status": "successful", + "last_job_run": _ts(timedelta(hours=4)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1001, "status": "successful", "finished": _ts(timedelta(hours=4))}, + }, + "created": _ts(timedelta(days=60)), + "modified": _ts(timedelta(days=2)), + }, + { + "id": 11, + "type": "job_template", + "name": "CVE Remediation - Package Update", + "description": "General package update for CVE remediation with needs-restarting check", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-package-update.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": False, + "job_type": "check", + "verbosity": 1, + "timeout": 1800, + "forks": 10, + "status": "successful", + "last_job_run": _ts(timedelta(hours=12)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1005, "status": "successful", "finished": _ts(timedelta(hours=12))}, + }, + "created": _ts(timedelta(days=45)), + "modified": _ts(timedelta(days=5)), + }, + { + "id": 12, + "type": "job_template", + "name": "CVE Remediation - Generic", + "description": "Generic CVE remediation template for ad-hoc patches", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-remediation.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": True, + "job_type": "check", + "verbosity": 1, + "timeout": 3600, + "forks": 5, + "status": "never updated", + "last_job_run": None, + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + }, + "created": _ts(timedelta(days=30)), + "modified": _ts(timedelta(days=30)), + }, + { + "id": 20, + "type": "job_template", + "name": "Compliance Check - STIG", + "description": "Run STIG compliance scan across fleet", + "inventory": 3, + "project": 7, + "playbook": "playbooks/compliance/check-all.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": False, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 7200, + "forks": 20, + "status": "successful", + "last_job_run": _ts(timedelta(days=1)), + "summary_fields": { + "project": {"id": 7, "name": "Compliance Checks", "status": "successful"}, + "inventory": {"id": 3, "name": "All Managed Systems", "total_hosts": 63}, + "credentials": [ + {"id": 2, "name": "compliance-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1010, "status": "successful", "finished": _ts(timedelta(days=1))}, + }, + "created": _ts(timedelta(days=180)), + "modified": _ts(timedelta(days=14)), + }, + { + "id": 25, + "type": "job_template", + "name": "Emergency Patching", + "description": "Emergency patch application — NO become enabled (misconfigured)", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/emergency-patch.yml", + "become_enabled": False, + "ask_job_type_on_launch": False, + "ask_variables_on_launch": False, + "ask_limit_on_launch": False, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 600, + "forks": 25, + "status": "failed", + "last_job_run": _ts(timedelta(days=7)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1020, "status": "failed", "finished": _ts(timedelta(days=7))}, + }, + "created": _ts(timedelta(days=200)), + "modified": _ts(timedelta(days=200)), + }, + { + "id": 30, + "type": "job_template", + "name": "Fleet Health Report", + "description": "Generate fleet health and inventory report", + "inventory": 3, + "project": 8, + "playbook": "playbooks/reporting/fleet-health.yml", + "become_enabled": False, + "ask_job_type_on_launch": False, + "ask_variables_on_launch": True, + "ask_limit_on_launch": False, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 1800, + "forks": 30, + "status": "successful", + "last_job_run": _ts(timedelta(hours=6)), + "summary_fields": { + "project": {"id": 8, "name": "Fleet Reporting", "status": "successful"}, + "inventory": {"id": 3, "name": "All Managed Systems", "total_hosts": 63}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1025, "status": "successful", "finished": _ts(timedelta(hours=6))}, + }, + "created": _ts(timedelta(days=120)), + "modified": _ts(timedelta(days=14)), + }, +] + +# --------------------------------------------------------------------------- +# Mock data: Jobs (recent runs) +# --------------------------------------------------------------------------- + +PROD_HOSTS = [ + "web-01.prod.example.com", + "web-02.prod.example.com", + "db-01.prod.example.com", + "db-02.prod.example.com", + "app-01.prod.example.com", + "app-02.prod.example.com", +] + +MOCK_JOBS = [ + { + "id": 1001, + "type": "job", + "name": "CVE Remediation - Kernel Update", + "job_type": "check", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=4, minutes=30)), + "finished": _ts(timedelta(hours=4)), + "elapsed": 1800.0, + "job_template": 10, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "limit": "web-01.prod.example.com,web-02.prod.example.com,db-01.prod.example.com", + "extra_vars": '{"target_cve": "CVE-2024-12345", "remediation_mode": "automated", "verify_after": true}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 10, "name": "CVE Remediation - Kernel Update"}, + }, + }, + { + "id": 1002, + "type": "job", + "name": "CVE Remediation - Kernel Update", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=3, minutes=45)), + "finished": _ts(timedelta(hours=3)), + "elapsed": 2700.0, + "job_template": 10, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "limit": "web-01.prod.example.com,web-02.prod.example.com,db-01.prod.example.com", + "extra_vars": '{"target_cve": "CVE-2024-12345", "remediation_mode": "automated", "verify_after": true}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 10, "name": "CVE Remediation - Kernel Update"}, + }, + }, + { + "id": 1005, + "type": "job", + "name": "CVE Remediation - Package Update", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=12, minutes=20)), + "finished": _ts(timedelta(hours=12)), + "elapsed": 1200.0, + "job_template": 11, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-package-update.yml", + "limit": "", + "extra_vars": '{"target_cve": "CVE-2024-54321"}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 11, "name": "CVE Remediation - Package Update"}, + }, + }, + { + "id": 1010, + "type": "job", + "name": "Compliance Check - STIG", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(days=1, hours=2)), + "finished": _ts(timedelta(days=1)), + "elapsed": 7200.0, + "job_template": 20, + "inventory": 3, + "project": 7, + "playbook": "playbooks/compliance/check-all.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "scheduled", + "summary_fields": { + "job_template": {"id": 20, "name": "Compliance Check - STIG"}, + }, + }, + { + "id": 1020, + "type": "job", + "name": "Emergency Patching", + "job_type": "run", + "status": "failed", + "failed": True, + "started": _ts(timedelta(days=7, hours=1)), + "finished": _ts(timedelta(days=7)), + "elapsed": 3600.0, + "job_template": 25, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/emergency-patch.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 25, "name": "Emergency Patching"}, + }, + }, + { + "id": 1025, + "type": "job", + "name": "Fleet Health Report", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=6, minutes=30)), + "finished": _ts(timedelta(hours=6)), + "elapsed": 1800.0, + "job_template": 30, + "inventory": 3, + "project": 8, + "playbook": "playbooks/reporting/fleet-health.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "scheduled", + "summary_fields": { + "job_template": {"id": 30, "name": "Fleet Health Report"}, + }, + }, +] + +_next_job_id = 2000 + + +# --------------------------------------------------------------------------- +# Mock stdout generators +# --------------------------------------------------------------------------- + +def _generate_stdout(job: dict) -> str: + """Generate realistic Ansible playbook stdout for a job.""" + playbook_name = job.get("name", "Unknown") + job_type = job.get("job_type", "run") + status = job.get("status", "successful") + limit = job.get("limit", "") + hosts = limit.split(",") if limit else PROD_HOSTS[:3] + hosts = [h.strip() for h in hosts if h.strip()] + extra_vars = job.get("extra_vars", "{}") + mode = " (CHECK MODE)" if job_type == "check" else "" + + lines = [] + lines.append(f"PLAY [{playbook_name}] *****") + lines.append("") + + lines.append(f"TASK [Gathering Facts{mode}] *****") + for h in hosts: + lines.append(f"ok: [{h}]") + lines.append("") + + if "kernel" in playbook_name.lower(): + lines.append(f"TASK [Create boom snapshot for rollback{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}] => {{\"msg\": \"boom create --title pre-remediation-CVE-2024-12345\"}}") + lines.append("") + + lines.append(f"TASK [Check disk space for kernel update{mode}] *****") + for h in hosts: + lines.append(f"ok: [{h}] => {{\"msg\": \"Disk space OK: 45% used\"}}") + lines.append("") + + lines.append(f"TASK [Update kernel package{mode}] *****") + for h in hosts: + result = "changed" if status == "successful" else "fatal" + if result == "changed": + lines.append(f'changed: [{h}] => {{"msg": "kernel-5.14.0-362.24.1.el9_3 -> kernel-5.14.0-362.24.2.el9_3"}}') + else: + lines.append(f'fatal: [{h}]: FAILED! => {{"msg": "Permission denied", "rc": 1}}') + lines.append("") + + lines.append(f"TASK [Check if reboot is needed (needs-restarting -r){mode}] *****") + for h in hosts: + lines.append(f'changed: [{h}] => {{"rc": 1, "msg": "Reboot is required to fully utilize updates."}}') + lines.append("") + + elif "package" in playbook_name.lower(): + lines.append(f"TASK [Update target packages for CVE remediation{mode}] *****") + for h in hosts: + lines.append(f'changed: [{h}] => {{"msg": "httpd-2.4.53-7.el9 -> httpd-2.4.57-8.el9"}}') + lines.append("") + + lines.append(f"TASK [Restart affected services{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}]") + lines.append("") + + lines.append(f"TASK [Verify service health{mode}] *****") + for h in hosts: + lines.append(f'ok: [{h}] => {{"msg": "Service httpd is running"}}') + lines.append("") + + elif "emergency" in playbook_name.lower() and status == "failed": + lines.append(f"TASK [Apply emergency patch{mode}] *****") + for h in hosts: + lines.append(f'fatal: [{h}]: FAILED! => {{"msg": "Missing sudo password (become_enabled not set)", "rc": 1}}') + lines.append("") + lines.append("NO MORE HOSTS LEFT *****") + lines.append("") + + else: + lines.append(f"TASK [Execute playbook tasks{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}]") + lines.append("") + + lines.append("PLAY RECAP *****") + for h in hosts: + if status == "successful": + ok_count = random.randint(3, 6) + changed_count = random.randint(1, 3) + lines.append(f"{h:<45} : ok={ok_count} changed={changed_count} unreachable=0 failed=0 skipped=0 rescued=0 ignored=0") + elif status == "failed": + lines.append(f"{h:<45} : ok=1 changed=0 unreachable=0 failed=1 skipped=0 rescued=0 ignored=0") + lines.append("") + + return "\n".join(lines) + + +def _generate_events(job: dict) -> list[dict]: + """Generate realistic Ansible task events for a job.""" + hosts = (job.get("limit", "").split(",") if job.get("limit") else PROD_HOSTS[:3]) + hosts = [h.strip() for h in hosts if h.strip()] + events: list[dict] = [] + eid = 1 + + task_names = ["Gathering Facts"] + if "kernel" in job.get("name", "").lower(): + task_names += [ + "Create boom snapshot for rollback", + "Check disk space for kernel update", + "Update kernel package", + "Check if reboot is needed (needs-restarting -r)", + ] + elif "package" in job.get("name", "").lower(): + task_names += [ + "Update target packages for CVE remediation", + "Restart affected services", + "Verify service health", + ] + else: + task_names += ["Execute playbook tasks"] + + for task_name in task_names: + for host in hosts: + is_failed = job.get("status") == "failed" and task_name != "Gathering Facts" + events.append({ + "id": eid, + "type": "job_event", + "event": "runner_on_ok" if not is_failed else "runner_on_failed", + "task": task_name, + "host": host, + "host_name": host, + "play": job.get("name", ""), + "changed": task_name != "Gathering Facts" and not is_failed, + "failed": is_failed, + "event_data": { + "task": task_name, + "host": host, + "res": { + "changed": task_name != "Gathering Facts" and not is_failed, + "msg": "Task completed" if not is_failed else "Permission denied", + }, + }, + "created": _ts(timedelta(hours=4, minutes=30 - eid)), + }) + eid += 1 + + return events + + +def _generate_host_summaries(job: dict) -> list[dict]: + """Generate per-host summaries for a job.""" + hosts = (job.get("limit", "").split(",") if job.get("limit") else PROD_HOSTS[:3]) + hosts = [h.strip() for h in hosts if h.strip()] + summaries: list[dict] = [] + + for i, host in enumerate(hosts): + is_failed = job.get("status") == "failed" + summaries.append({ + "id": i + 1, + "type": "job_host_summary", + "host": i + 1, + "host_name": host, + "ok": 1 if is_failed else random.randint(3, 6), + "changed": 0 if is_failed else random.randint(1, 3), + "dark": 0, + "failures": 1 if is_failed else 0, + "skipped": 0, + "processed": 1, + "failed": is_failed, + }) + + return summaries + + +# --------------------------------------------------------------------------- +# MCP Tools: Job Management +# --------------------------------------------------------------------------- + +@mcp.tool() +def job_templates_list( + page_size: int = 10, + search: Optional[str] = None, +) -> dict: + """List available job templates in AAP. + + Args: + page_size: Number of results per page (default 10, max 200). + search: Optional search string to filter templates by name. + """ + results = MOCK_JOB_TEMPLATES + if search: + s = search.lower() + results = [t for t in results if s in t["name"].lower() or s in t.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def job_templates_retrieve(id: str) -> dict: + """Retrieve detailed information about a specific job template. + + Args: + id: Job template ID (as string). + """ + tid = int(id) + template = next((t for t in MOCK_JOB_TEMPLATES if t["id"] == tid), None) + if not template: + return {"detail": f"Not found. Job template {id} does not exist."} + return template + + +@mcp.tool() +def projects_list( + page_size: int = 50, + search: Optional[str] = None, +) -> dict: + """List available projects in AAP. + + Args: + page_size: Number of results per page. + search: Optional search string to filter projects by name. + """ + results = MOCK_PROJECTS + if search: + s = search.lower() + results = [p for p in results if s in p["name"].lower() or s in p.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def job_templates_launch_retrieve( + id: str, + requestBody: Optional[dict] = None, +) -> dict: + """Launch a job from a job template. + + Args: + id: Job template ID to launch. + requestBody: Optional launch parameters including job_type ('run' or 'check'), + extra_vars (dict), and limit (comma-separated host list). + """ + global _next_job_id + tid = int(id) + template = next((t for t in MOCK_JOB_TEMPLATES if t["id"] == tid), None) + if not template: + return {"detail": f"Not found. Job template {id} does not exist."} + + body = requestBody or {} + job_type = body.get("job_type", template.get("job_type", "run")) + + if not template.get("ask_job_type_on_launch") and job_type != template.get("job_type"): + return { + "error": f"Cannot override job_type: ask_job_type_on_launch is disabled on template {id}", + } + + job_id = _next_job_id + _next_job_id += 1 + + new_job = { + "id": job_id, + "type": "job", + "name": template["name"], + "job_type": job_type, + "status": "pending", + "failed": False, + "started": _ts(timedelta(seconds=0)), + "finished": None, + "elapsed": 0.0, + "job_template": tid, + "inventory": template["inventory"], + "project": template["project"], + "playbook": template["playbook"], + "limit": body.get("limit", ""), + "extra_vars": str(body.get("extra_vars", {})), + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": tid, "name": template["name"]}, + }, + } + MOCK_JOBS.append(new_job) + + # Simulate job completion after launch + new_job["status"] = "successful" + new_job["finished"] = _ts(timedelta(seconds=-300)) + new_job["elapsed"] = 300.0 + + return { + "job": job_id, + "status": "pending", + "type": "job", + "url": f"/api/controller/v2/jobs/{job_id}/", + "related": { + "stdout": f"/api/controller/v2/jobs/{job_id}/stdout/", + "job_events": f"/api/controller/v2/jobs/{job_id}/job_events/", + "job_host_summaries": f"/api/controller/v2/jobs/{job_id}/job_host_summaries/", + }, + } + + +@mcp.tool() +def jobs_retrieve(id: int) -> dict: + """Get the status and details of a job run. + + Args: + id: Job ID to retrieve. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + return job + + +@mcp.tool() +def jobs_list(page_size: int = 10) -> dict: + """List recent job runs. + + Args: + page_size: Number of results to return. + """ + results = sorted(MOCK_JOBS, key=lambda j: j.get("started", ""), reverse=True) + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def jobs_stdout_retrieve(id: int, format: str = "txt") -> dict: + """Get the stdout (console output) from a job run. + + Args: + id: Job ID. + format: Output format ('txt' or 'json'). Default 'txt'. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + return { + "content": _generate_stdout(job), + "range": {"start": 0, "end": 1}, + } + + +@mcp.tool() +def jobs_job_events_list(id: int, page_size: int = 50) -> dict: + """Get task-level events for a job run. + + Args: + id: Job ID. + page_size: Number of events to return. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + events = _generate_events(job) + return { + "count": len(events), + "next": None, + "previous": None, + "results": events[:page_size], + } + + +@mcp.tool() +def jobs_job_host_summaries_list(id: int) -> dict: + """Get per-host execution summaries for a job run. + + Args: + id: Job ID. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + summaries = _generate_host_summaries(job) + return { + "count": len(summaries), + "next": None, + "previous": None, + "results": summaries, + } + + +@mcp.tool() +def jobs_relaunch_retrieve( + id: int, + hosts: str = "all", + job_type: str = "run", +) -> dict: + """Relaunch a previously completed or failed job. + + Args: + id: Original job ID to relaunch. + hosts: Which hosts to target ('all' or 'failed'). + job_type: Job type for relaunch ('run' or 'check'). + """ + global _next_job_id + original = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not original: + return {"detail": f"Not found. Job {id} does not exist."} + + new_id = _next_job_id + _next_job_id += 1 + + new_job = { + **original, + "id": new_id, + "job_type": job_type, + "status": "successful", + "failed": False, + "started": _ts(timedelta(seconds=0)), + "finished": _ts(timedelta(seconds=-300)), + "elapsed": 300.0, + "launch_type": "relaunch", + } + MOCK_JOBS.append(new_job) + + return { + "job": new_id, + "status": "pending", + "type": "job", + "url": f"/api/controller/v2/jobs/{new_id}/", + } + + +# --------------------------------------------------------------------------- +# MCP Tools: Inventory Management +# --------------------------------------------------------------------------- + +@mcp.tool() +def inventories_list( + page_size: int = 10, + search: Optional[str] = None, +) -> dict: + """List available inventories in AAP. + + Args: + page_size: Number of results per page. + search: Optional search string to filter inventories. + """ + results = MOCK_INVENTORIES + if search: + s = search.lower() + results = [inv for inv in results if s in inv["name"].lower() or s in inv.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def hosts_list( + inventory_id: Optional[int] = None, + page_size: int = 50, + search: Optional[str] = None, +) -> dict: + """List hosts in an inventory. + + Args: + inventory_id: Filter by inventory ID. If not provided, lists hosts from all inventories. + page_size: Number of results per page. + search: Optional search string to filter hosts by name. + """ + inv_id = inventory_id or 1 + hosts = _generate_hosts(inv_id) + if search: + s = search.lower() + hosts = [h for h in hosts if s in h["name"].lower()] + return { + "count": len(hosts), + "next": None if len(hosts) <= page_size else f"/api/controller/v2/hosts/?page=2", + "previous": None, + "results": hosts[:page_size], + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/without_skills/rh-sre__job-template-creator/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..fe5d549c --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-sre__job-template-creator/instruction.md b/evaluation/without_skills/rh-sre__job-template-creator/instruction.md new file mode 100644 index 00000000..77c24f2b --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/instruction.md @@ -0,0 +1,17 @@ +# Job Template Creator Task + +You are a Red Hat SRE. A remediation playbook has been written for a critical CVE, and you need to set up an Ansible Automation Platform job template so the team can run it against affected systems. + +## Scenario +The security team delivered a remediation playbook for CVE-2026-1234. You need to create a job template in AAP that the operations team can use to run this playbook against production hosts. + +## Requirements +- Check which projects and inventories are available in AAP +- Determine the correct project, inventory, and credentials for the remediation playbook +- Document the job template configuration: name, playbook path, inventory, project, credentials, and execution settings (privilege escalation, variable prompts, limit prompts) +- Explain any decisions about template settings (e.g., why `become` is enabled, whether to prompt for variables at launch) +- If template creation requires manual steps (e.g., via the AAP Web UI), document those steps clearly + +Document your methodology, plan, and configuration in `/root/report.md`. + +Use MCP tools to query AAP. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-sre__job-template-creator/solution/solve.sh b/evaluation/without_skills/rh-sre__job-template-creator/solution/solve.sh new file mode 100644 index 00000000..ec9c5b02 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/solution/solve.sh @@ -0,0 +1,19 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Job Template Creation + +## Template Fields +- Inventory: production-systems +- Project: remediation-playbooks +- Playbook: playbooks/remediation/cve-2024-12345.yml +- Credentials: machine-credential +- become_enabled: true + +## Prompt on Launch +- Job Type (REQUIRED for dry-run + run) +- Variables +- Limit + +## Note +No job_templates_create API in AAP MCP. Create via Web UI. Execute mcp-aap-validator before operations. +REPORT_EOF diff --git a/evaluation/without_skills/rh-sre__job-template-creator/task.toml b/evaluation/without_skills/rh-sre__job-template-creator/task.toml new file mode 100644 index 00000000..bc2620fa --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__job-template-creator" +name = "rh-sre AAP Job Template Creation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "job-template-creator", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-sre__job-template-creator/tests/llm_judge.py b/evaluation/without_skills/rh-sre__job-template-creator/tests/llm_judge.py new file mode 100644 index 00000000..54c93ce1 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "no_create_tool", "file": "/root/report.md", "question": "Does the report acknowledge that AAP MCP has no create/update tools and template creation must be done via Web UI?", "reference": "A skilled report notes the MCP limitation and directs to Web UI. An unskilled report attempts to create templates via API."}, + {"id": "playbook_path_and_git", "file": "/root/report.md", "question": "Does the report require the playbook to be in a Git repo with proper path convention before template creation?", "reference": "A skilled report follows playbooks/remediation/ path convention. An unskilled report skips Git integration."}, + {"id": "launch_configuration", "file": "/root/report.md", "question": "Does the report configure prompt-on-launch for job type and privilege escalation?", "reference": "A skilled report enables prompt-on-launch and become_enabled. An unskilled report skips these configuration details."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-sre__job-template-creator/tests/test.sh b/evaluation/without_skills/rh-sre__job-template-creator/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-sre__job-template-creator/tests/test_outputs.py b/evaluation/without_skills/rh-sre__job-template-creator/tests/test_outputs.py new file mode 100644 index 00000000..53140085 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-creator/tests/test_outputs.py @@ -0,0 +1,98 @@ +""" +Tests for rh-sre__job-template-creator per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['job template', 'template', 'ansible']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_git_before_template(self): + """Skill: Playbook must be in Git repo before template creation; AAP syncs from project.""" + c = read_report().lower() + has_git = any(t in c for t in ["git", "commit", "push", "repository", "sync"]) + has_project = any(t in c for t in ["project", "scm", "sync"]) + assert has_git or has_project, ( + "should add playbook to Git before template (skill: Phase 1)" + ) + + def test_manual_creation_required(self): + """Skill teaches template creation requires manual steps (e.g., Web UI) + because the automation API is read-only for templates.""" + c = read_report().lower() + assert any(t in c for t in [ + "web ui", "manual", "read-only", "cannot create", + "no create", "gui", "interface", + ]), "should acknowledge template creation requires manual steps" + + def test_playbook_path_convention(self): + """Skill teaches following a consistent directory structure or location + convention for remediation playbooks.""" + c = read_report().lower() + assert any(t in c for t in [ + "playbook path", "remediation playbook", "playbook location", + "playbook directory", "playbook structure", + ]), "should follow a playbook path convention for remediation" + + def test_privilege_escalation_required(self): + """Skill: become_enabled required for remediation (package updates).""" + c = read_report().lower() + assert any(t in c for t in ["privilege", "become", "sudo", "escalat", "root"]), ( + "should require privilege escalation (skill: required for package updates)" + ) + + def test_launch_prompts(self): + """Skill: Prompt on Launch for Job Type, Variables, Limit.""" + c = read_report().lower() + assert any(t in c for t in ["launch", "prompt", "variable", "limit", "job type"]), ( + "should configure prompt on launch (skill: Phase 4)" + ) + + def test_configurable_variables(self): + """Docs teach configuring variables for CVE targeting, remediation mode, + and post-remediation verification. Without docs, agents skip variable design.""" + c = read_report().lower() + concepts = sum(1 for t in [ + "target_cve", "cve", "remediation_mode", "mode", + "verify_after", "verification", "extra_var", "extra var", + "variable", "parameter", + ] if t in c) + assert concepts >= 3, ( + "should define configurable variables for CVE targeting, " + "remediation mode, and verification" + ) + + def test_version_control_sync(self): + """Skill teaches AAP projects sync playbooks from version control. + Without skill, agents describe playbook management without + version-control-backed project sync.""" + c = read_report().lower() + assert any(t in c for t in [ + "scm", "source control", "version control", + "repository sync", "git-backed", "git sync", + ]), "should reference version control sync for AAP project playbooks" diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/Dockerfile b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/Dockerfile new file mode 100644 index 00000000..51ce02e5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/Dockerfile @@ -0,0 +1,47 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + }, \ + "aap-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-aap-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..6c122770 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- Pass Rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/mcp-servers/mock-aap-mcp.py b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/mcp-servers/mock-aap-mcp.py new file mode 100644 index 00000000..d8ae4fd5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/mcp-servers/mock-aap-mcp.py @@ -0,0 +1,1048 @@ +#!/usr/bin/env python3 +""" +Mock AAP (Ansible Automation Platform) MCP Server + +Simulates the AAP MCP gateway for per-skill evaluation tasks. Implements +the full set of tools used by rh-sre skills: + - job_templates_list / job_templates_retrieve + - projects_list + - job_templates_launch_retrieve + - jobs_retrieve / jobs_stdout_retrieve + - jobs_job_events_list / jobs_job_host_summaries_list + - jobs_relaunch_retrieve + - inventories_list / hosts_list + +Data mirrors a realistic AAP deployment: + - 6 job templates (3 remediation, 1 compliance, 1 patching, 1 reporting) + - 3 projects (remediation, compliance, reporting) + - 3 inventories (production 30 hosts, staging 15 hosts, all-managed 63 hosts) + - 12 recent jobs with varied statuses + +Follows the same mock-server pattern as mock-lightspeed-mcp.py. +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +mcp = FastMCP("aap-mcp") + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +def _ts(delta: timedelta) -> str: + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +# --------------------------------------------------------------------------- +# Mock data: Projects +# --------------------------------------------------------------------------- + +MOCK_PROJECTS = [ + { + "id": 6, + "type": "project", + "name": "Remediation Playbooks", + "description": "CVE and security remediation playbooks managed via Git", + "scm_type": "git", + "scm_url": "https://github.com/org/remediation-playbooks.git", + "scm_branch": "main", + "scm_revision": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + "status": "successful", + "last_job_run": _ts(timedelta(hours=2)), + "last_update_failed": False, + "created": _ts(timedelta(days=90)), + "modified": _ts(timedelta(hours=2)), + }, + { + "id": 7, + "type": "project", + "name": "Compliance Checks", + "description": "STIG and CIS compliance scanning playbooks", + "scm_type": "git", + "scm_url": "https://github.com/org/compliance-playbooks.git", + "scm_branch": "main", + "scm_revision": "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3", + "status": "successful", + "last_job_run": _ts(timedelta(days=1)), + "last_update_failed": False, + "created": _ts(timedelta(days=120)), + "modified": _ts(timedelta(days=1)), + }, + { + "id": 8, + "type": "project", + "name": "Fleet Reporting", + "description": "System inventory and health reporting playbooks", + "scm_type": "git", + "scm_url": "https://github.com/org/fleet-reports.git", + "scm_branch": "main", + "scm_revision": "c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "status": "successful", + "last_job_run": _ts(timedelta(days=3)), + "last_update_failed": False, + "created": _ts(timedelta(days=180)), + "modified": _ts(timedelta(days=3)), + }, +] + +# --------------------------------------------------------------------------- +# Mock data: Inventories & Hosts +# --------------------------------------------------------------------------- + +MOCK_INVENTORIES = [ + { + "id": 1, + "type": "inventory", + "name": "Production Systems", + "description": "All production RHEL systems across data centers", + "total_hosts": 30, + "has_active_failures": False, + "hosts_with_active_failures": 0, + "total_groups": 5, + "groups_with_active_failures": 0, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=365)), + "modified": _ts(timedelta(days=1)), + }, + { + "id": 2, + "type": "inventory", + "name": "Staging Systems", + "description": "Pre-production staging environment", + "total_hosts": 15, + "has_active_failures": False, + "hosts_with_active_failures": 0, + "total_groups": 3, + "groups_with_active_failures": 0, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=300)), + "modified": _ts(timedelta(days=7)), + }, + { + "id": 3, + "type": "inventory", + "name": "All Managed Systems", + "description": "Complete fleet: production, staging, development, QA, legacy", + "total_hosts": 63, + "has_active_failures": True, + "hosts_with_active_failures": 2, + "total_groups": 8, + "groups_with_active_failures": 1, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=365)), + "modified": _ts(timedelta(hours=6)), + }, +] + + +def _generate_hosts(inventory_id: int) -> list[dict]: + """Generate realistic hosts for an inventory.""" + hosts: list[dict] = [] + if inventory_id == 1: + roles = ["web", "db", "app", "lb", "monitoring", "cache"] + for i, role in enumerate(roles): + for j in range(5 if role in ("web", "app") else 4 if role == "db" else 3 if role == "monitoring" else 2): + hosts.append({ + "id": len(hosts) + 1, + "type": "host", + "name": f"{role}-{j+1:02d}.prod.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "production", "role": "{role}"}}', + }) + if len(hosts) >= 30: + break + if len(hosts) >= 30: + break + elif inventory_id == 2: + for i in range(15): + role = ["web", "db", "app"][i % 3] + hosts.append({ + "id": 100 + i, + "type": "host", + "name": f"{role}-{i+1:02d}.staging.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "staging", "role": "{role}"}}', + }) + elif inventory_id == 3: + for i in range(30): + hosts.append({ + "id": 200 + i, + "type": "host", + "name": f"host-{i+1:02d}.prod.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": i in (45, 58), + "variables": f'{{"rhel_version": "9.3", "environment": "production"}}', + }) + for i in range(15): + hosts.append({ + "id": 230 + i, + "type": "host", + "name": f"host-{i+1:02d}.staging.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "staging"}}', + }) + for i in range(10): + hosts.append({ + "id": 245 + i, + "type": "host", + "name": f"dev-{i+1:02d}.dev.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "8.9", "environment": "development"}}', + }) + for i in range(5): + hosts.append({ + "id": 255 + i, + "type": "host", + "name": f"qa-{i+1:02d}.qa.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.2", "environment": "qa"}}', + }) + for i in range(3): + hosts.append({ + "id": 260 + i, + "type": "host", + "name": f"legacy-{i+1:02d}.corp.example.com", + "inventory": inventory_id, + "enabled": i < 2, + "has_active_failures": i == 2, + "variables": f'{{"rhel_version": "7.9", "environment": "legacy"}}', + }) + return hosts + + +# --------------------------------------------------------------------------- +# Mock data: Job Templates +# --------------------------------------------------------------------------- + +MOCK_JOB_TEMPLATES = [ + { + "id": 10, + "type": "job_template", + "name": "CVE Remediation - Kernel Update", + "description": "Kernel update with boom snapshot for rollback safety", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": True, + "job_type": "check", + "verbosity": 1, + "timeout": 3600, + "forks": 5, + "status": "successful", + "last_job_run": _ts(timedelta(hours=4)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1001, "status": "successful", "finished": _ts(timedelta(hours=4))}, + }, + "created": _ts(timedelta(days=60)), + "modified": _ts(timedelta(days=2)), + }, + { + "id": 11, + "type": "job_template", + "name": "CVE Remediation - Package Update", + "description": "General package update for CVE remediation with needs-restarting check", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-package-update.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": False, + "job_type": "check", + "verbosity": 1, + "timeout": 1800, + "forks": 10, + "status": "successful", + "last_job_run": _ts(timedelta(hours=12)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1005, "status": "successful", "finished": _ts(timedelta(hours=12))}, + }, + "created": _ts(timedelta(days=45)), + "modified": _ts(timedelta(days=5)), + }, + { + "id": 12, + "type": "job_template", + "name": "CVE Remediation - Generic", + "description": "Generic CVE remediation template for ad-hoc patches", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-remediation.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": True, + "job_type": "check", + "verbosity": 1, + "timeout": 3600, + "forks": 5, + "status": "never updated", + "last_job_run": None, + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + }, + "created": _ts(timedelta(days=30)), + "modified": _ts(timedelta(days=30)), + }, + { + "id": 20, + "type": "job_template", + "name": "Compliance Check - STIG", + "description": "Run STIG compliance scan across fleet", + "inventory": 3, + "project": 7, + "playbook": "playbooks/compliance/check-all.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": False, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 7200, + "forks": 20, + "status": "successful", + "last_job_run": _ts(timedelta(days=1)), + "summary_fields": { + "project": {"id": 7, "name": "Compliance Checks", "status": "successful"}, + "inventory": {"id": 3, "name": "All Managed Systems", "total_hosts": 63}, + "credentials": [ + {"id": 2, "name": "compliance-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1010, "status": "successful", "finished": _ts(timedelta(days=1))}, + }, + "created": _ts(timedelta(days=180)), + "modified": _ts(timedelta(days=14)), + }, + { + "id": 25, + "type": "job_template", + "name": "Emergency Patching", + "description": "Emergency patch application — NO become enabled (misconfigured)", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/emergency-patch.yml", + "become_enabled": False, + "ask_job_type_on_launch": False, + "ask_variables_on_launch": False, + "ask_limit_on_launch": False, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 600, + "forks": 25, + "status": "failed", + "last_job_run": _ts(timedelta(days=7)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1020, "status": "failed", "finished": _ts(timedelta(days=7))}, + }, + "created": _ts(timedelta(days=200)), + "modified": _ts(timedelta(days=200)), + }, + { + "id": 30, + "type": "job_template", + "name": "Fleet Health Report", + "description": "Generate fleet health and inventory report", + "inventory": 3, + "project": 8, + "playbook": "playbooks/reporting/fleet-health.yml", + "become_enabled": False, + "ask_job_type_on_launch": False, + "ask_variables_on_launch": True, + "ask_limit_on_launch": False, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 1800, + "forks": 30, + "status": "successful", + "last_job_run": _ts(timedelta(hours=6)), + "summary_fields": { + "project": {"id": 8, "name": "Fleet Reporting", "status": "successful"}, + "inventory": {"id": 3, "name": "All Managed Systems", "total_hosts": 63}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1025, "status": "successful", "finished": _ts(timedelta(hours=6))}, + }, + "created": _ts(timedelta(days=120)), + "modified": _ts(timedelta(days=14)), + }, +] + +# --------------------------------------------------------------------------- +# Mock data: Jobs (recent runs) +# --------------------------------------------------------------------------- + +PROD_HOSTS = [ + "web-01.prod.example.com", + "web-02.prod.example.com", + "db-01.prod.example.com", + "db-02.prod.example.com", + "app-01.prod.example.com", + "app-02.prod.example.com", +] + +MOCK_JOBS = [ + { + "id": 1001, + "type": "job", + "name": "CVE Remediation - Kernel Update", + "job_type": "check", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=4, minutes=30)), + "finished": _ts(timedelta(hours=4)), + "elapsed": 1800.0, + "job_template": 10, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "limit": "web-01.prod.example.com,web-02.prod.example.com,db-01.prod.example.com", + "extra_vars": '{"target_cve": "CVE-2024-12345", "remediation_mode": "automated", "verify_after": true}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 10, "name": "CVE Remediation - Kernel Update"}, + }, + }, + { + "id": 1002, + "type": "job", + "name": "CVE Remediation - Kernel Update", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=3, minutes=45)), + "finished": _ts(timedelta(hours=3)), + "elapsed": 2700.0, + "job_template": 10, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "limit": "web-01.prod.example.com,web-02.prod.example.com,db-01.prod.example.com", + "extra_vars": '{"target_cve": "CVE-2024-12345", "remediation_mode": "automated", "verify_after": true}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 10, "name": "CVE Remediation - Kernel Update"}, + }, + }, + { + "id": 1005, + "type": "job", + "name": "CVE Remediation - Package Update", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=12, minutes=20)), + "finished": _ts(timedelta(hours=12)), + "elapsed": 1200.0, + "job_template": 11, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-package-update.yml", + "limit": "", + "extra_vars": '{"target_cve": "CVE-2024-54321"}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 11, "name": "CVE Remediation - Package Update"}, + }, + }, + { + "id": 1010, + "type": "job", + "name": "Compliance Check - STIG", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(days=1, hours=2)), + "finished": _ts(timedelta(days=1)), + "elapsed": 7200.0, + "job_template": 20, + "inventory": 3, + "project": 7, + "playbook": "playbooks/compliance/check-all.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "scheduled", + "summary_fields": { + "job_template": {"id": 20, "name": "Compliance Check - STIG"}, + }, + }, + { + "id": 1020, + "type": "job", + "name": "Emergency Patching", + "job_type": "run", + "status": "failed", + "failed": True, + "started": _ts(timedelta(days=7, hours=1)), + "finished": _ts(timedelta(days=7)), + "elapsed": 3600.0, + "job_template": 25, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/emergency-patch.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 25, "name": "Emergency Patching"}, + }, + }, + { + "id": 1025, + "type": "job", + "name": "Fleet Health Report", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=6, minutes=30)), + "finished": _ts(timedelta(hours=6)), + "elapsed": 1800.0, + "job_template": 30, + "inventory": 3, + "project": 8, + "playbook": "playbooks/reporting/fleet-health.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "scheduled", + "summary_fields": { + "job_template": {"id": 30, "name": "Fleet Health Report"}, + }, + }, +] + +_next_job_id = 2000 + + +# --------------------------------------------------------------------------- +# Mock stdout generators +# --------------------------------------------------------------------------- + +def _generate_stdout(job: dict) -> str: + """Generate realistic Ansible playbook stdout for a job.""" + playbook_name = job.get("name", "Unknown") + job_type = job.get("job_type", "run") + status = job.get("status", "successful") + limit = job.get("limit", "") + hosts = limit.split(",") if limit else PROD_HOSTS[:3] + hosts = [h.strip() for h in hosts if h.strip()] + extra_vars = job.get("extra_vars", "{}") + mode = " (CHECK MODE)" if job_type == "check" else "" + + lines = [] + lines.append(f"PLAY [{playbook_name}] *****") + lines.append("") + + lines.append(f"TASK [Gathering Facts{mode}] *****") + for h in hosts: + lines.append(f"ok: [{h}]") + lines.append("") + + if "kernel" in playbook_name.lower(): + lines.append(f"TASK [Create boom snapshot for rollback{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}] => {{\"msg\": \"boom create --title pre-remediation-CVE-2024-12345\"}}") + lines.append("") + + lines.append(f"TASK [Check disk space for kernel update{mode}] *****") + for h in hosts: + lines.append(f"ok: [{h}] => {{\"msg\": \"Disk space OK: 45% used\"}}") + lines.append("") + + lines.append(f"TASK [Update kernel package{mode}] *****") + for h in hosts: + result = "changed" if status == "successful" else "fatal" + if result == "changed": + lines.append(f'changed: [{h}] => {{"msg": "kernel-5.14.0-362.24.1.el9_3 -> kernel-5.14.0-362.24.2.el9_3"}}') + else: + lines.append(f'fatal: [{h}]: FAILED! => {{"msg": "Permission denied", "rc": 1}}') + lines.append("") + + lines.append(f"TASK [Check if reboot is needed (needs-restarting -r){mode}] *****") + for h in hosts: + lines.append(f'changed: [{h}] => {{"rc": 1, "msg": "Reboot is required to fully utilize updates."}}') + lines.append("") + + elif "package" in playbook_name.lower(): + lines.append(f"TASK [Update target packages for CVE remediation{mode}] *****") + for h in hosts: + lines.append(f'changed: [{h}] => {{"msg": "httpd-2.4.53-7.el9 -> httpd-2.4.57-8.el9"}}') + lines.append("") + + lines.append(f"TASK [Restart affected services{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}]") + lines.append("") + + lines.append(f"TASK [Verify service health{mode}] *****") + for h in hosts: + lines.append(f'ok: [{h}] => {{"msg": "Service httpd is running"}}') + lines.append("") + + elif "emergency" in playbook_name.lower() and status == "failed": + lines.append(f"TASK [Apply emergency patch{mode}] *****") + for h in hosts: + lines.append(f'fatal: [{h}]: FAILED! => {{"msg": "Missing sudo password (become_enabled not set)", "rc": 1}}') + lines.append("") + lines.append("NO MORE HOSTS LEFT *****") + lines.append("") + + else: + lines.append(f"TASK [Execute playbook tasks{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}]") + lines.append("") + + lines.append("PLAY RECAP *****") + for h in hosts: + if status == "successful": + ok_count = random.randint(3, 6) + changed_count = random.randint(1, 3) + lines.append(f"{h:<45} : ok={ok_count} changed={changed_count} unreachable=0 failed=0 skipped=0 rescued=0 ignored=0") + elif status == "failed": + lines.append(f"{h:<45} : ok=1 changed=0 unreachable=0 failed=1 skipped=0 rescued=0 ignored=0") + lines.append("") + + return "\n".join(lines) + + +def _generate_events(job: dict) -> list[dict]: + """Generate realistic Ansible task events for a job.""" + hosts = (job.get("limit", "").split(",") if job.get("limit") else PROD_HOSTS[:3]) + hosts = [h.strip() for h in hosts if h.strip()] + events: list[dict] = [] + eid = 1 + + task_names = ["Gathering Facts"] + if "kernel" in job.get("name", "").lower(): + task_names += [ + "Create boom snapshot for rollback", + "Check disk space for kernel update", + "Update kernel package", + "Check if reboot is needed (needs-restarting -r)", + ] + elif "package" in job.get("name", "").lower(): + task_names += [ + "Update target packages for CVE remediation", + "Restart affected services", + "Verify service health", + ] + else: + task_names += ["Execute playbook tasks"] + + for task_name in task_names: + for host in hosts: + is_failed = job.get("status") == "failed" and task_name != "Gathering Facts" + events.append({ + "id": eid, + "type": "job_event", + "event": "runner_on_ok" if not is_failed else "runner_on_failed", + "task": task_name, + "host": host, + "host_name": host, + "play": job.get("name", ""), + "changed": task_name != "Gathering Facts" and not is_failed, + "failed": is_failed, + "event_data": { + "task": task_name, + "host": host, + "res": { + "changed": task_name != "Gathering Facts" and not is_failed, + "msg": "Task completed" if not is_failed else "Permission denied", + }, + }, + "created": _ts(timedelta(hours=4, minutes=30 - eid)), + }) + eid += 1 + + return events + + +def _generate_host_summaries(job: dict) -> list[dict]: + """Generate per-host summaries for a job.""" + hosts = (job.get("limit", "").split(",") if job.get("limit") else PROD_HOSTS[:3]) + hosts = [h.strip() for h in hosts if h.strip()] + summaries: list[dict] = [] + + for i, host in enumerate(hosts): + is_failed = job.get("status") == "failed" + summaries.append({ + "id": i + 1, + "type": "job_host_summary", + "host": i + 1, + "host_name": host, + "ok": 1 if is_failed else random.randint(3, 6), + "changed": 0 if is_failed else random.randint(1, 3), + "dark": 0, + "failures": 1 if is_failed else 0, + "skipped": 0, + "processed": 1, + "failed": is_failed, + }) + + return summaries + + +# --------------------------------------------------------------------------- +# MCP Tools: Job Management +# --------------------------------------------------------------------------- + +@mcp.tool() +def job_templates_list( + page_size: int = 10, + search: Optional[str] = None, +) -> dict: + """List available job templates in AAP. + + Args: + page_size: Number of results per page (default 10, max 200). + search: Optional search string to filter templates by name. + """ + results = MOCK_JOB_TEMPLATES + if search: + s = search.lower() + results = [t for t in results if s in t["name"].lower() or s in t.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def job_templates_retrieve(id: str) -> dict: + """Retrieve detailed information about a specific job template. + + Args: + id: Job template ID (as string). + """ + tid = int(id) + template = next((t for t in MOCK_JOB_TEMPLATES if t["id"] == tid), None) + if not template: + return {"detail": f"Not found. Job template {id} does not exist."} + return template + + +@mcp.tool() +def projects_list( + page_size: int = 50, + search: Optional[str] = None, +) -> dict: + """List available projects in AAP. + + Args: + page_size: Number of results per page. + search: Optional search string to filter projects by name. + """ + results = MOCK_PROJECTS + if search: + s = search.lower() + results = [p for p in results if s in p["name"].lower() or s in p.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def job_templates_launch_retrieve( + id: str, + requestBody: Optional[dict] = None, +) -> dict: + """Launch a job from a job template. + + Args: + id: Job template ID to launch. + requestBody: Optional launch parameters including job_type ('run' or 'check'), + extra_vars (dict), and limit (comma-separated host list). + """ + global _next_job_id + tid = int(id) + template = next((t for t in MOCK_JOB_TEMPLATES if t["id"] == tid), None) + if not template: + return {"detail": f"Not found. Job template {id} does not exist."} + + body = requestBody or {} + job_type = body.get("job_type", template.get("job_type", "run")) + + if not template.get("ask_job_type_on_launch") and job_type != template.get("job_type"): + return { + "error": f"Cannot override job_type: ask_job_type_on_launch is disabled on template {id}", + } + + job_id = _next_job_id + _next_job_id += 1 + + new_job = { + "id": job_id, + "type": "job", + "name": template["name"], + "job_type": job_type, + "status": "pending", + "failed": False, + "started": _ts(timedelta(seconds=0)), + "finished": None, + "elapsed": 0.0, + "job_template": tid, + "inventory": template["inventory"], + "project": template["project"], + "playbook": template["playbook"], + "limit": body.get("limit", ""), + "extra_vars": str(body.get("extra_vars", {})), + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": tid, "name": template["name"]}, + }, + } + MOCK_JOBS.append(new_job) + + # Simulate job completion after launch + new_job["status"] = "successful" + new_job["finished"] = _ts(timedelta(seconds=-300)) + new_job["elapsed"] = 300.0 + + return { + "job": job_id, + "status": "pending", + "type": "job", + "url": f"/api/controller/v2/jobs/{job_id}/", + "related": { + "stdout": f"/api/controller/v2/jobs/{job_id}/stdout/", + "job_events": f"/api/controller/v2/jobs/{job_id}/job_events/", + "job_host_summaries": f"/api/controller/v2/jobs/{job_id}/job_host_summaries/", + }, + } + + +@mcp.tool() +def jobs_retrieve(id: int) -> dict: + """Get the status and details of a job run. + + Args: + id: Job ID to retrieve. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + return job + + +@mcp.tool() +def jobs_list(page_size: int = 10) -> dict: + """List recent job runs. + + Args: + page_size: Number of results to return. + """ + results = sorted(MOCK_JOBS, key=lambda j: j.get("started", ""), reverse=True) + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def jobs_stdout_retrieve(id: int, format: str = "txt") -> dict: + """Get the stdout (console output) from a job run. + + Args: + id: Job ID. + format: Output format ('txt' or 'json'). Default 'txt'. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + return { + "content": _generate_stdout(job), + "range": {"start": 0, "end": 1}, + } + + +@mcp.tool() +def jobs_job_events_list(id: int, page_size: int = 50) -> dict: + """Get task-level events for a job run. + + Args: + id: Job ID. + page_size: Number of events to return. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + events = _generate_events(job) + return { + "count": len(events), + "next": None, + "previous": None, + "results": events[:page_size], + } + + +@mcp.tool() +def jobs_job_host_summaries_list(id: int) -> dict: + """Get per-host execution summaries for a job run. + + Args: + id: Job ID. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + summaries = _generate_host_summaries(job) + return { + "count": len(summaries), + "next": None, + "previous": None, + "results": summaries, + } + + +@mcp.tool() +def jobs_relaunch_retrieve( + id: int, + hosts: str = "all", + job_type: str = "run", +) -> dict: + """Relaunch a previously completed or failed job. + + Args: + id: Original job ID to relaunch. + hosts: Which hosts to target ('all' or 'failed'). + job_type: Job type for relaunch ('run' or 'check'). + """ + global _next_job_id + original = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not original: + return {"detail": f"Not found. Job {id} does not exist."} + + new_id = _next_job_id + _next_job_id += 1 + + new_job = { + **original, + "id": new_id, + "job_type": job_type, + "status": "successful", + "failed": False, + "started": _ts(timedelta(seconds=0)), + "finished": _ts(timedelta(seconds=-300)), + "elapsed": 300.0, + "launch_type": "relaunch", + } + MOCK_JOBS.append(new_job) + + return { + "job": new_id, + "status": "pending", + "type": "job", + "url": f"/api/controller/v2/jobs/{new_id}/", + } + + +# --------------------------------------------------------------------------- +# MCP Tools: Inventory Management +# --------------------------------------------------------------------------- + +@mcp.tool() +def inventories_list( + page_size: int = 10, + search: Optional[str] = None, +) -> dict: + """List available inventories in AAP. + + Args: + page_size: Number of results per page. + search: Optional search string to filter inventories. + """ + results = MOCK_INVENTORIES + if search: + s = search.lower() + results = [inv for inv in results if s in inv["name"].lower() or s in inv.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def hosts_list( + inventory_id: Optional[int] = None, + page_size: int = 50, + search: Optional[str] = None, +) -> dict: + """List hosts in an inventory. + + Args: + inventory_id: Filter by inventory ID. If not provided, lists hosts from all inventories. + page_size: Number of results per page. + search: Optional search string to filter hosts by name. + """ + inv_id = inventory_id or 1 + hosts = _generate_hosts(inv_id) + if search: + s = search.lower() + hosts = [h for h in hosts if s in h["name"].lower()] + return { + "count": len(hosts), + "next": None if len(hosts) <= page_size else f"/api/controller/v2/hosts/?page=2", + "previous": None, + "results": hosts[:page_size], + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..fe5d549c --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/instruction.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/instruction.md new file mode 100644 index 00000000..55b78ca1 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/instruction.md @@ -0,0 +1,18 @@ +# Job Template Validation Task + +You are a Red Hat SRE. Before running a CVE remediation playbook through AAP, you need to verify that the job template is correctly configured and safe to execute. + +## Scenario +The team wants to use an existing AAP job template to remediate a critical vulnerability. Before giving the green light, you need to confirm the template meets all requirements for a safe remediation run. + +## Requirements +- Retrieve the job template configuration from AAP +- Verify required fields are set: inventory, project, playbook, credentials, and privilege escalation +- Check recommended settings: whether the template prompts for variables, limit, and inventory at launch +- Verify the referenced project and inventory actually exist in AAP +- Produce a pass/warn/fail assessment for each configuration item +- Summarize whether the template is ready for production remediation use + +Document your methodology, validation results, and assessment in `/root/report.md`. + +Use MCP tools to query AAP. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/solution/solve.sh b/evaluation/without_skills/rh-sre__job-template-remediation-validator/solution/solve.sh new file mode 100644 index 00000000..6e9ff39d --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/solution/solve.sh @@ -0,0 +1,21 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Job Template Validation + +## Required Checks +| Field | Expected | Status | +|-------|----------|--------| +| ask_job_type_on_launch | true | ✅ | +| become_enabled | true | ✅ | +| credentials | present | ✅ | +| inventory | present | ✅ | +| project | present | ✅ | +| playbook | present | ✅ | + +## Recommended +- ask_variables_on_launch: true +- ask_limit_on_launch: true + +## Overall +✓ PASSED - Template ready for remediation playbook execution. +REPORT_EOF diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/task.toml b/evaluation/without_skills/rh-sre__job-template-remediation-validator/task.toml new file mode 100644 index 00000000..2b6428ba --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__job-template-remediation-validator" +name = "rh-sre Job Template Validation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "job-template-remediation-validator", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/tests/llm_judge.py b/evaluation/without_skills/rh-sre__job-template-remediation-validator/tests/llm_judge.py new file mode 100644 index 00000000..106f21c9 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "ask_job_type_required", "file": "/root/report.md", "question": "Does the report require ask_job_type_on_launch: true for dual check/run mode support?", "reference": "A skilled report requires this for dry-run vs run flexibility. An unskilled report doesn't validate this field."}, + {"id": "become_and_credentials", "file": "/root/report.md", "question": "Does the report validate both become_enabled and credentials (checking summary_fields.credentials or credentials array)?", "reference": "A skilled report checks both credential locations. An unskilled report checks only one."}, + {"id": "required_vs_recommended", "file": "/root/report.md", "question": "Does the report distinguish required fields (inventory, project, playbook, credentials, become, ask_job_type) from recommended (ask_variables, ask_limit)?", "reference": "A skilled report categorizes validation checks by priority. An unskilled report treats all checks equally."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/tests/test.sh b/evaluation/without_skills/rh-sre__job-template-remediation-validator/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/tests/test_outputs.py b/evaluation/without_skills/rh-sre__job-template-remediation-validator/tests/test_outputs.py new file mode 100644 index 00000000..b39c5886 --- /dev/null +++ b/evaluation/without_skills/rh-sre__job-template-remediation-validator/tests/test_outputs.py @@ -0,0 +1,63 @@ +""" +Tests for rh-sre__job-template-remediation-validator per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['valid', 'job template', 'check']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_ask_job_type_on_launch(self): + """Skill teaches ask_job_type_on_launch: true is required for check vs run modes.""" + c = read_report().lower() + assert any(t in c for t in ["ask_job_type", "ask_job_type_on_launch"]), ( + "should require ask_job_type_on_launch (skill: for check vs run)" + ) + + def test_credentials_check_both_fields(self): + """Skill teaches credentials may be in summary_fields.credentials OR credentials array.""" + c = read_report().lower() + assert any(t in c for t in ["summary_fields", "credentials array", "both"]), ( + "should check credentials in summary_fields or credentials array (skill-specific)" + ) + + def test_become_enabled_required(self): + """Skill: become_enabled required for package updates.""" + c = read_report().lower() + assert any(t in c for t in ["become", "privilege", "escalat", "sudo"]), ( + "should require privilege escalation (skill: required for remediation)" + ) + + def test_required_vs_recommended(self): + """Skill: Distinguish required (inventory, project, playbook, credentials, become, ask_job_type) vs recommended (ask_variables, ask_limit).""" + c = read_report().lower() + has_required = any(t in c for t in ["required", "must", "inventory", "project", "playbook"]) + has_recommended = any(t in c for t in ["recommended", "warn", "variable", "limit"]) + assert has_required or has_recommended, ( + "should distinguish required vs recommended checks (skill: Phase 2 vs 3)" + ) diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/Dockerfile b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/Dockerfile new file mode 100644 index 00000000..51ce02e5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/Dockerfile @@ -0,0 +1,47 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + }, \ + "aap-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-aap-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..6c122770 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- Pass Rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/mcp-servers/mock-aap-mcp.py b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/mcp-servers/mock-aap-mcp.py new file mode 100644 index 00000000..d8ae4fd5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/mcp-servers/mock-aap-mcp.py @@ -0,0 +1,1048 @@ +#!/usr/bin/env python3 +""" +Mock AAP (Ansible Automation Platform) MCP Server + +Simulates the AAP MCP gateway for per-skill evaluation tasks. Implements +the full set of tools used by rh-sre skills: + - job_templates_list / job_templates_retrieve + - projects_list + - job_templates_launch_retrieve + - jobs_retrieve / jobs_stdout_retrieve + - jobs_job_events_list / jobs_job_host_summaries_list + - jobs_relaunch_retrieve + - inventories_list / hosts_list + +Data mirrors a realistic AAP deployment: + - 6 job templates (3 remediation, 1 compliance, 1 patching, 1 reporting) + - 3 projects (remediation, compliance, reporting) + - 3 inventories (production 30 hosts, staging 15 hosts, all-managed 63 hosts) + - 12 recent jobs with varied statuses + +Follows the same mock-server pattern as mock-lightspeed-mcp.py. +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +mcp = FastMCP("aap-mcp") + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +def _ts(delta: timedelta) -> str: + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +# --------------------------------------------------------------------------- +# Mock data: Projects +# --------------------------------------------------------------------------- + +MOCK_PROJECTS = [ + { + "id": 6, + "type": "project", + "name": "Remediation Playbooks", + "description": "CVE and security remediation playbooks managed via Git", + "scm_type": "git", + "scm_url": "https://github.com/org/remediation-playbooks.git", + "scm_branch": "main", + "scm_revision": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + "status": "successful", + "last_job_run": _ts(timedelta(hours=2)), + "last_update_failed": False, + "created": _ts(timedelta(days=90)), + "modified": _ts(timedelta(hours=2)), + }, + { + "id": 7, + "type": "project", + "name": "Compliance Checks", + "description": "STIG and CIS compliance scanning playbooks", + "scm_type": "git", + "scm_url": "https://github.com/org/compliance-playbooks.git", + "scm_branch": "main", + "scm_revision": "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3", + "status": "successful", + "last_job_run": _ts(timedelta(days=1)), + "last_update_failed": False, + "created": _ts(timedelta(days=120)), + "modified": _ts(timedelta(days=1)), + }, + { + "id": 8, + "type": "project", + "name": "Fleet Reporting", + "description": "System inventory and health reporting playbooks", + "scm_type": "git", + "scm_url": "https://github.com/org/fleet-reports.git", + "scm_branch": "main", + "scm_revision": "c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "status": "successful", + "last_job_run": _ts(timedelta(days=3)), + "last_update_failed": False, + "created": _ts(timedelta(days=180)), + "modified": _ts(timedelta(days=3)), + }, +] + +# --------------------------------------------------------------------------- +# Mock data: Inventories & Hosts +# --------------------------------------------------------------------------- + +MOCK_INVENTORIES = [ + { + "id": 1, + "type": "inventory", + "name": "Production Systems", + "description": "All production RHEL systems across data centers", + "total_hosts": 30, + "has_active_failures": False, + "hosts_with_active_failures": 0, + "total_groups": 5, + "groups_with_active_failures": 0, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=365)), + "modified": _ts(timedelta(days=1)), + }, + { + "id": 2, + "type": "inventory", + "name": "Staging Systems", + "description": "Pre-production staging environment", + "total_hosts": 15, + "has_active_failures": False, + "hosts_with_active_failures": 0, + "total_groups": 3, + "groups_with_active_failures": 0, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=300)), + "modified": _ts(timedelta(days=7)), + }, + { + "id": 3, + "type": "inventory", + "name": "All Managed Systems", + "description": "Complete fleet: production, staging, development, QA, legacy", + "total_hosts": 63, + "has_active_failures": True, + "hosts_with_active_failures": 2, + "total_groups": 8, + "groups_with_active_failures": 1, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=365)), + "modified": _ts(timedelta(hours=6)), + }, +] + + +def _generate_hosts(inventory_id: int) -> list[dict]: + """Generate realistic hosts for an inventory.""" + hosts: list[dict] = [] + if inventory_id == 1: + roles = ["web", "db", "app", "lb", "monitoring", "cache"] + for i, role in enumerate(roles): + for j in range(5 if role in ("web", "app") else 4 if role == "db" else 3 if role == "monitoring" else 2): + hosts.append({ + "id": len(hosts) + 1, + "type": "host", + "name": f"{role}-{j+1:02d}.prod.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "production", "role": "{role}"}}', + }) + if len(hosts) >= 30: + break + if len(hosts) >= 30: + break + elif inventory_id == 2: + for i in range(15): + role = ["web", "db", "app"][i % 3] + hosts.append({ + "id": 100 + i, + "type": "host", + "name": f"{role}-{i+1:02d}.staging.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "staging", "role": "{role}"}}', + }) + elif inventory_id == 3: + for i in range(30): + hosts.append({ + "id": 200 + i, + "type": "host", + "name": f"host-{i+1:02d}.prod.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": i in (45, 58), + "variables": f'{{"rhel_version": "9.3", "environment": "production"}}', + }) + for i in range(15): + hosts.append({ + "id": 230 + i, + "type": "host", + "name": f"host-{i+1:02d}.staging.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "staging"}}', + }) + for i in range(10): + hosts.append({ + "id": 245 + i, + "type": "host", + "name": f"dev-{i+1:02d}.dev.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "8.9", "environment": "development"}}', + }) + for i in range(5): + hosts.append({ + "id": 255 + i, + "type": "host", + "name": f"qa-{i+1:02d}.qa.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.2", "environment": "qa"}}', + }) + for i in range(3): + hosts.append({ + "id": 260 + i, + "type": "host", + "name": f"legacy-{i+1:02d}.corp.example.com", + "inventory": inventory_id, + "enabled": i < 2, + "has_active_failures": i == 2, + "variables": f'{{"rhel_version": "7.9", "environment": "legacy"}}', + }) + return hosts + + +# --------------------------------------------------------------------------- +# Mock data: Job Templates +# --------------------------------------------------------------------------- + +MOCK_JOB_TEMPLATES = [ + { + "id": 10, + "type": "job_template", + "name": "CVE Remediation - Kernel Update", + "description": "Kernel update with boom snapshot for rollback safety", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": True, + "job_type": "check", + "verbosity": 1, + "timeout": 3600, + "forks": 5, + "status": "successful", + "last_job_run": _ts(timedelta(hours=4)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1001, "status": "successful", "finished": _ts(timedelta(hours=4))}, + }, + "created": _ts(timedelta(days=60)), + "modified": _ts(timedelta(days=2)), + }, + { + "id": 11, + "type": "job_template", + "name": "CVE Remediation - Package Update", + "description": "General package update for CVE remediation with needs-restarting check", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-package-update.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": False, + "job_type": "check", + "verbosity": 1, + "timeout": 1800, + "forks": 10, + "status": "successful", + "last_job_run": _ts(timedelta(hours=12)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1005, "status": "successful", "finished": _ts(timedelta(hours=12))}, + }, + "created": _ts(timedelta(days=45)), + "modified": _ts(timedelta(days=5)), + }, + { + "id": 12, + "type": "job_template", + "name": "CVE Remediation - Generic", + "description": "Generic CVE remediation template for ad-hoc patches", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-remediation.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": True, + "job_type": "check", + "verbosity": 1, + "timeout": 3600, + "forks": 5, + "status": "never updated", + "last_job_run": None, + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + }, + "created": _ts(timedelta(days=30)), + "modified": _ts(timedelta(days=30)), + }, + { + "id": 20, + "type": "job_template", + "name": "Compliance Check - STIG", + "description": "Run STIG compliance scan across fleet", + "inventory": 3, + "project": 7, + "playbook": "playbooks/compliance/check-all.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": False, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 7200, + "forks": 20, + "status": "successful", + "last_job_run": _ts(timedelta(days=1)), + "summary_fields": { + "project": {"id": 7, "name": "Compliance Checks", "status": "successful"}, + "inventory": {"id": 3, "name": "All Managed Systems", "total_hosts": 63}, + "credentials": [ + {"id": 2, "name": "compliance-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1010, "status": "successful", "finished": _ts(timedelta(days=1))}, + }, + "created": _ts(timedelta(days=180)), + "modified": _ts(timedelta(days=14)), + }, + { + "id": 25, + "type": "job_template", + "name": "Emergency Patching", + "description": "Emergency patch application — NO become enabled (misconfigured)", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/emergency-patch.yml", + "become_enabled": False, + "ask_job_type_on_launch": False, + "ask_variables_on_launch": False, + "ask_limit_on_launch": False, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 600, + "forks": 25, + "status": "failed", + "last_job_run": _ts(timedelta(days=7)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1020, "status": "failed", "finished": _ts(timedelta(days=7))}, + }, + "created": _ts(timedelta(days=200)), + "modified": _ts(timedelta(days=200)), + }, + { + "id": 30, + "type": "job_template", + "name": "Fleet Health Report", + "description": "Generate fleet health and inventory report", + "inventory": 3, + "project": 8, + "playbook": "playbooks/reporting/fleet-health.yml", + "become_enabled": False, + "ask_job_type_on_launch": False, + "ask_variables_on_launch": True, + "ask_limit_on_launch": False, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 1800, + "forks": 30, + "status": "successful", + "last_job_run": _ts(timedelta(hours=6)), + "summary_fields": { + "project": {"id": 8, "name": "Fleet Reporting", "status": "successful"}, + "inventory": {"id": 3, "name": "All Managed Systems", "total_hosts": 63}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1025, "status": "successful", "finished": _ts(timedelta(hours=6))}, + }, + "created": _ts(timedelta(days=120)), + "modified": _ts(timedelta(days=14)), + }, +] + +# --------------------------------------------------------------------------- +# Mock data: Jobs (recent runs) +# --------------------------------------------------------------------------- + +PROD_HOSTS = [ + "web-01.prod.example.com", + "web-02.prod.example.com", + "db-01.prod.example.com", + "db-02.prod.example.com", + "app-01.prod.example.com", + "app-02.prod.example.com", +] + +MOCK_JOBS = [ + { + "id": 1001, + "type": "job", + "name": "CVE Remediation - Kernel Update", + "job_type": "check", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=4, minutes=30)), + "finished": _ts(timedelta(hours=4)), + "elapsed": 1800.0, + "job_template": 10, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "limit": "web-01.prod.example.com,web-02.prod.example.com,db-01.prod.example.com", + "extra_vars": '{"target_cve": "CVE-2024-12345", "remediation_mode": "automated", "verify_after": true}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 10, "name": "CVE Remediation - Kernel Update"}, + }, + }, + { + "id": 1002, + "type": "job", + "name": "CVE Remediation - Kernel Update", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=3, minutes=45)), + "finished": _ts(timedelta(hours=3)), + "elapsed": 2700.0, + "job_template": 10, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "limit": "web-01.prod.example.com,web-02.prod.example.com,db-01.prod.example.com", + "extra_vars": '{"target_cve": "CVE-2024-12345", "remediation_mode": "automated", "verify_after": true}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 10, "name": "CVE Remediation - Kernel Update"}, + }, + }, + { + "id": 1005, + "type": "job", + "name": "CVE Remediation - Package Update", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=12, minutes=20)), + "finished": _ts(timedelta(hours=12)), + "elapsed": 1200.0, + "job_template": 11, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-package-update.yml", + "limit": "", + "extra_vars": '{"target_cve": "CVE-2024-54321"}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 11, "name": "CVE Remediation - Package Update"}, + }, + }, + { + "id": 1010, + "type": "job", + "name": "Compliance Check - STIG", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(days=1, hours=2)), + "finished": _ts(timedelta(days=1)), + "elapsed": 7200.0, + "job_template": 20, + "inventory": 3, + "project": 7, + "playbook": "playbooks/compliance/check-all.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "scheduled", + "summary_fields": { + "job_template": {"id": 20, "name": "Compliance Check - STIG"}, + }, + }, + { + "id": 1020, + "type": "job", + "name": "Emergency Patching", + "job_type": "run", + "status": "failed", + "failed": True, + "started": _ts(timedelta(days=7, hours=1)), + "finished": _ts(timedelta(days=7)), + "elapsed": 3600.0, + "job_template": 25, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/emergency-patch.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 25, "name": "Emergency Patching"}, + }, + }, + { + "id": 1025, + "type": "job", + "name": "Fleet Health Report", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=6, minutes=30)), + "finished": _ts(timedelta(hours=6)), + "elapsed": 1800.0, + "job_template": 30, + "inventory": 3, + "project": 8, + "playbook": "playbooks/reporting/fleet-health.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "scheduled", + "summary_fields": { + "job_template": {"id": 30, "name": "Fleet Health Report"}, + }, + }, +] + +_next_job_id = 2000 + + +# --------------------------------------------------------------------------- +# Mock stdout generators +# --------------------------------------------------------------------------- + +def _generate_stdout(job: dict) -> str: + """Generate realistic Ansible playbook stdout for a job.""" + playbook_name = job.get("name", "Unknown") + job_type = job.get("job_type", "run") + status = job.get("status", "successful") + limit = job.get("limit", "") + hosts = limit.split(",") if limit else PROD_HOSTS[:3] + hosts = [h.strip() for h in hosts if h.strip()] + extra_vars = job.get("extra_vars", "{}") + mode = " (CHECK MODE)" if job_type == "check" else "" + + lines = [] + lines.append(f"PLAY [{playbook_name}] *****") + lines.append("") + + lines.append(f"TASK [Gathering Facts{mode}] *****") + for h in hosts: + lines.append(f"ok: [{h}]") + lines.append("") + + if "kernel" in playbook_name.lower(): + lines.append(f"TASK [Create boom snapshot for rollback{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}] => {{\"msg\": \"boom create --title pre-remediation-CVE-2024-12345\"}}") + lines.append("") + + lines.append(f"TASK [Check disk space for kernel update{mode}] *****") + for h in hosts: + lines.append(f"ok: [{h}] => {{\"msg\": \"Disk space OK: 45% used\"}}") + lines.append("") + + lines.append(f"TASK [Update kernel package{mode}] *****") + for h in hosts: + result = "changed" if status == "successful" else "fatal" + if result == "changed": + lines.append(f'changed: [{h}] => {{"msg": "kernel-5.14.0-362.24.1.el9_3 -> kernel-5.14.0-362.24.2.el9_3"}}') + else: + lines.append(f'fatal: [{h}]: FAILED! => {{"msg": "Permission denied", "rc": 1}}') + lines.append("") + + lines.append(f"TASK [Check if reboot is needed (needs-restarting -r){mode}] *****") + for h in hosts: + lines.append(f'changed: [{h}] => {{"rc": 1, "msg": "Reboot is required to fully utilize updates."}}') + lines.append("") + + elif "package" in playbook_name.lower(): + lines.append(f"TASK [Update target packages for CVE remediation{mode}] *****") + for h in hosts: + lines.append(f'changed: [{h}] => {{"msg": "httpd-2.4.53-7.el9 -> httpd-2.4.57-8.el9"}}') + lines.append("") + + lines.append(f"TASK [Restart affected services{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}]") + lines.append("") + + lines.append(f"TASK [Verify service health{mode}] *****") + for h in hosts: + lines.append(f'ok: [{h}] => {{"msg": "Service httpd is running"}}') + lines.append("") + + elif "emergency" in playbook_name.lower() and status == "failed": + lines.append(f"TASK [Apply emergency patch{mode}] *****") + for h in hosts: + lines.append(f'fatal: [{h}]: FAILED! => {{"msg": "Missing sudo password (become_enabled not set)", "rc": 1}}') + lines.append("") + lines.append("NO MORE HOSTS LEFT *****") + lines.append("") + + else: + lines.append(f"TASK [Execute playbook tasks{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}]") + lines.append("") + + lines.append("PLAY RECAP *****") + for h in hosts: + if status == "successful": + ok_count = random.randint(3, 6) + changed_count = random.randint(1, 3) + lines.append(f"{h:<45} : ok={ok_count} changed={changed_count} unreachable=0 failed=0 skipped=0 rescued=0 ignored=0") + elif status == "failed": + lines.append(f"{h:<45} : ok=1 changed=0 unreachable=0 failed=1 skipped=0 rescued=0 ignored=0") + lines.append("") + + return "\n".join(lines) + + +def _generate_events(job: dict) -> list[dict]: + """Generate realistic Ansible task events for a job.""" + hosts = (job.get("limit", "").split(",") if job.get("limit") else PROD_HOSTS[:3]) + hosts = [h.strip() for h in hosts if h.strip()] + events: list[dict] = [] + eid = 1 + + task_names = ["Gathering Facts"] + if "kernel" in job.get("name", "").lower(): + task_names += [ + "Create boom snapshot for rollback", + "Check disk space for kernel update", + "Update kernel package", + "Check if reboot is needed (needs-restarting -r)", + ] + elif "package" in job.get("name", "").lower(): + task_names += [ + "Update target packages for CVE remediation", + "Restart affected services", + "Verify service health", + ] + else: + task_names += ["Execute playbook tasks"] + + for task_name in task_names: + for host in hosts: + is_failed = job.get("status") == "failed" and task_name != "Gathering Facts" + events.append({ + "id": eid, + "type": "job_event", + "event": "runner_on_ok" if not is_failed else "runner_on_failed", + "task": task_name, + "host": host, + "host_name": host, + "play": job.get("name", ""), + "changed": task_name != "Gathering Facts" and not is_failed, + "failed": is_failed, + "event_data": { + "task": task_name, + "host": host, + "res": { + "changed": task_name != "Gathering Facts" and not is_failed, + "msg": "Task completed" if not is_failed else "Permission denied", + }, + }, + "created": _ts(timedelta(hours=4, minutes=30 - eid)), + }) + eid += 1 + + return events + + +def _generate_host_summaries(job: dict) -> list[dict]: + """Generate per-host summaries for a job.""" + hosts = (job.get("limit", "").split(",") if job.get("limit") else PROD_HOSTS[:3]) + hosts = [h.strip() for h in hosts if h.strip()] + summaries: list[dict] = [] + + for i, host in enumerate(hosts): + is_failed = job.get("status") == "failed" + summaries.append({ + "id": i + 1, + "type": "job_host_summary", + "host": i + 1, + "host_name": host, + "ok": 1 if is_failed else random.randint(3, 6), + "changed": 0 if is_failed else random.randint(1, 3), + "dark": 0, + "failures": 1 if is_failed else 0, + "skipped": 0, + "processed": 1, + "failed": is_failed, + }) + + return summaries + + +# --------------------------------------------------------------------------- +# MCP Tools: Job Management +# --------------------------------------------------------------------------- + +@mcp.tool() +def job_templates_list( + page_size: int = 10, + search: Optional[str] = None, +) -> dict: + """List available job templates in AAP. + + Args: + page_size: Number of results per page (default 10, max 200). + search: Optional search string to filter templates by name. + """ + results = MOCK_JOB_TEMPLATES + if search: + s = search.lower() + results = [t for t in results if s in t["name"].lower() or s in t.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def job_templates_retrieve(id: str) -> dict: + """Retrieve detailed information about a specific job template. + + Args: + id: Job template ID (as string). + """ + tid = int(id) + template = next((t for t in MOCK_JOB_TEMPLATES if t["id"] == tid), None) + if not template: + return {"detail": f"Not found. Job template {id} does not exist."} + return template + + +@mcp.tool() +def projects_list( + page_size: int = 50, + search: Optional[str] = None, +) -> dict: + """List available projects in AAP. + + Args: + page_size: Number of results per page. + search: Optional search string to filter projects by name. + """ + results = MOCK_PROJECTS + if search: + s = search.lower() + results = [p for p in results if s in p["name"].lower() or s in p.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def job_templates_launch_retrieve( + id: str, + requestBody: Optional[dict] = None, +) -> dict: + """Launch a job from a job template. + + Args: + id: Job template ID to launch. + requestBody: Optional launch parameters including job_type ('run' or 'check'), + extra_vars (dict), and limit (comma-separated host list). + """ + global _next_job_id + tid = int(id) + template = next((t for t in MOCK_JOB_TEMPLATES if t["id"] == tid), None) + if not template: + return {"detail": f"Not found. Job template {id} does not exist."} + + body = requestBody or {} + job_type = body.get("job_type", template.get("job_type", "run")) + + if not template.get("ask_job_type_on_launch") and job_type != template.get("job_type"): + return { + "error": f"Cannot override job_type: ask_job_type_on_launch is disabled on template {id}", + } + + job_id = _next_job_id + _next_job_id += 1 + + new_job = { + "id": job_id, + "type": "job", + "name": template["name"], + "job_type": job_type, + "status": "pending", + "failed": False, + "started": _ts(timedelta(seconds=0)), + "finished": None, + "elapsed": 0.0, + "job_template": tid, + "inventory": template["inventory"], + "project": template["project"], + "playbook": template["playbook"], + "limit": body.get("limit", ""), + "extra_vars": str(body.get("extra_vars", {})), + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": tid, "name": template["name"]}, + }, + } + MOCK_JOBS.append(new_job) + + # Simulate job completion after launch + new_job["status"] = "successful" + new_job["finished"] = _ts(timedelta(seconds=-300)) + new_job["elapsed"] = 300.0 + + return { + "job": job_id, + "status": "pending", + "type": "job", + "url": f"/api/controller/v2/jobs/{job_id}/", + "related": { + "stdout": f"/api/controller/v2/jobs/{job_id}/stdout/", + "job_events": f"/api/controller/v2/jobs/{job_id}/job_events/", + "job_host_summaries": f"/api/controller/v2/jobs/{job_id}/job_host_summaries/", + }, + } + + +@mcp.tool() +def jobs_retrieve(id: int) -> dict: + """Get the status and details of a job run. + + Args: + id: Job ID to retrieve. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + return job + + +@mcp.tool() +def jobs_list(page_size: int = 10) -> dict: + """List recent job runs. + + Args: + page_size: Number of results to return. + """ + results = sorted(MOCK_JOBS, key=lambda j: j.get("started", ""), reverse=True) + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def jobs_stdout_retrieve(id: int, format: str = "txt") -> dict: + """Get the stdout (console output) from a job run. + + Args: + id: Job ID. + format: Output format ('txt' or 'json'). Default 'txt'. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + return { + "content": _generate_stdout(job), + "range": {"start": 0, "end": 1}, + } + + +@mcp.tool() +def jobs_job_events_list(id: int, page_size: int = 50) -> dict: + """Get task-level events for a job run. + + Args: + id: Job ID. + page_size: Number of events to return. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + events = _generate_events(job) + return { + "count": len(events), + "next": None, + "previous": None, + "results": events[:page_size], + } + + +@mcp.tool() +def jobs_job_host_summaries_list(id: int) -> dict: + """Get per-host execution summaries for a job run. + + Args: + id: Job ID. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + summaries = _generate_host_summaries(job) + return { + "count": len(summaries), + "next": None, + "previous": None, + "results": summaries, + } + + +@mcp.tool() +def jobs_relaunch_retrieve( + id: int, + hosts: str = "all", + job_type: str = "run", +) -> dict: + """Relaunch a previously completed or failed job. + + Args: + id: Original job ID to relaunch. + hosts: Which hosts to target ('all' or 'failed'). + job_type: Job type for relaunch ('run' or 'check'). + """ + global _next_job_id + original = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not original: + return {"detail": f"Not found. Job {id} does not exist."} + + new_id = _next_job_id + _next_job_id += 1 + + new_job = { + **original, + "id": new_id, + "job_type": job_type, + "status": "successful", + "failed": False, + "started": _ts(timedelta(seconds=0)), + "finished": _ts(timedelta(seconds=-300)), + "elapsed": 300.0, + "launch_type": "relaunch", + } + MOCK_JOBS.append(new_job) + + return { + "job": new_id, + "status": "pending", + "type": "job", + "url": f"/api/controller/v2/jobs/{new_id}/", + } + + +# --------------------------------------------------------------------------- +# MCP Tools: Inventory Management +# --------------------------------------------------------------------------- + +@mcp.tool() +def inventories_list( + page_size: int = 10, + search: Optional[str] = None, +) -> dict: + """List available inventories in AAP. + + Args: + page_size: Number of results per page. + search: Optional search string to filter inventories. + """ + results = MOCK_INVENTORIES + if search: + s = search.lower() + results = [inv for inv in results if s in inv["name"].lower() or s in inv.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def hosts_list( + inventory_id: Optional[int] = None, + page_size: int = 50, + search: Optional[str] = None, +) -> dict: + """List hosts in an inventory. + + Args: + inventory_id: Filter by inventory ID. If not provided, lists hosts from all inventories. + page_size: Number of results per page. + search: Optional search string to filter hosts by name. + """ + inv_id = inventory_id or 1 + hosts = _generate_hosts(inv_id) + if search: + s = search.lower() + hosts = [h for h in hosts if s in h["name"].lower()] + return { + "count": len(hosts), + "next": None if len(hosts) <= page_size else f"/api/controller/v2/hosts/?page=2", + "previous": None, + "results": hosts[:page_size], + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..fe5d549c --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/instruction.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/instruction.md new file mode 100644 index 00000000..54d1a0e6 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/instruction.md @@ -0,0 +1,16 @@ +# AAP Connectivity Check Task + +You are a Red Hat SRE. Before starting a remediation workflow that depends on Ansible Automation Platform, you need to verify that the AAP integration is working correctly. + +## Scenario +You are about to run a remediation workflow that uses AAP to execute playbooks. First, you need to confirm that the AAP connection is healthy and that you can access the necessary resources. + +## Requirements +- Test connectivity to the AAP server by querying job templates and inventories +- Verify that the response is valid and contains expected data +- If any connection fails, document the error and provide troubleshooting guidance (credentials, network, SSL, permissions) +- Report the overall AAP readiness status: which capabilities are available and which are not + +Document your methodology, connectivity check results, and troubleshooting findings in `/root/report.md`. + +Use MCP tools to interact with AAP. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/solution/solve.sh b/evaluation/without_skills/rh-sre__mcp-aap-validator/solution/solve.sh new file mode 100644 index 00000000..88542def --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/solution/solve.sh @@ -0,0 +1,25 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# AAP MCP Validation + +## Test Calls +- `job_templates_list(page_size: 10)` from aap-mcp-job-management ✅ +- `inventories_list(page_size: 10)` from aap-mcp-inventory-management ✅ + +## Result +| Server | Outcome | +|--------|---------| +| aap-mcp-job-management | ✅ PASSED | +| aap-mcp-inventory-management | ✅ PASSED | + +## Diagnostics +| Code | Meaning | +|------|---------| +| 401 | Token expired or invalid → regenerate in AAP Web UI → Users → Tokens | +| 403 | Insufficient RBAC (need Job Templates, Inventories) | +| 404 | Wrong URL — AAP_MCP_SERVER must point to MCP gateway, not main AAP UI | + +## Environment +- AAP_MCP_SERVER: Set (must point to MCP gateway) +- AAP_API_TOKEN: Set +REPORT_EOF diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/task.toml b/evaluation/without_skills/rh-sre__mcp-aap-validator/task.toml new file mode 100644 index 00000000..aad389ea --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__mcp-aap-validator" +name = "rh-sre AAP MCP Validation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "mcp-aap-validator", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/tests/llm_judge.py b/evaluation/without_skills/rh-sre__mcp-aap-validator/tests/llm_judge.py new file mode 100644 index 00000000..474598a6 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "gateway_vs_ui_url", "file": "/root/report.md", "question": "Does the report note that AAP_MCP_SERVER must point to the MCP gateway endpoint, not the main AAP UI URL, and that 404 indicates wrong URL?", "reference": "A skilled report explains the gateway/UI URL distinction and maps 404 to wrong URL. An unskilled report doesn't distinguish these endpoints."}, + {"id": "both_servers_tested", "file": "/root/report.md", "question": "Does the report test both job_templates_list and inventories_list for AAP MCP validation?", "reference": "A skilled report validates both MCP servers. An unskilled report tests only one."}, + {"id": "structured_outcome", "file": "/root/report.md", "question": "Does the report present per-server validation outcomes (PASSED/FAILED/PARTIAL) in table format?", "reference": "A skilled report uses structured table with per-server status. An unskilled report uses unstructured text."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/tests/test.sh b/evaluation/without_skills/rh-sre__mcp-aap-validator/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/tests/test_outputs.py b/evaluation/without_skills/rh-sre__mcp-aap-validator/tests/test_outputs.py new file mode 100644 index 00000000..615713b5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-aap-validator/tests/test_outputs.py @@ -0,0 +1,66 @@ +""" +Tests for rh-sre__mcp-aap-validator per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['aap', 'mcp', 'valid', 'connect']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_both_servers_tested(self): + """Skill: Test BOTH job_templates_list (job-management) AND inventories_list (inventory-management).""" + c = read_report().lower() + has_job = any(t in c for t in ["job_template", "job template", "job-management"]) + has_inv = any(t in c for t in ["inventor", "inventory-management"]) + assert has_job or has_inv, ( + "should test both AAP MCP servers (skill: job-management + inventory-management)" + ) + + def test_mcp_gateway_not_ui(self): + """Skill teaches AAP_MCP_SERVER must point to MCP gateway endpoint, not main AAP UI URL.""" + c = read_report().lower() + assert ("gateway" in c and "mcp" in c) or "aap_mcp_server" in c, ( + "should note AAP_MCP_SERVER must point to MCP gateway, not UI (skill: wrong URL = 404)" + ) + + def test_404_wrong_url(self): + """Skill teaches HTTP 404 = wrong AAP_MCP_SERVER URL.""" + c = read_report().lower() + assert "404" in c and any(t in c for t in ["url", "wrong"]), ( + "should explain 404 indicates wrong URL (skill: troubleshooting)" + ) + + def test_table_format(self): + """Skill: Output table with Server | Outcome (PASSED/FAILED/PARTIAL).""" + content = read_report() + c = content.lower() + has_table = "|" in content + has_outcome = any(t in c for t in ["passed", "failed", "partial", "job_templates_list", "inventories_list"]) + assert has_table or has_outcome, ( + "should use table format with outcome (skill: Report Format)" + ) diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/Dockerfile b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/Dockerfile new file mode 100644 index 00000000..5a2cfdee --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/Dockerfile @@ -0,0 +1,43 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..6c122770 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- Pass Rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..fe5d549c --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/instruction.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/instruction.md new file mode 100644 index 00000000..37d450b8 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/instruction.md @@ -0,0 +1,16 @@ +# Lightspeed Connectivity Check Task + +You are a Red Hat SRE. Before querying CVE data or generating remediation playbooks, you need to verify that the Red Hat Insights/Lightspeed integration is working correctly. + +## Scenario +You are about to start a CVE investigation that depends on querying vulnerability data from Red Hat Insights. First, you need to confirm the Lightspeed connection is healthy and returning valid data. + +## Requirements +- Test connectivity to the Lightspeed service by querying CVE data +- Verify the response is valid and contains expected vulnerability information +- If the connection fails, document the error and provide troubleshooting guidance (expired tokens, credentials, network issues, server availability) +- Report the overall Lightspeed readiness status + +Document your methodology, connectivity check results, and troubleshooting findings in `/root/report.md`. + +Use MCP tools to interact with the Lightspeed service. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/solution/solve.sh b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/solution/solve.sh new file mode 100644 index 00000000..8336f1ee --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/solution/solve.sh @@ -0,0 +1,29 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Lightspeed MCP Validation + +## Test: Call vulnerability__get_cves with no parameters +- Do NOT pass `limit` parameter (serialization issue: `limit` → `limit_`) +- Default limit=10 is applied automatically + +## Result +| Server | Outcome | +|--------|---------| +| lightspeed-mcp | ✅ PASSED | + +## Failure Root Causes (when connection fails) +- **Credentials**: LIGHTSPEED_CLIENT_ID or LIGHTSPEED_CLIENT_SECRET not set or invalid +- **Expired credentials**: Red Hat Console tokens may have expired +- **Server not running**: MCP server/container may be stopped +- **Network**: Firewall or proxy blocking console.redhat.com +- **Configuration**: .mcp.json misconfigured or server not registered + +## Troubleshooting +1. Verify env vars: LIGHTSPEED_CLIENT_ID, LIGHTSPEED_CLIENT_SECRET (never echo values) +2. Check credentials at: https://console.redhat.com/settings/integrations +3. Restart MCP server or host after config changes + +## Environment +- LIGHTSPEED_CLIENT_ID: Set +- LIGHTSPEED_CLIENT_SECRET: Set +REPORT_EOF diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/task.toml b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/task.toml new file mode 100644 index 00000000..1e356701 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__mcp-lightspeed-validator" +name = "rh-sre Lightspeed MCP Validation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "mcp-lightspeed-validator", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/tests/llm_judge.py b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/tests/llm_judge.py new file mode 100644 index 00000000..905e9250 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "no_params_get_cves", "file": "/root/report.md", "question": "Does the report call get_cves with no parameters (due to limit_ serialization bug)?", "reference": "A skilled report avoids passing limit parameter. An unskilled report passes limit which may break the call."}, + {"id": "credential_handling", "file": "/root/report.md", "question": "Does the report reference LIGHTSPEED_CLIENT_ID/CLIENT_SECRET env vars and warn against echoing credentials?", "reference": "A skilled report identifies the correct env vars and warns about credential exposure. An unskilled report doesn't know the specific variable names."}, + {"id": "validation_structure", "file": "/root/report.md", "question": "Does the report present Lightspeed MCP validation in structured table format?", "reference": "A skilled report uses table with PASSED/FAILED outcome. An unskilled report uses unstructured text."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/tests/test.sh b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/tests/test_outputs.py b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/tests/test_outputs.py new file mode 100644 index 00000000..05e6bf9b --- /dev/null +++ b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/tests/test_outputs.py @@ -0,0 +1,64 @@ +""" +Tests for rh-sre__mcp-lightspeed-validator per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['lightspeed', 'mcp', 'valid']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_get_cves_no_params(self): + """Skill: Call vulnerability__get_cves with NO parameters (limit causes limit_ serialization bug).""" + c = read_report().lower() + assert any(t in c for t in ["no param", "without param", "limit_"]), ( + "should call get_cves without parameters (skill: passing limit breaks some clients)" + ) + + def test_lightspeed_credentials(self): + """Skill: LIGHTSPEED_CLIENT_ID + LIGHTSPEED_CLIENT_SECRET are the env vars.""" + c = read_report().lower() + assert any(t in c for t in ["lightspeed_client_id", "client_id", "client_secret"]), ( + "should reference Lightspeed credential env vars (skill: LIGHTSPEED_CLIENT_ID/SECRET)" + ) + + def test_never_echo_credentials(self): + """Skill: Never echo or log credential values.""" + c = read_report().lower() + has_security = any(t in c for t in ["never echo", "do not echo", "redact", "sensitive", "protect"]) + assert has_security or "credential" in c, ( + "should address credential handling (skill: never echo values)" + ) + + def test_table_format(self): + """Skill: Output table with Server | Outcome.""" + c = read_report().lower() + has_table = "|" in read_report() + has_outcome = any(t in c for t in ["passed", "failed", "get_cves", "lightspeed"]) + assert has_table or has_outcome, ( + "should use table format (skill: Report Format)" + ) diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/Dockerfile b/evaluation/without_skills/rh-sre__playbook-executor/environment/Dockerfile new file mode 100644 index 00000000..51ce02e5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/Dockerfile @@ -0,0 +1,47 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + }, \ + "aap-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-aap-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..6c122770 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- Pass Rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/mcp-servers/mock-aap-mcp.py b/evaluation/without_skills/rh-sre__playbook-executor/environment/mcp-servers/mock-aap-mcp.py new file mode 100644 index 00000000..d8ae4fd5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/mcp-servers/mock-aap-mcp.py @@ -0,0 +1,1048 @@ +#!/usr/bin/env python3 +""" +Mock AAP (Ansible Automation Platform) MCP Server + +Simulates the AAP MCP gateway for per-skill evaluation tasks. Implements +the full set of tools used by rh-sre skills: + - job_templates_list / job_templates_retrieve + - projects_list + - job_templates_launch_retrieve + - jobs_retrieve / jobs_stdout_retrieve + - jobs_job_events_list / jobs_job_host_summaries_list + - jobs_relaunch_retrieve + - inventories_list / hosts_list + +Data mirrors a realistic AAP deployment: + - 6 job templates (3 remediation, 1 compliance, 1 patching, 1 reporting) + - 3 projects (remediation, compliance, reporting) + - 3 inventories (production 30 hosts, staging 15 hosts, all-managed 63 hosts) + - 12 recent jobs with varied statuses + +Follows the same mock-server pattern as mock-lightspeed-mcp.py. +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +mcp = FastMCP("aap-mcp") + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +def _ts(delta: timedelta) -> str: + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +# --------------------------------------------------------------------------- +# Mock data: Projects +# --------------------------------------------------------------------------- + +MOCK_PROJECTS = [ + { + "id": 6, + "type": "project", + "name": "Remediation Playbooks", + "description": "CVE and security remediation playbooks managed via Git", + "scm_type": "git", + "scm_url": "https://github.com/org/remediation-playbooks.git", + "scm_branch": "main", + "scm_revision": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + "status": "successful", + "last_job_run": _ts(timedelta(hours=2)), + "last_update_failed": False, + "created": _ts(timedelta(days=90)), + "modified": _ts(timedelta(hours=2)), + }, + { + "id": 7, + "type": "project", + "name": "Compliance Checks", + "description": "STIG and CIS compliance scanning playbooks", + "scm_type": "git", + "scm_url": "https://github.com/org/compliance-playbooks.git", + "scm_branch": "main", + "scm_revision": "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3", + "status": "successful", + "last_job_run": _ts(timedelta(days=1)), + "last_update_failed": False, + "created": _ts(timedelta(days=120)), + "modified": _ts(timedelta(days=1)), + }, + { + "id": 8, + "type": "project", + "name": "Fleet Reporting", + "description": "System inventory and health reporting playbooks", + "scm_type": "git", + "scm_url": "https://github.com/org/fleet-reports.git", + "scm_branch": "main", + "scm_revision": "c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "status": "successful", + "last_job_run": _ts(timedelta(days=3)), + "last_update_failed": False, + "created": _ts(timedelta(days=180)), + "modified": _ts(timedelta(days=3)), + }, +] + +# --------------------------------------------------------------------------- +# Mock data: Inventories & Hosts +# --------------------------------------------------------------------------- + +MOCK_INVENTORIES = [ + { + "id": 1, + "type": "inventory", + "name": "Production Systems", + "description": "All production RHEL systems across data centers", + "total_hosts": 30, + "has_active_failures": False, + "hosts_with_active_failures": 0, + "total_groups": 5, + "groups_with_active_failures": 0, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=365)), + "modified": _ts(timedelta(days=1)), + }, + { + "id": 2, + "type": "inventory", + "name": "Staging Systems", + "description": "Pre-production staging environment", + "total_hosts": 15, + "has_active_failures": False, + "hosts_with_active_failures": 0, + "total_groups": 3, + "groups_with_active_failures": 0, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=300)), + "modified": _ts(timedelta(days=7)), + }, + { + "id": 3, + "type": "inventory", + "name": "All Managed Systems", + "description": "Complete fleet: production, staging, development, QA, legacy", + "total_hosts": 63, + "has_active_failures": True, + "hosts_with_active_failures": 2, + "total_groups": 8, + "groups_with_active_failures": 1, + "has_inventory_sources": True, + "organization": 1, + "created": _ts(timedelta(days=365)), + "modified": _ts(timedelta(hours=6)), + }, +] + + +def _generate_hosts(inventory_id: int) -> list[dict]: + """Generate realistic hosts for an inventory.""" + hosts: list[dict] = [] + if inventory_id == 1: + roles = ["web", "db", "app", "lb", "monitoring", "cache"] + for i, role in enumerate(roles): + for j in range(5 if role in ("web", "app") else 4 if role == "db" else 3 if role == "monitoring" else 2): + hosts.append({ + "id": len(hosts) + 1, + "type": "host", + "name": f"{role}-{j+1:02d}.prod.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "production", "role": "{role}"}}', + }) + if len(hosts) >= 30: + break + if len(hosts) >= 30: + break + elif inventory_id == 2: + for i in range(15): + role = ["web", "db", "app"][i % 3] + hosts.append({ + "id": 100 + i, + "type": "host", + "name": f"{role}-{i+1:02d}.staging.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "staging", "role": "{role}"}}', + }) + elif inventory_id == 3: + for i in range(30): + hosts.append({ + "id": 200 + i, + "type": "host", + "name": f"host-{i+1:02d}.prod.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": i in (45, 58), + "variables": f'{{"rhel_version": "9.3", "environment": "production"}}', + }) + for i in range(15): + hosts.append({ + "id": 230 + i, + "type": "host", + "name": f"host-{i+1:02d}.staging.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.3", "environment": "staging"}}', + }) + for i in range(10): + hosts.append({ + "id": 245 + i, + "type": "host", + "name": f"dev-{i+1:02d}.dev.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "8.9", "environment": "development"}}', + }) + for i in range(5): + hosts.append({ + "id": 255 + i, + "type": "host", + "name": f"qa-{i+1:02d}.qa.example.com", + "inventory": inventory_id, + "enabled": True, + "has_active_failures": False, + "variables": f'{{"rhel_version": "9.2", "environment": "qa"}}', + }) + for i in range(3): + hosts.append({ + "id": 260 + i, + "type": "host", + "name": f"legacy-{i+1:02d}.corp.example.com", + "inventory": inventory_id, + "enabled": i < 2, + "has_active_failures": i == 2, + "variables": f'{{"rhel_version": "7.9", "environment": "legacy"}}', + }) + return hosts + + +# --------------------------------------------------------------------------- +# Mock data: Job Templates +# --------------------------------------------------------------------------- + +MOCK_JOB_TEMPLATES = [ + { + "id": 10, + "type": "job_template", + "name": "CVE Remediation - Kernel Update", + "description": "Kernel update with boom snapshot for rollback safety", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": True, + "job_type": "check", + "verbosity": 1, + "timeout": 3600, + "forks": 5, + "status": "successful", + "last_job_run": _ts(timedelta(hours=4)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1001, "status": "successful", "finished": _ts(timedelta(hours=4))}, + }, + "created": _ts(timedelta(days=60)), + "modified": _ts(timedelta(days=2)), + }, + { + "id": 11, + "type": "job_template", + "name": "CVE Remediation - Package Update", + "description": "General package update for CVE remediation with needs-restarting check", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-package-update.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": False, + "job_type": "check", + "verbosity": 1, + "timeout": 1800, + "forks": 10, + "status": "successful", + "last_job_run": _ts(timedelta(hours=12)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1005, "status": "successful", "finished": _ts(timedelta(hours=12))}, + }, + "created": _ts(timedelta(days=45)), + "modified": _ts(timedelta(days=5)), + }, + { + "id": 12, + "type": "job_template", + "name": "CVE Remediation - Generic", + "description": "Generic CVE remediation template for ad-hoc patches", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-remediation.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": True, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": True, + "job_type": "check", + "verbosity": 1, + "timeout": 3600, + "forks": 5, + "status": "never updated", + "last_job_run": None, + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + }, + "created": _ts(timedelta(days=30)), + "modified": _ts(timedelta(days=30)), + }, + { + "id": 20, + "type": "job_template", + "name": "Compliance Check - STIG", + "description": "Run STIG compliance scan across fleet", + "inventory": 3, + "project": 7, + "playbook": "playbooks/compliance/check-all.yml", + "become_enabled": True, + "ask_job_type_on_launch": True, + "ask_variables_on_launch": False, + "ask_limit_on_launch": True, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 7200, + "forks": 20, + "status": "successful", + "last_job_run": _ts(timedelta(days=1)), + "summary_fields": { + "project": {"id": 7, "name": "Compliance Checks", "status": "successful"}, + "inventory": {"id": 3, "name": "All Managed Systems", "total_hosts": 63}, + "credentials": [ + {"id": 2, "name": "compliance-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1010, "status": "successful", "finished": _ts(timedelta(days=1))}, + }, + "created": _ts(timedelta(days=180)), + "modified": _ts(timedelta(days=14)), + }, + { + "id": 25, + "type": "job_template", + "name": "Emergency Patching", + "description": "Emergency patch application — NO become enabled (misconfigured)", + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/emergency-patch.yml", + "become_enabled": False, + "ask_job_type_on_launch": False, + "ask_variables_on_launch": False, + "ask_limit_on_launch": False, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 600, + "forks": 25, + "status": "failed", + "last_job_run": _ts(timedelta(days=7)), + "summary_fields": { + "project": {"id": 6, "name": "Remediation Playbooks", "status": "successful"}, + "inventory": {"id": 1, "name": "Production Systems", "total_hosts": 30}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1020, "status": "failed", "finished": _ts(timedelta(days=7))}, + }, + "created": _ts(timedelta(days=200)), + "modified": _ts(timedelta(days=200)), + }, + { + "id": 30, + "type": "job_template", + "name": "Fleet Health Report", + "description": "Generate fleet health and inventory report", + "inventory": 3, + "project": 8, + "playbook": "playbooks/reporting/fleet-health.yml", + "become_enabled": False, + "ask_job_type_on_launch": False, + "ask_variables_on_launch": True, + "ask_limit_on_launch": False, + "ask_inventory_on_launch": False, + "job_type": "run", + "verbosity": 0, + "timeout": 1800, + "forks": 30, + "status": "successful", + "last_job_run": _ts(timedelta(hours=6)), + "summary_fields": { + "project": {"id": 8, "name": "Fleet Reporting", "status": "successful"}, + "inventory": {"id": 3, "name": "All Managed Systems", "total_hosts": 63}, + "credentials": [ + {"id": 1, "name": "machine-credential", "kind": "ssh"}, + ], + "last_job": {"id": 1025, "status": "successful", "finished": _ts(timedelta(hours=6))}, + }, + "created": _ts(timedelta(days=120)), + "modified": _ts(timedelta(days=14)), + }, +] + +# --------------------------------------------------------------------------- +# Mock data: Jobs (recent runs) +# --------------------------------------------------------------------------- + +PROD_HOSTS = [ + "web-01.prod.example.com", + "web-02.prod.example.com", + "db-01.prod.example.com", + "db-02.prod.example.com", + "app-01.prod.example.com", + "app-02.prod.example.com", +] + +MOCK_JOBS = [ + { + "id": 1001, + "type": "job", + "name": "CVE Remediation - Kernel Update", + "job_type": "check", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=4, minutes=30)), + "finished": _ts(timedelta(hours=4)), + "elapsed": 1800.0, + "job_template": 10, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "limit": "web-01.prod.example.com,web-02.prod.example.com,db-01.prod.example.com", + "extra_vars": '{"target_cve": "CVE-2024-12345", "remediation_mode": "automated", "verify_after": true}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 10, "name": "CVE Remediation - Kernel Update"}, + }, + }, + { + "id": 1002, + "type": "job", + "name": "CVE Remediation - Kernel Update", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=3, minutes=45)), + "finished": _ts(timedelta(hours=3)), + "elapsed": 2700.0, + "job_template": 10, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-kernel-update.yml", + "limit": "web-01.prod.example.com,web-02.prod.example.com,db-01.prod.example.com", + "extra_vars": '{"target_cve": "CVE-2024-12345", "remediation_mode": "automated", "verify_after": true}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 10, "name": "CVE Remediation - Kernel Update"}, + }, + }, + { + "id": 1005, + "type": "job", + "name": "CVE Remediation - Package Update", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=12, minutes=20)), + "finished": _ts(timedelta(hours=12)), + "elapsed": 1200.0, + "job_template": 11, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/cve-package-update.yml", + "limit": "", + "extra_vars": '{"target_cve": "CVE-2024-54321"}', + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 11, "name": "CVE Remediation - Package Update"}, + }, + }, + { + "id": 1010, + "type": "job", + "name": "Compliance Check - STIG", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(days=1, hours=2)), + "finished": _ts(timedelta(days=1)), + "elapsed": 7200.0, + "job_template": 20, + "inventory": 3, + "project": 7, + "playbook": "playbooks/compliance/check-all.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "scheduled", + "summary_fields": { + "job_template": {"id": 20, "name": "Compliance Check - STIG"}, + }, + }, + { + "id": 1020, + "type": "job", + "name": "Emergency Patching", + "job_type": "run", + "status": "failed", + "failed": True, + "started": _ts(timedelta(days=7, hours=1)), + "finished": _ts(timedelta(days=7)), + "elapsed": 3600.0, + "job_template": 25, + "inventory": 1, + "project": 6, + "playbook": "playbooks/remediation/emergency-patch.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": 25, "name": "Emergency Patching"}, + }, + }, + { + "id": 1025, + "type": "job", + "name": "Fleet Health Report", + "job_type": "run", + "status": "successful", + "failed": False, + "started": _ts(timedelta(hours=6, minutes=30)), + "finished": _ts(timedelta(hours=6)), + "elapsed": 1800.0, + "job_template": 30, + "inventory": 3, + "project": 8, + "playbook": "playbooks/reporting/fleet-health.yml", + "limit": "", + "extra_vars": "{}", + "launch_type": "scheduled", + "summary_fields": { + "job_template": {"id": 30, "name": "Fleet Health Report"}, + }, + }, +] + +_next_job_id = 2000 + + +# --------------------------------------------------------------------------- +# Mock stdout generators +# --------------------------------------------------------------------------- + +def _generate_stdout(job: dict) -> str: + """Generate realistic Ansible playbook stdout for a job.""" + playbook_name = job.get("name", "Unknown") + job_type = job.get("job_type", "run") + status = job.get("status", "successful") + limit = job.get("limit", "") + hosts = limit.split(",") if limit else PROD_HOSTS[:3] + hosts = [h.strip() for h in hosts if h.strip()] + extra_vars = job.get("extra_vars", "{}") + mode = " (CHECK MODE)" if job_type == "check" else "" + + lines = [] + lines.append(f"PLAY [{playbook_name}] *****") + lines.append("") + + lines.append(f"TASK [Gathering Facts{mode}] *****") + for h in hosts: + lines.append(f"ok: [{h}]") + lines.append("") + + if "kernel" in playbook_name.lower(): + lines.append(f"TASK [Create boom snapshot for rollback{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}] => {{\"msg\": \"boom create --title pre-remediation-CVE-2024-12345\"}}") + lines.append("") + + lines.append(f"TASK [Check disk space for kernel update{mode}] *****") + for h in hosts: + lines.append(f"ok: [{h}] => {{\"msg\": \"Disk space OK: 45% used\"}}") + lines.append("") + + lines.append(f"TASK [Update kernel package{mode}] *****") + for h in hosts: + result = "changed" if status == "successful" else "fatal" + if result == "changed": + lines.append(f'changed: [{h}] => {{"msg": "kernel-5.14.0-362.24.1.el9_3 -> kernel-5.14.0-362.24.2.el9_3"}}') + else: + lines.append(f'fatal: [{h}]: FAILED! => {{"msg": "Permission denied", "rc": 1}}') + lines.append("") + + lines.append(f"TASK [Check if reboot is needed (needs-restarting -r){mode}] *****") + for h in hosts: + lines.append(f'changed: [{h}] => {{"rc": 1, "msg": "Reboot is required to fully utilize updates."}}') + lines.append("") + + elif "package" in playbook_name.lower(): + lines.append(f"TASK [Update target packages for CVE remediation{mode}] *****") + for h in hosts: + lines.append(f'changed: [{h}] => {{"msg": "httpd-2.4.53-7.el9 -> httpd-2.4.57-8.el9"}}') + lines.append("") + + lines.append(f"TASK [Restart affected services{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}]") + lines.append("") + + lines.append(f"TASK [Verify service health{mode}] *****") + for h in hosts: + lines.append(f'ok: [{h}] => {{"msg": "Service httpd is running"}}') + lines.append("") + + elif "emergency" in playbook_name.lower() and status == "failed": + lines.append(f"TASK [Apply emergency patch{mode}] *****") + for h in hosts: + lines.append(f'fatal: [{h}]: FAILED! => {{"msg": "Missing sudo password (become_enabled not set)", "rc": 1}}') + lines.append("") + lines.append("NO MORE HOSTS LEFT *****") + lines.append("") + + else: + lines.append(f"TASK [Execute playbook tasks{mode}] *****") + for h in hosts: + lines.append(f"changed: [{h}]") + lines.append("") + + lines.append("PLAY RECAP *****") + for h in hosts: + if status == "successful": + ok_count = random.randint(3, 6) + changed_count = random.randint(1, 3) + lines.append(f"{h:<45} : ok={ok_count} changed={changed_count} unreachable=0 failed=0 skipped=0 rescued=0 ignored=0") + elif status == "failed": + lines.append(f"{h:<45} : ok=1 changed=0 unreachable=0 failed=1 skipped=0 rescued=0 ignored=0") + lines.append("") + + return "\n".join(lines) + + +def _generate_events(job: dict) -> list[dict]: + """Generate realistic Ansible task events for a job.""" + hosts = (job.get("limit", "").split(",") if job.get("limit") else PROD_HOSTS[:3]) + hosts = [h.strip() for h in hosts if h.strip()] + events: list[dict] = [] + eid = 1 + + task_names = ["Gathering Facts"] + if "kernel" in job.get("name", "").lower(): + task_names += [ + "Create boom snapshot for rollback", + "Check disk space for kernel update", + "Update kernel package", + "Check if reboot is needed (needs-restarting -r)", + ] + elif "package" in job.get("name", "").lower(): + task_names += [ + "Update target packages for CVE remediation", + "Restart affected services", + "Verify service health", + ] + else: + task_names += ["Execute playbook tasks"] + + for task_name in task_names: + for host in hosts: + is_failed = job.get("status") == "failed" and task_name != "Gathering Facts" + events.append({ + "id": eid, + "type": "job_event", + "event": "runner_on_ok" if not is_failed else "runner_on_failed", + "task": task_name, + "host": host, + "host_name": host, + "play": job.get("name", ""), + "changed": task_name != "Gathering Facts" and not is_failed, + "failed": is_failed, + "event_data": { + "task": task_name, + "host": host, + "res": { + "changed": task_name != "Gathering Facts" and not is_failed, + "msg": "Task completed" if not is_failed else "Permission denied", + }, + }, + "created": _ts(timedelta(hours=4, minutes=30 - eid)), + }) + eid += 1 + + return events + + +def _generate_host_summaries(job: dict) -> list[dict]: + """Generate per-host summaries for a job.""" + hosts = (job.get("limit", "").split(",") if job.get("limit") else PROD_HOSTS[:3]) + hosts = [h.strip() for h in hosts if h.strip()] + summaries: list[dict] = [] + + for i, host in enumerate(hosts): + is_failed = job.get("status") == "failed" + summaries.append({ + "id": i + 1, + "type": "job_host_summary", + "host": i + 1, + "host_name": host, + "ok": 1 if is_failed else random.randint(3, 6), + "changed": 0 if is_failed else random.randint(1, 3), + "dark": 0, + "failures": 1 if is_failed else 0, + "skipped": 0, + "processed": 1, + "failed": is_failed, + }) + + return summaries + + +# --------------------------------------------------------------------------- +# MCP Tools: Job Management +# --------------------------------------------------------------------------- + +@mcp.tool() +def job_templates_list( + page_size: int = 10, + search: Optional[str] = None, +) -> dict: + """List available job templates in AAP. + + Args: + page_size: Number of results per page (default 10, max 200). + search: Optional search string to filter templates by name. + """ + results = MOCK_JOB_TEMPLATES + if search: + s = search.lower() + results = [t for t in results if s in t["name"].lower() or s in t.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def job_templates_retrieve(id: str) -> dict: + """Retrieve detailed information about a specific job template. + + Args: + id: Job template ID (as string). + """ + tid = int(id) + template = next((t for t in MOCK_JOB_TEMPLATES if t["id"] == tid), None) + if not template: + return {"detail": f"Not found. Job template {id} does not exist."} + return template + + +@mcp.tool() +def projects_list( + page_size: int = 50, + search: Optional[str] = None, +) -> dict: + """List available projects in AAP. + + Args: + page_size: Number of results per page. + search: Optional search string to filter projects by name. + """ + results = MOCK_PROJECTS + if search: + s = search.lower() + results = [p for p in results if s in p["name"].lower() or s in p.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def job_templates_launch_retrieve( + id: str, + requestBody: Optional[dict] = None, +) -> dict: + """Launch a job from a job template. + + Args: + id: Job template ID to launch. + requestBody: Optional launch parameters including job_type ('run' or 'check'), + extra_vars (dict), and limit (comma-separated host list). + """ + global _next_job_id + tid = int(id) + template = next((t for t in MOCK_JOB_TEMPLATES if t["id"] == tid), None) + if not template: + return {"detail": f"Not found. Job template {id} does not exist."} + + body = requestBody or {} + job_type = body.get("job_type", template.get("job_type", "run")) + + if not template.get("ask_job_type_on_launch") and job_type != template.get("job_type"): + return { + "error": f"Cannot override job_type: ask_job_type_on_launch is disabled on template {id}", + } + + job_id = _next_job_id + _next_job_id += 1 + + new_job = { + "id": job_id, + "type": "job", + "name": template["name"], + "job_type": job_type, + "status": "pending", + "failed": False, + "started": _ts(timedelta(seconds=0)), + "finished": None, + "elapsed": 0.0, + "job_template": tid, + "inventory": template["inventory"], + "project": template["project"], + "playbook": template["playbook"], + "limit": body.get("limit", ""), + "extra_vars": str(body.get("extra_vars", {})), + "launch_type": "manual", + "summary_fields": { + "job_template": {"id": tid, "name": template["name"]}, + }, + } + MOCK_JOBS.append(new_job) + + # Simulate job completion after launch + new_job["status"] = "successful" + new_job["finished"] = _ts(timedelta(seconds=-300)) + new_job["elapsed"] = 300.0 + + return { + "job": job_id, + "status": "pending", + "type": "job", + "url": f"/api/controller/v2/jobs/{job_id}/", + "related": { + "stdout": f"/api/controller/v2/jobs/{job_id}/stdout/", + "job_events": f"/api/controller/v2/jobs/{job_id}/job_events/", + "job_host_summaries": f"/api/controller/v2/jobs/{job_id}/job_host_summaries/", + }, + } + + +@mcp.tool() +def jobs_retrieve(id: int) -> dict: + """Get the status and details of a job run. + + Args: + id: Job ID to retrieve. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + return job + + +@mcp.tool() +def jobs_list(page_size: int = 10) -> dict: + """List recent job runs. + + Args: + page_size: Number of results to return. + """ + results = sorted(MOCK_JOBS, key=lambda j: j.get("started", ""), reverse=True) + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def jobs_stdout_retrieve(id: int, format: str = "txt") -> dict: + """Get the stdout (console output) from a job run. + + Args: + id: Job ID. + format: Output format ('txt' or 'json'). Default 'txt'. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + return { + "content": _generate_stdout(job), + "range": {"start": 0, "end": 1}, + } + + +@mcp.tool() +def jobs_job_events_list(id: int, page_size: int = 50) -> dict: + """Get task-level events for a job run. + + Args: + id: Job ID. + page_size: Number of events to return. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + events = _generate_events(job) + return { + "count": len(events), + "next": None, + "previous": None, + "results": events[:page_size], + } + + +@mcp.tool() +def jobs_job_host_summaries_list(id: int) -> dict: + """Get per-host execution summaries for a job run. + + Args: + id: Job ID. + """ + job = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not job: + return {"detail": f"Not found. Job {id} does not exist."} + summaries = _generate_host_summaries(job) + return { + "count": len(summaries), + "next": None, + "previous": None, + "results": summaries, + } + + +@mcp.tool() +def jobs_relaunch_retrieve( + id: int, + hosts: str = "all", + job_type: str = "run", +) -> dict: + """Relaunch a previously completed or failed job. + + Args: + id: Original job ID to relaunch. + hosts: Which hosts to target ('all' or 'failed'). + job_type: Job type for relaunch ('run' or 'check'). + """ + global _next_job_id + original = next((j for j in MOCK_JOBS if j["id"] == id), None) + if not original: + return {"detail": f"Not found. Job {id} does not exist."} + + new_id = _next_job_id + _next_job_id += 1 + + new_job = { + **original, + "id": new_id, + "job_type": job_type, + "status": "successful", + "failed": False, + "started": _ts(timedelta(seconds=0)), + "finished": _ts(timedelta(seconds=-300)), + "elapsed": 300.0, + "launch_type": "relaunch", + } + MOCK_JOBS.append(new_job) + + return { + "job": new_id, + "status": "pending", + "type": "job", + "url": f"/api/controller/v2/jobs/{new_id}/", + } + + +# --------------------------------------------------------------------------- +# MCP Tools: Inventory Management +# --------------------------------------------------------------------------- + +@mcp.tool() +def inventories_list( + page_size: int = 10, + search: Optional[str] = None, +) -> dict: + """List available inventories in AAP. + + Args: + page_size: Number of results per page. + search: Optional search string to filter inventories. + """ + results = MOCK_INVENTORIES + if search: + s = search.lower() + results = [inv for inv in results if s in inv["name"].lower() or s in inv.get("description", "").lower()] + return { + "count": len(results), + "next": None, + "previous": None, + "results": results[:page_size], + } + + +@mcp.tool() +def hosts_list( + inventory_id: Optional[int] = None, + page_size: int = 50, + search: Optional[str] = None, +) -> dict: + """List hosts in an inventory. + + Args: + inventory_id: Filter by inventory ID. If not provided, lists hosts from all inventories. + page_size: Number of results per page. + search: Optional search string to filter hosts by name. + """ + inv_id = inventory_id or 1 + hosts = _generate_hosts(inv_id) + if search: + s = search.lower() + hosts = [h for h in hosts if s in h["name"].lower()] + return { + "count": len(hosts), + "next": None if len(hosts) <= page_size else f"/api/controller/v2/hosts/?page=2", + "previous": None, + "results": hosts[:page_size], + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/without_skills/rh-sre__playbook-executor/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..fe5d549c --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-sre__playbook-executor/instruction.md b/evaluation/without_skills/rh-sre__playbook-executor/instruction.md new file mode 100644 index 00000000..5cced969 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/instruction.md @@ -0,0 +1,18 @@ +# Playbook Execution Task + +You are a Red Hat SRE. A remediation playbook needs to be executed against production systems through Ansible Automation Platform. You are responsible for the safe execution and monitoring of this job. + +## Scenario +A CVE remediation playbook has been prepared and a job template exists in AAP. You need to execute it safely: validate the template first, consider running a dry-run, launch the production job, monitor its progress, and report the results. + +## Requirements +- Locate and validate the job template in AAP (check it has the right inventory, project, credentials, and privilege escalation) +- Document a pre-flight checklist: template readiness, target hosts, and any prerequisites +- Plan the execution: whether to run a dry-run (check mode) first, how to monitor job progress, and what to do if it fails +- Launch the job (or document the launch procedure) and monitor its status +- Report per-host results: which hosts succeeded, which failed, and any error details +- Include guidance for handling failures (retry, rollback, escalation) + +Document your methodology, execution plan, and results in `/root/report.md`. + +Use MCP tools to interact with AAP. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-sre__playbook-executor/solution/solve.sh b/evaluation/without_skills/rh-sre__playbook-executor/solution/solve.sh new file mode 100644 index 00000000..090c2294 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/solution/solve.sh @@ -0,0 +1,21 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Playbook Execution Report + +## Execution Steps +1. Dry-run: job_type='check' (Ansible check mode) +2. Review results +3. Execute: job_type='run' + +## Git Flow +Playbook stored at playbooks/remediation/cve-2024-12345.yml. Commit, push, wait for sync complete before launch. No override at launch—AAP runs from synced project. + +## Job Template Validation +Invoke job-template-remediation-validator for each candidate template. + +## Execution Report +- Status: Success +- Systems patched: 4/4 +- Validate job log (jobs_stdout_retrieve) for CVE handling +- Suggest remediation-verifier after success +REPORT_EOF diff --git a/evaluation/without_skills/rh-sre__playbook-executor/task.toml b/evaluation/without_skills/rh-sre__playbook-executor/task.toml new file mode 100644 index 00000000..eaa9b790 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__playbook-executor" +name = "rh-sre Playbook Execution Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "playbook-executor", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-sre__playbook-executor/tests/llm_judge.py b/evaluation/without_skills/rh-sre__playbook-executor/tests/llm_judge.py new file mode 100644 index 00000000..15da24ed --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "launch_config_and_git_flow", "file": "/root/report.md", "question": "Does the report configure launch-time prompts for flexibility (variables, host limits, job type) and require Git synchronization before execution?", "reference": "A skilled report configures launch-time prompts and requires Git sync. An unskilled report hardcodes execution settings and skips synchronization requirements."}, + {"id": "relaunch_failed_hosts", "file": "/root/report.md", "question": "Does the report mention relaunching with hosts: failed to retry only failed hosts?", "reference": "A skilled report uses jobs_relaunch_retrieve with hosts: failed. An unskilled report suggests full re-execution."}, + {"id": "dry_run_and_monitoring", "file": "/root/report.md", "question": "Does the report recommend dry-run first and include per-host execution monitoring?", "reference": "A skilled report follows check mode before run and monitors per-host. An unskilled report runs directly without dry-run."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-sre__playbook-executor/tests/test.sh b/evaluation/without_skills/rh-sre__playbook-executor/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-sre__playbook-executor/tests/test_outputs.py b/evaluation/without_skills/rh-sre__playbook-executor/tests/test_outputs.py new file mode 100644 index 00000000..dab37078 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-executor/tests/test_outputs.py @@ -0,0 +1,89 @@ +""" +Tests for rh-sre__playbook-executor per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['playbook', 'execut', 'job']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_git_flow_mandatory(self): + """Skill: When template playbook path differs from generated playbook, Git Flow (commit, push, sync) is MANDATORY before launch.""" + c = read_report().lower() + has_git = any(t in c for t in ["git", "commit", "push", "sync"]) + has_block = any(t in c for t in ["before launch", "mandatory", "must", "block", "sync complete"]) + assert has_git or has_block, ( + "should require Git Flow when path differs (skill: no override at launch)" + ) + + def test_launch_configuration(self): + """Skill teaches configuring launch-time prompts for execution flexibility + (job type, variables, host limiting). Without skill, agents run playbooks + with hardcoded settings.""" + c = read_report().lower() + has_launch = any(t in c for t in ["launch", "prompt", "on launch"]) + has_config = any(t in c for t in [ + "variable", "limit", "job type", "configur", + ]) + assert has_launch and has_config, ( + "should configure launch-time prompts for execution flexibility" + ) + + def test_relaunch_failed_hosts(self): + """Skill: jobs_relaunch_retrieve with hosts: 'failed' to retry only failed hosts.""" + c = read_report().lower() + assert any(t in c for t in ["relaunch", "failed hosts", "retry failed"]), ( + "should mention relaunch for failed hosts (skill: jobs_relaunch_retrieve)" + ) + + def test_dry_run_first(self): + """Skill: Recommend dry-run (check mode) before production execution.""" + c = read_report().lower() + assert any(t in c for t in ["dry", "check mode", "check_mode", "preview", "before launch"]), ( + "should recommend dry-run first (skill: Phase 3)" + ) + + def test_per_host_results(self): + """Skill: Report per-host results (succeeded, failed, error details).""" + c = read_report().lower() + has_per_host = any(t in c for t in ["per host", "each host", "host result", "stdout", "host summary"]) + has_ansible_outcome = any(t in c for t in ["succeeded", "failed", "unreachable", "skipped", "changed"]) + assert has_per_host or has_ansible_outcome, ( + "should report per-host execution results (skill: host summaries)" + ) + + def test_error_taxonomy(self): + """Docs teach error taxonomy: connection/permissions/package/service/disk + failure categories with specific recovery paths. + Without docs, agents treat all errors generically.""" + c = read_report().lower() + categories = ["connection", "permission", "package", "service", "disk"] + mentioned = sum(1 for cat in categories if cat in c) + assert mentioned >= 2, ( + "should categorize errors by type (connection/permissions/package/service/disk)" + ) diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/Dockerfile b/evaluation/without_skills/rh-sre__playbook-generator/environment/Dockerfile new file mode 100644 index 00000000..5a2cfdee --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/Dockerfile @@ -0,0 +1,43 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..6c122770 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- Pass Rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/without_skills/rh-sre__playbook-generator/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..2269a235 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,722 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2026-1234": { + "cve_id": "CVE-2026-1234", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Critical kernel vulnerability: remote code execution in kernel network stack allows unauthenticated attackers to execute arbitrary code via crafted packets", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2026-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 8, + "total_remediated": 2, + "total_vulnerable": 6, + }, + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-sre__playbook-generator/instruction.md b/evaluation/without_skills/rh-sre__playbook-generator/instruction.md new file mode 100644 index 00000000..585c6f73 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/instruction.md @@ -0,0 +1,17 @@ +# Playbook Generation Task + +You are a Red Hat SRE. A critical CVE has been identified affecting systems in your fleet. You need to generate a remediation playbook that can be used to patch the affected hosts. + +## Scenario +CVE-2026-1234 has been confirmed as a critical kernel vulnerability affecting multiple production RHEL systems. You need to generate an Ansible playbook that remediates this CVE on the affected hosts. + +## Requirements +- Use available tools to generate a remediation playbook for the CVE +- Review the generated playbook content: what packages it updates, whether it requires a reboot, and any risk factors +- Document the playbook metadata: target CVE, affected systems, reboot requirements, and delegation safety notes +- If playbook generation fails, document the failure and describe alternative approaches +- The playbook should be ready to hand off for execution (do not execute it yourself) + +Document the generated playbook and your analysis in `/root/report.md`. You MUST write the report file — do not just display the content. + +Use MCP tools to interact with the environment. If reference documentation or skills are available in this environment, consult them before beginning work. Complete the entire analysis autonomously — do not stop after preliminary steps. Proceed through playbook generation, review, and report writing without waiting for user input. diff --git a/evaluation/without_skills/rh-sre__playbook-generator/solution/solve.sh b/evaluation/without_skills/rh-sre__playbook-generator/solution/solve.sh new file mode 100644 index 00000000..2543cf93 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/solution/solve.sh @@ -0,0 +1,38 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Playbook Generation Report + +## Methodology +Used `create_vulnerability_playbook` MCP tool (remediations endpoint via Lightspeed) to generate the remediation playbook for CVE-2026-1234. The playbook is returned AS IS — unmodified from the generation tool output. No pre-flight, backup, or restart steps were added. + +## Generated Playbook (returned AS IS, unmodified) +```yaml +- hosts: affected_systems + tasks: + - block: + - name: Create boom snapshot + command: boom create --title "pre-cve-{{ cve_id }}" + - name: Apply patch + dnf: + name: '*' + state: latest + rescue: + - name: Rollback + command: boom rollback + always: + - name: Check reboot needed + command: needs-restarting -r + register: needs_restarting +``` + +## Key Patterns +- block/rescue/always for error handling +- needs-restarting -r for reboot detection (RHEL 8/9) +- boom create for kernel/snapshot before remediation + +## Failure Handling +If the create_vulnerability_playbook tool fails, do not auto-generate a playbook from general knowledge. Present the user with options: (A) Retry the tool, (B) Generate from knowledge with explicit user approval, or (C) Exit and escalate. + +## Execution +Do NOT run ansible-playbook directly. Delegate execution to the playbook-executor skill/workflow. +REPORT_EOF diff --git a/evaluation/without_skills/rh-sre__playbook-generator/task.toml b/evaluation/without_skills/rh-sre__playbook-generator/task.toml new file mode 100644 index 00000000..1ea4f1ac --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__playbook-generator" +name = "rh-sre Playbook Generation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "playbook-generator", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-sre__playbook-generator/tests/llm_judge.py b/evaluation/without_skills/rh-sre__playbook-generator/tests/llm_judge.py new file mode 100644 index 00000000..05cd660f --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "mcp_tool_for_generation", "file": "/root/report.md", "question": "Does the report describe using an MCP tool (such as create_vulnerability_playbook or a remediations/Lightspeed endpoint) to generate the playbook, rather than writing it manually from scratch?", "reference": "A skilled report uses the Lightspeed MCP create_vulnerability_playbook tool. An unskilled report writes the playbook manually from general Ansible knowledge without using an MCP generation tool."}, + {"id": "return_as_is", "file": "/root/report.md", "question": "Does the report explicitly state that the generated playbook should be returned AS IS or unmodified, without adding extra steps like pre-flight checks, backup tasks, or restart handlers?", "reference": "A skilled report emphasizes returning the tool output unmodified. An unskilled report adds pre-flight checks, backup steps, restart handlers, or other enhancements to the generated playbook."}, + {"id": "delegation_not_execution", "file": "/root/report.md", "question": "Does the report explicitly state that playbook execution should be delegated to a separate execution workflow and NOT run directly via ansible-playbook?", "reference": "A skilled report delegates execution to a dedicated execution workflow rather than running ansible-playbook directly. An unskilled report runs ansible-playbook directly or doesn't address the execution boundary."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-sre__playbook-generator/tests/test.sh b/evaluation/without_skills/rh-sre__playbook-generator/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-sre__playbook-generator/tests/test_outputs.py b/evaluation/without_skills/rh-sre__playbook-generator/tests/test_outputs.py new file mode 100644 index 00000000..00518d36 --- /dev/null +++ b/evaluation/without_skills/rh-sre__playbook-generator/tests/test_outputs.py @@ -0,0 +1,74 @@ +""" +Tests for rh-sre__playbook-generator per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['playbook', 'generat', 'cve']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_mcp_tool_for_generation(self): + """Skill: Use create_vulnerability_playbook MCP tool, not manual playbook writing.""" + c = read_report().lower() + assert any(t in c for t in [ + "create_vulnerability_playbook", "create_vuln_playbook", + "remediations", "lightspeed", + ]) and any(t in c for t in ["tool", "mcp", "generat"]), ( + "should reference MCP tool usage for playbook generation (not manual writing)" + ) + + def test_no_modifications_to_playbook(self): + """Skill: Return playbook AS IS, no modifications—never add pre-flight, backup, restart.""" + c = read_report().lower() + assert any(t in c for t in [ + "as is", "as-is", "unmodified", "do not modify", "no modification", + "unchanged", "without modification", "returned unchanged", + "original output", "generated output", + ]), "should return playbook unmodified (skill: no enhancements without user approval)" + + def test_no_auto_generate_on_failure(self): + """Skill: Never auto-generate playbooks from general knowledge without approval.""" + c = read_report().lower() + has_constraint = any(t in c for t in [ + "do not auto", "never auto", "not auto-generat", + "without approval", "explicit approval", "user approval", + "do not generat", "never generat", + ]) + has_options = any(t in c for t in ["retry", "option", "escalat"]) + assert has_constraint or has_options, ( + "should state not to auto-generate playbooks without user approval" + ) + + def test_delegation_to_executor(self): + """Skill: This skill ONLY generates; execution delegated to playbook-executor.""" + c = read_report().lower() + assert any(t in c for t in [ + 'delegat', 'executor', 'playbook-executor', 'hand off', + 'not execute', 'do not run', 'do not execute', + 'not run ansible-playbook', 'not ansible-playbook', + ]), "should delegate execution (not run ansible-playbook directly)" diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/Dockerfile b/evaluation/without_skills/rh-sre__remediation-verifier/environment/Dockerfile new file mode 100644 index 00000000..5a2cfdee --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/Dockerfile @@ -0,0 +1,43 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..6c122770 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- Pass Rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/without_skills/rh-sre__remediation-verifier/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..e826c96e --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,759 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def _system_profile_for_host(host_type: str, rhel_version: str, sid: int) -> dict: + """Generate system_profile fields for a host based on type and RHEL version.""" + el = "el9" if rhel_version.startswith("9") else "el8" + kernel = f"5.14.0-362.24.1.{el}_3.x86_64" if "9" in rhel_version else f"4.18.0-477.27.1.{el}.x86_64" + base_pkgs = [ + {"name": "kernel-core", "version": f"5.14.0-362.24.1.{el}.x86_64"}, + {"name": "httpd", "version": f"2.4.57-5.{el}"}, + {"name": "sshd", "version": f"8.9p1-23.{el}"}, + {"name": "firewalld", "version": f"1.2.5-4.{el}"}, + {"name": "systemd", "version": f"250-19.{el}"}, + ] + if "web" in host_type or "lb" in host_type: + base_pkgs.extend([ + {"name": "nginx", "version": f"1.24.1-3.{el}"}, + {"name": "openssl", "version": f"3.0.7-24.{el}"}, + ]) + elif "db" in host_type: + base_pkgs.extend([ + {"name": "postgresql", "version": f"15.4-1.{el}"}, + {"name": "openssl", "version": f"3.0.7-24.{el}"}, + ]) + elif "mon" in host_type: + base_pkgs.extend([ + {"name": "prometheus", "version": f"2.45.0-1.{el}"}, + {"name": "node_exporter", "version": f"1.6.1-2.{el}"}, + ]) + else: + base_pkgs.extend([ + {"name": "java-17-openjdk", "version": f"17.0.8-4.{el}"}, + {"name": "openssl", "version": f"3.0.7-24.{el}"}, + ]) + services = ["sshd.service", "firewalld.service", "chronyd.service"] + if "web" in host_type or "lb" in host_type: + services.append("httpd.service") + elif "db" in host_type: + services.extend(["postgresql.service", "postgresql-15.service"]) + elif "mon" in host_type: + services.extend(["prometheus.service", "node_exporter.service"]) + else: + services.append("httpd.service") + ip_octet = 10 + (sid % 245) + mac_hex = f"{(sid % 256):02x}" + return { + "installed_packages": base_pkgs[:8], + "running_services": services, + "network_interfaces": [ + {"name": "eth0", "ipv4": [f"10.0.1.{ip_octet}"], "mac": f"52:54:00:a1:b2:{mac_hex}"}, + {"name": "lo", "ipv4": ["127.0.0.1"], "mac": "00:00:00:00:00:00"}, + ], + "kernel_version": kernel, + } + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + # Add system_profile to each host + for idx, s in enumerate(systems): + host_type = "app" # default + for ht in ["web", "db", "app", "lb", "mon", "cache"]: + if ht in s["id"]: + host_type = ht + break + s["system_profile"] = _system_profile_for_host( + host_type, s["rhel_version"], idx + 1 + ) + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/instruction.md b/evaluation/without_skills/rh-sre__remediation-verifier/instruction.md new file mode 100644 index 00000000..845397fa --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/instruction.md @@ -0,0 +1,18 @@ +# Post-Remediation Verification Task + +You are a Red Hat SRE. A CVE remediation playbook was recently executed against production systems. You need to verify that the fix was successfully applied and that the systems are healthy. + +## Scenario +The operations team ran a remediation playbook for CVE-2026-1234 against affected RHEL systems earlier today. You need to confirm the remediation actually worked and that no systems were left in a broken state. + +## Requirements +- Check whether the affected systems are still listed as vulnerable to the CVE +- Verify that the relevant packages have been updated to the fixed versions +- Confirm that critical services on each system are running and healthy after the patch +- Identify any systems where remediation failed or is incomplete +- For any failures, provide troubleshooting guidance (package conflicts, services not restarting, partial patches) +- Produce a verification summary: how many systems passed, how many failed, and any remaining action items + +Document your methodology, verification results, and assessment in `/root/report.md`. + +Use MCP tools to query system data. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/solution/solve.sh b/evaluation/without_skills/rh-sre__remediation-verifier/solution/solve.sh new file mode 100644 index 00000000..49714723 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/solution/solve.sh @@ -0,0 +1,20 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Remediation Verification + +## CVE-2024-12345 Status +| System | RPM Status | CVE Status | Service Health | +|--------|-----------|------------|----------------| +| web-01 | installed >= fixed | Patched | Healthy | + +## Checks Performed +- get_cve_systems: System removed from affected list or status=patched +- get_host_details: system_profile.installed_packages >= expected fixed version +- systemd_failed_units: No service disruptions +- enabled_services, running_processes: verified + +## Notes +- Lightspeed inventory lag: up to 24 hours +- Recommend: insights-client --check-results to update inventory +- RPM comparison: installed version >= expected fixed version +REPORT_EOF diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/task.toml b/evaluation/without_skills/rh-sre__remediation-verifier/task.toml new file mode 100644 index 00000000..23f81673 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__remediation-verifier" +name = "rh-sre Remediation Verification Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "remediation-verifier", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/tests/llm_judge.py b/evaluation/without_skills/rh-sre__remediation-verifier/tests/llm_judge.py new file mode 100644 index 00000000..15b8919b --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "inventory_24h_lag", "file": "/root/report.md", "question": "Does the report note that Lightspeed inventory can take up to 24 hours to update and recommend insights-client --check-results for re-sync?", "reference": "A skilled report warns about inventory lag. An unskilled report expects immediate updates."}, + {"id": "system_profile_checks", "file": "/root/report.md", "question": "Does the report use get_host_details with include_system_profile for installed packages and service health verification?", "reference": "A skilled report uses system profile data. An unskilled report only checks CVE status."}, + {"id": "three_verification_layers", "file": "/root/report.md", "question": "Does the report verify at least 2 of: CVE status, package version, service health?", "reference": "A skilled report performs defense-in-depth verification. An unskilled report only checks one layer."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/tests/test.sh b/evaluation/without_skills/rh-sre__remediation-verifier/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/tests/test_outputs.py b/evaluation/without_skills/rh-sre__remediation-verifier/tests/test_outputs.py new file mode 100644 index 00000000..00ddada6 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation-verifier/tests/test_outputs.py @@ -0,0 +1,75 @@ +""" +Tests for rh-sre__remediation-verifier per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['verif', 'remediation', 'confirm']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_three_checks(self): + """Skill: Verify CVE status + package version + service health (defense in depth).""" + c = read_report().lower() + has_cve = any(t in c for t in ["cve", "vulnerab", "patched", "affected"]) + has_pkg = any(t in c for t in ["package", "version", "installed", "fixed"]) + has_svc = any(t in c for t in ["service", "running", "health", "enabled"]) + assert (has_cve and has_pkg) or (has_cve and has_svc) or (has_pkg and has_svc), ( + "should perform at least 2 of 3 checks (skill: CVE status, package, service)" + ) + + def test_package_version_comparison(self): + """Skill: Compare installed package version to expected fixed version (RPM-style).""" + c = read_report().lower() + has_compare = any(t in c for t in ["compare", "version", "expected", "installed"]) + has_fixed = any(t in c for t in ["fixed", "updated", "el8", "el9"]) + assert has_compare or has_fixed, ( + "should compare package versions (skill: verify_package_version)" + ) + + def test_inventory_24h_lag(self): + """Skill: Lightspeed inventory can take up to 24 hours to reflect updated package versions.""" + c = read_report().lower() + has_24 = "24" in c + has_timing = any(t in c for t in ["hour", "propagat", "delay"]) + assert has_24 and has_timing, ( + "should note inventory 24h lag (skill: Best Practices)" + ) + + def test_include_system_profile(self): + """Skill: get_host_details with include_system_profile: true returns installed_packages, enabled_services.""" + c = read_report().lower() + assert any(t in c for t in ["include_system_profile", "system_profile", "installed_packages"]), ( + "should reference include_system_profile for packages/services (skill)" + ) + + def test_insights_client_resync(self): + """Skill: insights-client --check-results triggers inventory re-sync.""" + c = read_report().lower() + assert any(t in c for t in ["insights-client", "check-results"]), ( + "should mention insights-client for inventory resync (skill)" + ) diff --git a/evaluation/without_skills/rh-sre__remediation/environment/Dockerfile b/evaluation/without_skills/rh-sre__remediation/environment/Dockerfile new file mode 100644 index 00000000..5a2cfdee --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/Dockerfile @@ -0,0 +1,43 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..6c122770 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- Pass Rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__remediation/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/without_skills/rh-sre__remediation/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..2269a235 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,722 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2026-1234": { + "cve_id": "CVE-2026-1234", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Critical kernel vulnerability: remote code execution in kernel network stack allows unauthenticated attackers to execute arbitrary code via crafted packets", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2026-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 8, + "total_remediated": 2, + "total_vulnerable": 6, + }, + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-sre__remediation/instruction.md b/evaluation/without_skills/rh-sre__remediation/instruction.md new file mode 100644 index 00000000..ffd80028 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/instruction.md @@ -0,0 +1,19 @@ +# CVE Remediation Workflow Task + +You are a Red Hat SRE. A critical CVE has been reported and you need to plan and document a complete end-to-end remediation workflow, from initial validation through execution and verification. + +## Scenario +CVE-2026-1234 (Critical, CVSS 9.8) has been identified as affecting production RHEL systems in your fleet. Management wants a comprehensive remediation plan that covers every phase of the response. + +## Requirements +- Validate the CVE: confirm it is real, assess its severity, and determine if a remediation is available +- Assess the impact: identify which systems are affected and their criticality +- Gather system context: understand each affected system's role, dependencies, and constraints before patching +- Plan playbook generation: how the remediation playbook will be created +- Plan execution: how the playbook will be run (dry-run first, then production), including approval gates and rollback strategy +- Plan verification: how you will confirm remediation was successful after execution +- Present a phased workflow with clear decision points and user confirmation steps at each gate + +Document the complete workflow plan in `/root/report.md`. + +Use MCP tools to query data. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-sre__remediation/solution/solve.sh b/evaluation/without_skills/rh-sre__remediation/solution/solve.sh new file mode 100644 index 00000000..2721e5ff --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/solution/solve.sh @@ -0,0 +1,21 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# Remediation Plan + +## Orchestration Order +1. Validate MCP connectivity +2. CVE impact analysis +3. Validate CVE remediation availability +4. Gather system context +5. Generate playbook +6. Execute playbook +7. Verify remediation + +## CVE-2024-12345 +- Remediatable: Yes +- Systems: 4 production +- Template: Kernel update with boom snapshot + +## Execution +Wait for user confirmation (yes/proceed) before Step 5 (Execute playbook). Dry-run first, then production run. +REPORT_EOF diff --git a/evaluation/without_skills/rh-sre__remediation/task.toml b/evaluation/without_skills/rh-sre__remediation/task.toml new file mode 100644 index 00000000..1922d4d5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__remediation" +name = "rh-sre CVE Remediation Planning Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "remediation", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-sre__remediation/tests/llm_judge.py b/evaluation/without_skills/rh-sre__remediation/tests/llm_judge.py new file mode 100644 index 00000000..c5278840 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "remediation_gate", "file": "/root/report.md", "question": "Does the report gate on remediation availability (checking whether automated remediation is possible for a CVE) before proceeding with playbook generation?", "reference": "A skilled report checks whether automated remediation is available as a prerequisite gate before attempting playbook generation. An unskilled report proceeds to generate playbooks without first verifying that remediation is available for the target CVEs."}, + {"id": "plan_before_execution", "file": "/root/report.md", "question": "Does the report present a Remediation Plan with summary/table/checklist for user confirmation before execution?", "reference": "A skilled report requires plan validation before execution. An unskilled report executes without plan review."}, + {"id": "two_part_confirmation", "file": "/root/report.md", "question": "Does the report describe two distinct confirmation checkpoints: one BEFORE starting (upfront planned tasks / Part A) and one AFTER playbook generation but BEFORE execution (execution plan / Part B)?", "reference": "A skilled report has Part A (upfront planned tasks before any remediation step) and Part B (execution plan confirmation after playbook is generated but before running it). An unskilled report has at most one confirmation checkpoint or no structured confirmation phases."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-sre__remediation/tests/test.sh b/evaluation/without_skills/rh-sre__remediation/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-sre__remediation/tests/test_outputs.py b/evaluation/without_skills/rh-sre__remediation/tests/test_outputs.py new file mode 100644 index 00000000..bad4f7c8 --- /dev/null +++ b/evaluation/without_skills/rh-sre__remediation/tests/test_outputs.py @@ -0,0 +1,78 @@ +""" +Tests for rh-sre__remediation per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['remediation', 'orchestrat', 'workflow']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_seven_step_sequence(self): + """Skill: Orchestrate in order: validate MCP → impact → validate CVE → context → playbook → execute → verify.""" + c = read_report().lower() + has_sequence = any(t in c for t in ["validate", "impact", "context", "playbook", "execute", "verify"]) + has_order = any(t in c for t in ["step", "phase", "before", "workflow order", "sequence"]) + assert has_sequence and has_order, ( + "should define 7-step orchestration sequence (skill: workflow order)" + ) + + def test_remediatable_gate(self): + """Skill: Gate on cve-validation: if not remediatable, stop or warn before playbook generation.""" + c = read_report().lower() + has_gate = any(t in c for t in ["remediat", "gate", "remediation_available", "advisory"]) + has_stop = any(t in c for t in ["stop", "cannot proceed", "no automated", "manual"]) + assert has_gate or has_stop, ( + "should gate on remediation availability (skill: Remediatable Gate)" + ) + + def test_plan_validation_before_execute(self): + """Skill: Present Remediation Plan (summary, table, checklist); wait for user yes/proceed before Step 5.""" + c = read_report().lower() + has_plan = any(t in c for t in ["plan", "checklist", "summary", "table"]) + has_confirm = any(t in c for t in ["confirm", "proceed", "approval", "yes", "abort"]) + assert has_plan and has_confirm, ( + "should require plan validation before execution (skill: Remediation Plan)" + ) + + def test_dry_run_recommendation(self): + """Skill: Recommend dry-run first; wait for explicit approval before actual execution.""" + c = read_report().lower() + assert any(t in c for t in ["dry-run", "dry run", "check mode", "preview"]), ( + "should recommend dry-run first (skill: before Step 5)" + ) + + def test_two_part_confirmation(self): + """Docs teach Part A (pre-Step-0) and Part B (post-Step-4) confirmations + with ordered step completion marking. Without docs, agents use single confirmation.""" + c = read_report().lower() + assert any(t in c for t in [ + "part a", "part b", "pre-step", "post-step", "two-part", + "before step 0", "after step 4", + ]) or ("confirm" in c and "step" in c), ( + "should use two-part confirmation (Part A pre-Step-0, Part B post-Step-4)" + ) diff --git a/evaluation/without_skills/rh-sre__system-context/environment/Dockerfile b/evaluation/without_skills/rh-sre__system-context/environment/Dockerfile new file mode 100644 index 00000000..5a2cfdee --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/Dockerfile @@ -0,0 +1,43 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "lightspeed-mcp": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-lightspeed-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/cross-reference-graph.json new file mode 100644 index 00000000..20837038 --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/cross-reference-graph.json @@ -0,0 +1,147 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", + "graph": { + "ansible/cve-remediation-templates.md": { + "complements": [ + { + "doc": "rhel/package-management.md", + "reason": "Templates reference DNF/YUM patterns and reboot detection methods", + "confidence": 0.95, + "use_when": "Any package update template (1, 4, 6)" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Templates should include error handling and rollback strategies", + "confidence": 0.85, + "use_when": "Batch remediation or complex workflows" + } + ], + "prerequisites": [ + { + "doc": "ansible/README.md", + "reason": "Overview of Ansible documentation structure", + "confidence": 0.50, + "use_when": "First-time users or orientation needed" + } + ], + "specializations": [ + { + "doc": "rhel/selinux-context.md", + "condition": "if selinux_cve", + "reason": "SELinux CVEs need context restoration patterns", + "confidence": 0.90, + "use_when": "Template 5 (SELinux update)" + } + ] + }, + "rhel/package-management.md": { + "complements": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Playbook templates implement these package management patterns", + "confidence": 0.95, + "use_when": "Creating or modifying remediation playbooks" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "Multi-version environments need compatibility guidance", + "confidence": 0.75, + "use_when": "Managing RHEL 7, 8, and 9 simultaneously" + }, + { + "doc": "rhel/systemd-services.md", + "reason": "Package updates often require service restarts", + "confidence": 0.70, + "use_when": "Service restart needed after package update" + } + ], + "prerequisites": [], + "specializations": [] + }, + "ansible/README.md": { + "leads_to": [ + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "Primary Ansible documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to detailed templates" + }, + { + "doc": "ansible/error-handling.md", + "reason": "Error handling patterns for production playbooks", + "confidence": 0.70, + "use_when": "Need advanced error handling beyond templates" + } + ] + }, + "rhel/README.md": { + "leads_to": [ + { + "doc": "rhel/package-management.md", + "reason": "Primary RHEL documentation for CVE remediation", + "confidence": 0.95, + "use_when": "User navigating from overview to package management" + }, + { + "doc": "rhel/version-compatibility.md", + "reason": "RHEL version differences and compatibility", + "confidence": 0.75, + "use_when": "Multi-version environment" + } + ] + }, + "insights/vulnerability-logic.md": { + "complements": [ + { + "doc": "references/cvss-scoring.md", + "reason": "CVSS scoring complements Red Hat vulnerability assessment", + "confidence": 0.90, + "use_when": "Performing risk assessment" + }, + { + "doc": "ansible/cve-remediation-templates.md", + "reason": "After assessment, create remediation playbook", + "confidence": 0.85, + "use_when": "Proceeding from assessment to remediation" + } + ], + "prerequisites": [], + "specializations": [] + }, + "references/cvss-scoring.md": { + "complements": [ + { + "doc": "insights/vulnerability-logic.md", + "reason": "Red Hat-specific risk assessment methodology", + "confidence": 0.90, + "use_when": "Understanding Red Hat severity ratings" + }, + { + "doc": "references/compliance-frameworks.md", + "reason": "Compliance requirements often tied to CVSS scores", + "confidence": 0.70, + "use_when": "Compliance-driven remediation" + } + ], + "prerequisites": [], + "specializations": [] + } + }, + "relationship_types": { + "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", + "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", + "specializations": "Documents needed only under specific conditions. Check condition field before loading.", + "leads_to": "Natural navigation path from overview to detailed content." + }, + "usage_instructions": { + "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", + "confidence_thresholds": { + "high": ">= 0.90 - Always load if relevant", + "medium": "0.70-0.89 - Load if task complexity warrants it", + "low": "< 0.70 - Optional, load only if explicitly needed" + }, + "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." + } +} diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/semantic-index.json new file mode 100644 index 00000000..9cfee28f --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/semantic-index.json @@ -0,0 +1,297 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "documents": [ + { + "path": "ansible/cve-remediation-templates.md", + "title": "CVE Remediation Playbook Templates", + "category": "ansible", + "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], + "semantic_keywords": [ + "package update", + "kernel update", + "service restart", + "configuration change", + "SELinux context", + "batch remediation", + "reboot handling", + "rollback strategy", + "idempotent playbook", + "CVE patch", + "Ansible playbook", + "error handling", + "audit logging" + ], + "use_cases": [ + "package_update_cve", + "kernel_cve", + "service_restart_cve", + "config_file_cve", + "selinux_cve", + "batch_remediation" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", + "related_docs": [ + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "token_estimate": 2500 + }, + { + "path": "rhel/package-management.md", + "title": "RHEL Package Management for CVE Remediation", + "category": "rhel", + "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], + "semantic_keywords": [ + "DNF package manager", + "YUM package manager", + "package update", + "repository management", + "reboot detection", + "systemd service management", + "needs-restarting", + "subscription manager", + "RHEL compatibility", + "service restart" + ], + "use_cases": [ + "package_update_cve", + "rhel_version_compatibility", + "reboot_detection", + "service_restart_after_update" + ], + "rhel_versions": ["rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", + "related_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/version-compatibility.md", + "rhel/systemd-services.md" + ], + "token_estimate": 1600 + }, + { + "path": "insights/vulnerability-logic.md", + "title": "Red Hat Lightspeed Vulnerability Assessment Logic", + "category": "insights", + "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], + "semantic_keywords": [ + "Red Hat Lightspeed", + "vulnerability assessment", + "CVE risk scoring", + "vulnerable vs affected", + "security rules", + "severity rating", + "remediation priority", + "CVSS scoring", + "threat intelligence", + "patch prioritization", + "Red Hat severity", + "exploit availability", + "priority matrix" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "vulnerability_analysis", + "remediation_planning", + "compliance_reporting" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", + "related_docs": [ + "references/cvss-scoring.md", + "ansible/cve-remediation-templates.md", + "references/compliance-frameworks.md" + ], + "token_estimate": 900 + }, + { + "path": "references/cvss-scoring.md", + "title": "CVSS Scoring and Red Hat Severity Mappings", + "category": "references", + "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], + "semantic_keywords": [ + "CVSS score", + "severity rating", + "Red Hat severity", + "vulnerability scoring", + "risk assessment", + "priority matrix", + "CVSS v3.1", + "attack vector", + "exploitability", + "impact metrics", + "CVSS calculator", + "compliance requirements", + "PCI-DSS", + "SOC 2", + "NIST 800-53" + ], + "use_cases": [ + "risk_assessment", + "cve_prioritization", + "compliance_reporting", + "stakeholder_communication", + "severity_interpretation" + ], + "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], + "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], + "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", + "related_docs": [ + "insights/vulnerability-logic.md", + "references/compliance-frameworks.md", + "ansible/cve-remediation-templates.md" + ], + "token_estimate": 1000 + } + ], + "task_mappings": { + "package_update_rhel": { + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "Standard package update CVE on bare metal or VM RHEL systems" + }, + "service_restart_cve": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt"], + "estimated_tokens": 4100, + "description": "CVE requiring service configuration changes and restart" + }, + "selinux_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": ["templates", "selinux-context"], + "estimated_tokens": 2500, + "description": "CVE affecting SELinux file contexts or policies" + }, + "batch_remediation": { + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": ["templates", "package-mgmt", "error-handling"], + "estimated_tokens": 4100, + "description": "Multiple CVEs across fleet of systems" + }, + "risk_assessment": { + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": ["vulnerability-logic", "cvss-scoring"], + "estimated_tokens": 1900, + "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" + } + }, + "inference_hints": { + "cve_type_detection": { + "kernel": { + "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], + "confidence": 0.95, + "implies_reboot": true, + "recommended_templates": ["template_4_kernel_update"] + }, + "package": { + "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], + "confidence": 0.85, + "implies_reboot": false, + "recommended_templates": ["template_1_package_update"] + }, + "service": { + "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], + "confidence": 0.80, + "implies_reboot": false, + "recommended_templates": ["template_2_service_restart"] + }, + "configuration": { + "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], + "confidence": 0.75, + "implies_reboot": false, + "recommended_templates": ["template_3_config_update"] + }, + "selinux": { + "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], + "confidence": 0.90, + "implies_reboot": false, + "recommended_templates": ["template_5_selinux"] + } + }, + "system_type_detection": { + "kubernetes": { + "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], + "confidence": 0.90, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "openshift": { + "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], + "confidence": 0.95, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "bare_metal": { + "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], + "confidence": 0.70, + "requires_pod_eviction": false, + "additional_docs": [] + }, + "vm": { + "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], + "confidence": 0.75, + "requires_pod_eviction": false, + "additional_docs": [] + } + }, + "rhel_version_detection": { + "rhel7": { + "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], + "package_manager": "yum", + "systemd_version": "219", + "needs_restarting_available": false + }, + "rhel8": { + "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], + "package_manager": "dnf", + "systemd_version": "239", + "needs_restarting_available": true + }, + "rhel9": { + "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], + "package_manager": "dnf", + "systemd_version": "252", + "needs_restarting_available": true + } + } + }, + "usage_instructions": { + "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", + "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", + "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", + "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." + } +} diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/task-to-docs-mapping.json new file mode 100644 index 00000000..19eff879 --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/task-to-docs-mapping.json @@ -0,0 +1,230 @@ +{ + "version": "1.0", + "generated": "2026-01-20T00:00:00Z", + "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", + "task_mappings": { + "kernel_cve_bare_metal": { + "description": "Kernel CVE on bare metal or VM RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", + "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" + }, + "prerequisites": [ + "System backup created", + "Maintenance window scheduled" + ] + }, + "package_update_rhel": { + "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update", + "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "RHEL subscription active", + "Repository access verified" + ] + }, + "service_restart_cve": { + "description": "CVE requiring service configuration changes and restart", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 2: Service Restart", + "rhel/package-management.md": "Service Restart After Package Updates" + }, + "prerequisites": [ + "Service configuration backup", + "Config syntax validation available" + ] + }, + "config_file_cve": { + "description": "CVE requiring system configuration file updates", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" + }, + "prerequisites": [ + "Configuration backup created", + "Changes reviewed and approved" + ] + }, + "selinux_remediation": { + "description": "CVE affecting SELinux file contexts or policies", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "rhel/selinux-context.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/selinux-context.md" + ], + "estimated_tokens": 2500, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" + }, + "prerequisites": [ + "SELinux enabled and enforcing", + "AVC denials reviewed" + ] + }, + "batch_remediation": { + "description": "Multiple CVEs across fleet of RHEL systems", + "required_docs": [ + "ansible/cve-remediation-templates.md" + ], + "optional_docs": [ + "ansible/error-handling.md", + "rhel/version-compatibility.md", + "rhel/package-management.md" + ], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md", + "ansible/error-handling.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages" + }, + "prerequisites": [ + "Inventory grouped by environment", + "Batch size determined", + "Rollback plan documented" + ] + }, + "risk_assessment": { + "description": "Analyze CVE impact without creating remediation", + "required_docs": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "optional_docs": [ + "references/compliance-frameworks.md" + ], + "workflow_order": [ + "insights/vulnerability-logic.md", + "references/cvss-scoring.md" + ], + "estimated_tokens": 1000, + "critical_sections": { + "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", + "references/cvss-scoring.md": "Red Hat Severity Mappings" + }, + "prerequisites": [ + "CVE ID known", + "Red Hat Lightspeed access" + ] + }, + "httpd_cve": { + "description": "Apache httpd package CVE remediation", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" + }, + "prerequisites": [ + "httpd service can be restarted", + "Web traffic can tolerate brief interruption" + ] + }, + "openssl_cve": { + "description": "OpenSSL library CVE requiring service restarts", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", + "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" + }, + "prerequisites": [ + "All services using openssl identified", + "Service restart order planned (dependencies)" + ] + }, + "glibc_cve": { + "description": "glibc CVE requiring reboot", + "required_docs": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "optional_docs": [], + "workflow_order": [ + "ansible/cve-remediation-templates.md", + "rhel/package-management.md" + ], + "estimated_tokens": 4100, + "critical_sections": { + "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", + "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" + }, + "prerequisites": [ + "Reboot can be scheduled", + "Maintenance window available" + ] + } + }, + "usage_guide": { + "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", + "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", + "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", + "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." + } +} diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/INDEX.md new file mode 100644 index 00000000..4fa6725f --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/docs/INDEX.md @@ -0,0 +1,389 @@ +--- +title: Red Hat Remediation Agent - Documentation Index +category: meta +sources: + - title: Red Hat Product Documentation + url: https://docs.redhat.com + sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed + date_accessed: 2026-02-24 +last_updated: 2026-02-24 +--- + +# Red Hat Remediation Agent - Documentation Index + +This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. + +## Quick Navigation + +### Priority P0 (Core Documentation) +- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE + - 6 production-ready Ansible playbook templates + - Package updates, kernel updates, service restarts, SELinux, batch remediation + +- **[RHEL Package Management](rhel/package-management.md)** + - DNF/YUM workflows for RHEL 7/8/9 + - Systemd service management + - Reboot detection and handling + +### Priority P1 (Extended Documentation) +- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ + - CVE risk assessment methodology + - CVSS score interpretation + - System inventory correlation + +- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ + - CVSS v3.1 metrics breakdown + - Red Hat severity mappings (Critical/Important/Moderate/Low) + - Priority decision matrix + +- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ + - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) + - Consult before calling inventory__list_hosts to avoid validation errors + +- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ + - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure + +- **RHEL Version Compatibility** (planned) + - RHEL 7/8/9 compatibility matrix + - Package naming differences + - Migration considerations + +- **SELinux Context Remediation** (planned) + - SELinux context fixes in playbooks + - `restorecon` patterns + - Policy package updates + +- **Ansible Error Handling** (planned) + - Block/rescue/always patterns + - Rollback strategies + - Idempotency best practices + +- **OpenShift Node Maintenance** (planned) + - Node drain procedures + - Maintenance mode patterns + - Uncordoning after updates + +- **RHEL 9 Security Hardening** (planned) + - RHEL 9 security baseline + - CIS benchmark alignment + - Common hardening patterns + +### Priority P2 (Reference Documentation - Planned) +- **Ansible Playbook Patterns** (planned) + - Reusable playbook components + - Variable management + - Role organization + +- **Ansible Automation Platform Integration** (planned) + - AAP/Tower workflows + - Job template configuration + - Credential management + +- **OpenShift Rolling Updates** (planned) + - Deployment strategies + - StatefulSet handling + - Health check verification + +- **OpenShift Security & Compliance** (planned) + - OCP security best practices + - Compliance scanning + - Security context constraints + +- **Compliance Frameworks** (planned) + - PCI-DSS requirements + - SOC 2 controls + - NIST guidelines + +- **RHEL Systemd Services** (planned) + - Service management patterns + - Service restart logic + - Health checks + +## Documentation Structure + +``` +docs/ +├── INDEX.md (this file) ✅ +├── SOURCES.md (official Red Hat source attribution) ✅ +├── rhel/ # RHEL-specific patterns +│ ├── README.md ✅ +│ ├── package-management.md (P0) ✅ +│ ├── selinux-context.md (P1 - planned) +│ ├── systemd-services.md (P2 - planned) +│ ├── version-compatibility.md (P1 - planned) +│ └── security-hardening-rhel9.md (P1 - planned) +├── ansible/ # Ansible playbook patterns +│ ├── README.md ✅ +│ ├── cve-remediation-templates.md (P0) ⭐ ✅ +│ ├── playbook-patterns.md (P2 - planned) +│ ├── error-handling.md (P1 - planned) +│ ├── idempotency.md (P2 - planned) +│ └── aap-integration.md (P2 - planned) +├── insights/ # Red Hat Lightspeed patterns +│ ├── README.md ✅ +│ ├── vulnerability-logic.md (P1) ✅ +│ ├── remediation-workflow.md (P2 - planned) +│ └── system-inventory.md (P2 - planned) +├── references/ # Reference documentation +│ ├── README.md ✅ +│ ├── cvss-scoring.md (P1) ✅ +│ ├── compliance-frameworks.md (P2 - planned) +│ └── glossary.md (P2 - planned) +└── .ai-index/ # AI inference optimization + ├── semantic-index.json ✅ + ├── task-to-docs-mapping.json ✅ + ├── cross-reference-graph.json ✅ + └── generate-index.py (planned) +``` + +## How to Use This Documentation (For AI Agents) + +### 1. Intelligent Document Discovery + +**Always start by reading the semantic index**: +``` +Read: docs/.ai-index/semantic-index.json (~200 tokens) +``` + +The semantic index enables: +- **Query-based discovery**: Match semantic keywords to your task +- **Task mapping shortcuts**: Pre-computed doc sets for common workflows +- **CVE type inference**: Automatic doc selection based on CVE characteristics +- **System type detection**: Context-aware doc loading (K8s vs bare metal) + +### 2. Task-Based Document Loading + +**Example Workflow - Kernel CVE**: +``` +1. Read semantic-index.json +2. Detect: CVE type = "kernel" (requires reboot) +3. Load from task_mappings["kernel_cve"]: + - ansible/cve-remediation-templates.md (Template 4: Kernel Update) + - rhel/package-management.md (DNF/YUM workflows) +4. Generate playbook using patterns from loaded docs +``` + +**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) + +### 3. Progressive Disclosure Pattern + +**Load docs incrementally as needed**: +- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment +- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations +- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns + +### 4. Cross-Reference Navigation + +Use the cross-reference graph to find related documentation: +``` +If reading: ansible/cve-remediation-templates.md +Also consider: + - rhel/package-management.md (complements: DNF patterns) ✅ + - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ +``` + +## Common Remediation Workflows + +### Workflow 1: Simple Package CVE +**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ +2. `rhel/package-management.md` (DNF workflows) ✅ + +### Workflow 2: Kernel CVE +**Task**: "Remediate kernel CVE on RHEL production nodes" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ +2. `rhel/package-management.md` (kernel update procedures) ✅ + +### Workflow 3: Batch Remediation +**Task**: "Remediate 5 CVEs across 20 RHEL servers" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ +2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ + +### Workflow 4: Risk Assessment +**Task**: "Analyze impact of CVE-2024-YYYY" + +**Required Docs**: +1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ +2. `references/cvss-scoring.md` (CVSS interpretation) ✅ + +### Workflow 5: SELinux CVE +**Task**: "Fix SELinux context vulnerability" + +**Required Docs**: +1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ +2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ + +## Documentation Quality Standards + +All documents follow these standards: + +### YAML Frontmatter (Required) +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Red Hat Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keyword1, keyword2, keyword3] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: [keyword phrases for AI discovery] +use_cases: [use_case_ids for task mapping] +related_docs: [cross-references] +last_updated: YYYY-MM-DD +--- +``` + +### Content Structure (Required) +```markdown +# [Title] + +## Overview +[2-3 sentence summary] + +## When to Use This +[Specific scenarios] + +## [Main Content Sections] +### [Subsection] +**Context**: [When this applies] +**Pattern**: [How to implement] +**Example**: +```yaml +[Code block with working example] +``` +**Pitfalls**: [Common mistakes to avoid] + +## Related Documentation +- [Cross-references to other docs] + +## Quick Reference +[Summary table or bullet points] +``` + +### Code Examples +- **Lead with code**: Show working examples first, explain after +- **Production-ready**: Use real-world patterns (not toy examples) +- **Complete**: Include error handling, logging, verification +- **Tested**: Patterns validated on actual RHEL/OpenShift systems + +## Official Source Attribution + +**All documentation in this knowledge base is derived from official Red Hat sources**. + +See [SOURCES.md](SOURCES.md) for complete source attribution table including: +- Official Red Hat Product Documentation URLs +- Red Hat Customer Portal knowledge base articles +- OpenShift official documentation +- Red Hat Lightspeed documentation +- Red Hat security advisories and bulletins + +**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. + +**Verification**: All sources verified active and current as of 2026-02-24. + +## AI Inference Optimization + +This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: + +### Semantic Index (`semantic-index.json`) +- Document metadata with semantic keywords +- Use case mappings for task-based discovery +- RHEL version applicability +- System type applicability (bare metal, VM, K8s, OpenShift) +- Token estimates for each document +- Related docs cross-references + +### Task-to-Docs Mapping (`task-to-docs-mapping.json`) +- Pre-computed doc sets for common remediation workflows +- Required vs optional doc indicators +- Workflow execution order +- Estimated token usage per workflow + +### Cross-Reference Graph (`cross-reference-graph.json`) +- Document relationship graph +- Complement relationships (docs that enhance each other) +- Prerequisite relationships (foundational docs) +- Specialization relationships (conditional docs) +- Confidence scores for relationships + +### Index Generation (`generate-index.py`) +- Auto-generates indexes from YAML frontmatter +- Validates doc structure +- Updates semantic keywords +- Rebuilds cross-reference graph + +## Performance Benefits + +**Token Savings**: +- Simple Package CVE: 21% reduction (~1,000 tokens saved) +- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) +- Batch Remediation: 31% reduction (~1,800 tokens saved) +- Risk Assessment: 34% reduction (~1,100 tokens saved) +- **Average**: 29% reduction across all task types + +**Response Time**: +- 85% reduction in navigation overhead +- 30-40% faster end-to-end response time +- Fewer Read tool calls (direct doc access) + +**Accuracy**: +- Zero missed related docs (cross-reference graph ensures completeness) +- Zero irrelevant doc reads (semantic matching prevents false positives) +- 85% improvement in doc discovery accuracy + +## Quick Reference Tables + +### RHEL Version Support Matrix +| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | +|--------------|-----------------|---------|---------|--------|--------| +| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | +| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | +| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | + +### OpenShift Version Support Matrix +| OCP Version | Kubernetes | RHEL CoreOS | Status | +|-------------|------------|-------------|--------| +| 4.7 | 1.20 | 8.x | Legacy | +| 4.8 | 1.21 | 8.x | Supported | +| 4.10 | 1.23 | 8.x | Supported | +| 4.16 | 1.29 | 9.x | Current | + +### CVE Severity Mapping (Red Hat) +| CVSS Score | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +## Documentation Maintenance + +### Update Process +1. Update or add markdown documentation +2. Update YAML frontmatter with sources and metadata +3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes +4. Verify source URLs in SOURCES.md are current +5. Update "Last Verified" dates + +### Source Verification Schedule +- **Monthly**: Verify all source URLs are active +- **Quarterly**: Check for updated Red Hat documentation versions +- **Per CVE**: Validate remediation patterns against latest RH advisories + +## Support + +For questions about this documentation: +- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation +- Consult official Red Hat Customer Portal: https://access.redhat.com +- Check Red Hat Product Documentation: https://docs.redhat.com + +**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/SOURCES.md new file mode 100644 index 00000000..06478094 --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/docs/SOURCES.md @@ -0,0 +1,107 @@ +# Red Hat Documentation Sources + +This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. + +## Source Attribution Table + +| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | +|----------|---------------|---------------------|-------------------|---------------| +| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | +| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | +| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | +| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | +| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | +| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | +| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | +| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | +| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | +| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | +| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | +| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | +| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | +| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | +| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | +| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | +| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | +| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | +| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | + +## Documentation Categories + +### RHEL (Red Hat Enterprise Linux) +- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) +- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening +- **Versions Covered**: RHEL 7, 8, 9 +- **Update Frequency**: Continuous (latest release notes include 2026 updates) + +### Ansible Automation Platform +- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation +- **Focus**: CVE remediation playbooks, automation patterns, AAP integration +- **Current Version**: Ansible Automation Platform 2.4 +- **Update Frequency**: Regular security advisories and feature updates + +### OpenShift Container Platform +- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) +- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security +- **Versions Covered**: OpenShift 4.7-4.16 +- **Update Frequency**: Per-release documentation updates + +### Red Hat Lightspeed +- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal +- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory +- **Current Version**: 1-latest (continuously updated) +- **Update Frequency**: Real-time CVE database updates + +### Security & CVSS +- **Primary Source**: Red Hat Customer Portal - Product Security Center +- **Focus**: CVSS v3.1 scoring, severity classification, security advisories +- **Update Frequency**: Daily security bulletins and advisories + +## Attribution Format + +All documentation files in this knowledge base include YAML frontmatter with source attribution: + +```yaml +--- +title: [Document Title] +category: rhel|ansible|openshift|insights|references +sources: + - title: [Official Doc Title] + url: [Official URL] + sections: [Relevant sections] + date_accessed: YYYY-MM-DD +tags: [keywords] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +last_updated: YYYY-MM-DD +--- +``` + +## Verification + +All sources listed above were verified as active and current as of January 20, 2026. The sources are: + +1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation +2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data +3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles +4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides + +## License and Usage + +This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. + +**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. + +## Source Maintenance + +This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: + +1. Update this SOURCES.md file with new entries +2. Update the YAML frontmatter in affected documentation files +3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` +4. Update the "Last Verified" date in the table above + +## Contact + +For questions about Red Hat documentation sources or to report broken links: +- Red Hat Customer Portal: https://access.redhat.com/support +- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/README.md new file mode 100644 index 00000000..846ef682 --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/README.md @@ -0,0 +1,50 @@ +--- +title: Ansible Documentation Overview +category: ansible +last_updated: 2026-01-20 +--- + +# Ansible Documentation Overview + +This directory contains Ansible playbook patterns and best practices for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** + - 6 production-ready playbook templates + - Package updates, kernel updates, service restarts + - SELinux, batch remediation patterns + - Error handling, rollback, audit logging + +### Future Enhancements (P1-P2 Priority) +- **error-handling.md** - Block/rescue/always patterns (planned) +- **idempotency.md** - Safe re-run patterns (planned) +- **playbook-patterns.md** - Reusable components (planned) +- **aap-integration.md** - Ansible Automation Platform workflows (planned) + +## When to Use These Docs + +**Use cve-remediation-templates.md when**: +- Generating CVE remediation playbooks +- Need production-ready patterns with error handling +- Working with Kubernetes/OpenShift systems (includes pod eviction integration) +- Handling kernel updates requiring reboots +- Implementing batch remediation across multiple systems + +## Template Selection Guide + +| CVE Type | Template | Complexity | +|----------|----------|------------| +| User-space package | Template 1: Package Update | Low | +| Service config | Template 2: Service Restart | Low | +| System config | Template 3: Config Update | Low | +| Kernel CVE | Template 4: Kernel Update | High | +| SELinux issue | Template 5: SELinux Context | Medium | +| Multiple CVEs | Template 6: Batch Remediation | High | + +## Quick Links + +- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index +- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/aap-job-execution.md new file mode 100644 index 00000000..386fd114 --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/aap-job-execution.md @@ -0,0 +1,532 @@ +--- +title: AAP Job Execution Guide +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Job Templates + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates + date_accessed: 2026-02-24 +tags: [aap, job-execution, playbook, dry-run, check-mode] +semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] +use_cases: [playbook-executor, remediation] +related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# AAP Job Execution Guide + +## Overview + +This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. + +## Job Template Requirements for Remediation + +### Minimum Requirements + +For a job template to be suitable for CVE remediation, it must have: + +1. **Inventory**: Contains target systems identified in CVE analysis +2. **Project**: Contains or can receive remediation playbooks from Git +3. **Credentials**: + - Machine credential (SSH) for host access + - Privilege escalation enabled (sudo/become) +4. **Execution Environment**: Compatible with RHEL versions of target systems + +### Recommended Settings + +- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters +- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime +- **Job Type**: Should support both "Run" and "Check" modes +- **Verbosity**: Set to at least "1 (Verbose)" for debugging +- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) +- **Enable Webhook**: Optional for CI/CD integration + +### Example Template Configuration + +```yaml +Name: CVE Remediation Template +Job Type: Run +Inventory: Production Servers +Project: Remediation Playbooks +Playbook: playbooks/remediation/remediation-template.yml +Credentials: + - SSH Credential (Machine) + - Privilege Escalation: Yes +Prompt on Launch: + - Variables: Yes + - Limit: Yes +Options: + - Enable Privilege Escalation: Yes + - Allow Simultaneous: No +``` + +## Dry-Run vs Production Execution + +### Dry-Run (Check Mode) + +**Purpose**: Simulate playbook execution without making actual changes. + +**Use When**: +- Testing new remediation playbooks +- Validating changes before production +- Identifying potential issues (permissions, package availability, dependencies) +- Understanding impact scope + +**How to Execute**: +```json +{ + "job_type": "check", + "extra_vars": {...} +} +``` + +**What It Does**: +- Gathers facts from target systems +- Evaluates conditionals and variables +- Simulates task execution +- Reports **would change** counts +- Does NOT apply any changes + +**Limitations**: +- Some modules don't support check mode (command, shell, raw) +- Services that would restart are not actually restarted +- Can't detect runtime failures that occur during actual execution +- Package dependencies may not be fully validated + +**Output Interpretation**: +``` +PLAY RECAP ************************************************************* +prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 +prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 + +"changed=3" means 3 tasks WOULD make changes +"failed=0" means no errors detected in check mode +``` + +### Production Execution (Run Mode) + +**Purpose**: Apply actual changes to systems. + +**Use When**: +- Dry-run passed successfully +- User has approved changes +- Maintenance window scheduled (if required) +- Backups completed + +**How to Execute**: +```json +{ + "job_type": "run", + "extra_vars": {...} +} +``` + +**What It Does**: +- Executes all playbook tasks +- Applies actual changes (package updates, config modifications, service restarts) +- Reports real results +- Can trigger system reboots if specified + +**Best Practices**: +1. Always run dry-run first +2. Review dry-run results carefully +3. Ensure maintenance window if downtime expected +4. Have rollback plan ready +5. Monitor execution in real-time +6. Verify success after completion + +## Job Type Parameter + +### job_type: "check" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "check" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml --check +``` + +**Behavior**: +- Runs in check mode (dry-run) +- No actual changes applied +- Reports what WOULD happen +- Useful for validation + +### job_type: "run" + +**API Parameter**: +```json +{ + "id": "10", + "requestBody": { + "job_type": "run" + } +} +``` + +**Equivalent Command Line**: +```bash +ansible-playbook playbook.yml +``` + +**Behavior**: +- Runs in execution mode +- Applies actual changes +- Reports what DID happen +- Production execution + +## Interpreting Job Results + +### Job Status Values + +| Status | Meaning | Action | +|--------|---------|--------| +| `pending` | Job queued, not yet started | Wait for execution | +| `waiting` | Waiting for resources/dependencies | Monitor for start | +| `running` | Currently executing | Monitor progress | +| `successful` | Completed without errors | Verify changes | +| `failed` | Completed with errors | Review error logs | +| `error` | Job could not execute | Check configuration | +| `canceled` | User cancelled job | N/A | + +### Per-Host Statistics + +**ok**: Number of tasks that executed successfully without changes +**changed**: Number of tasks that made actual changes +**failed**: Number of tasks that failed +**unreachable**: Number of hosts that couldn't be reached +**rescued**: Number of tasks that recovered from failures +**ignored**: Number of failed tasks that were ignored + +**Success Criteria**: +- `failed: 0` AND `unreachable: 0` = Success +- `changed > 0` = Remediation applied changes +- `ok > 0` = Some tasks ran successfully + +**Failure Indicators**: +- `failed > 0` = At least one task failed +- `unreachable > 0` = Host connectivity issues +- `ok: 0` AND `changed: 0` = Nothing executed successfully + +### Task Timeline Interpretation + +Example timeline: +``` +1. ✅ Gather Facts (2s) +2. ✅ Check disk space (1s) +3. ✅ Backup configuration (3s) +4. ✅ Update package httpd (45s) +5. ⚠️ Restart httpd service (FAILED on prod-web-03) +6. ✅ Verify service status (2s) +``` + +**Analysis**: +- Tasks 1-4: Successful across all hosts +- Task 5: Failed on one host (prod-web-03) +- Task 6: Likely skipped on failed host + +**Action**: +- Investigate why httpd restart failed on prod-web-03 +- Check logs for that specific host +- Verify httpd package was actually installed +- Relaunch job for failed host after fixing issue + +## AAP URL Structure + +### Job Details URL + +**Format**: +``` +https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} +``` + +**Example**: +``` +https://aap.example.com/#/jobs/playbook/1235 +``` + +**What It Shows**: +- Real-time job status +- Live output stream +- Per-host statistics +- Task-level details +- Error messages +- Job parameters used + +### Template URL + +**Format**: +``` +https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details +``` + +**Example**: +``` +https://aap.example.com/#/templates/job_template/10/details +``` + +### Project URL + +**Format**: +``` +https://{your-aap-instance}/#/projects/{PROJECT_ID}/details +``` + +## Troubleshooting Common Execution Failures + +### Connection Failures + +**Symptoms**: +- `unreachable: 1` in host statistics +- "SSH timeout" errors +- "Connection refused" messages + +**Common Causes**: +1. SSH service not running on target +2. Firewall blocking port 22 +3. Network connectivity issues +4. Wrong SSH credentials + +**Troubleshooting Steps**: +```bash +# Test SSH connectivity +ssh -i /path/to/key user@target-host + +# Check SSH service +systemctl status sshd + +# Verify firewall rules +firewall-cmd --list-all + +# Test network connectivity +ping target-host +``` + +**Resolution**: +- Fix SSH service or network issues +- Update credentials in AAP +- Relaunch job after fixing + +### Permission Errors + +**Symptoms**: +- `failed: 1` with "Permission denied" errors +- "sudo: required but not available" messages +- "This command has to be run under the root user" errors + +**Common Causes**: +1. Privilege escalation not enabled +2. User doesn't have sudo rights +3. SELinux blocking operation +4. File permissions incorrect + +**Troubleshooting Steps**: +```bash +# Check sudo access +sudo -l + +# Test privilege escalation +sudo whoami + +# Check SELinux status +getenforce + +# Review SELinux denials +ausearch -m avc -ts recent +``` + +**Resolution**: +- Enable "Privilege Escalation" in job template +- Grant sudo rights to SSH user +- Adjust SELinux policies +- Fix file permissions + +### Package Manager Issues + +**Symptoms**: +- "No package X available" errors +- "Repository not found" messages +- "Dependency problems" errors +- Package installation timeouts + +**Common Causes**: +1. Repository not configured or unavailable +2. Package name incorrect +3. Network issues accessing repos +4. Insufficient disk space + +**Troubleshooting Steps**: +```bash +# Check repository configuration +dnf repolist + +# Test package availability +dnf info httpd + +# Check disk space +df -h + +# Verify repository URLs +dnf repolist -v +``` + +**Resolution**: +- Configure required repositories +- Verify package names +- Fix network issues +- Free up disk space + +### Service Restart Failures + +**Symptoms**: +- `failed: 1` on service restart tasks +- "Failed to restart X.service" errors +- "Unit not found" messages +- Service timeout errors + +**Common Causes**: +1. Service not installed +2. Configuration errors +3. Service dependencies not met +4. Systemd issues + +**Troubleshooting Steps**: +```bash +# Check if service exists +systemctl status httpd + +# Verify service file +systemctl cat httpd + +# Check service logs +journalctl -u httpd -n 50 + +# Test manual restart +systemctl restart httpd +``` + +**Resolution**: +- Ensure service is installed +- Fix configuration errors +- Start required dependencies first +- Review systemd logs + +### Disk Space Issues + +**Symptoms**: +- "No space left on device" errors +- Package installation failures +- Download failures + +**Common Causes**: +1. /var partition full +2. /tmp partition full +3. Log files consuming space + +**Troubleshooting Steps**: +```bash +# Check disk usage +df -h + +# Find large files +du -sh /var/* | sort -hr | head -10 + +# Check package cache size +du -sh /var/cache/dnf +``` + +**Resolution**: +- Clean package cache: `dnf clean all` +- Remove old logs: `journalctl --vacuum-time=7d` +- Remove unused packages: `dnf autoremove` + +## Job Monitoring Best Practices + +### Real-Time Monitoring + +1. **Watch AAP Web UI**: Real-time output and status +2. **Monitor Task Progress**: Track which tasks are running +3. **Check Per-Host Stats**: Identify failing hosts early +4. **Review Event Log**: See task-level events as they occur + +### Alert Configuration + +Configure notifications for: +- Job failures +- Long-running jobs (timeout warnings) +- Partial successes (some hosts failed) + +### Post-Execution Verification + +After job completes: +1. **Review per-host statistics**: Ensure all hosts succeeded +2. **Check full output**: Look for warnings or errors +3. **Verify actual changes**: Confirm packages updated, services restarted +4. **Run remediation-verifier**: Validate CVE status changed + +## Performance Optimization + +### Parallelism + +AAP can run tasks in parallel across multiple hosts. Configure: +- **Forks**: Number of parallel processes (default: 5) +- **Instance Groups**: Distribute jobs across multiple AAP nodes +- **Job Slicing**: Split large inventories into parallel jobs + +### Timeout Settings + +Set appropriate timeouts based on: +- Number of target systems +- Package size to download +- Network bandwidth +- System resources + +**Recommended Timeouts**: +- Small remediations (1-10 hosts): 10 minutes +- Medium remediations (10-50 hosts): 30 minutes +- Large remediations (50+ hosts): 60+ minutes + +## Security Considerations + +### Credential Management + +- Use AAP credential vault for secrets +- Rotate credentials regularly +- Limit credential scope to necessary hosts +- Never hardcode credentials in playbooks + +### Audit Logging + +AAP automatically logs: +- Who launched the job +- When it was launched +- What parameters were used +- Full execution output +- Final job status + +**Retention**: Configure appropriate log retention for compliance. + +### Change Control + +Integrate AAP jobs with change management: +- Require approval workflows for production +- Document job execution in change tickets +- Link jobs to CVE remediation tracking +- Maintain audit trail + +## Related Documentation + +- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/cve-remediation-templates.md new file mode 100644 index 00000000..d612b2f6 --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/cve-remediation-templates.md @@ -0,0 +1,1500 @@ +--- +title: CVE Remediation Playbook Templates +category: ansible +sources: + - title: Red Hat Lightspeed Remediations Guide + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index + sections: Creating remediation plans, playbook generation + date_accessed: 2026-02-24 + - title: Creating and Managing Remediation Plans + url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide + sections: Playbook templates, execution patterns + date_accessed: 2026-02-24 + - title: Creating Remediation Playbooks (RHEL 7 Security Guide) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + sections: Ansible playbook patterns for security compliance + date_accessed: 2026-02-24 +tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] +applies_to: [rhel7, rhel8, rhel9, openshift4.x] +semantic_keywords: + - "package update" + - "kernel update" + - "service restart" + - "configuration change" + - "SELinux context" + - "batch remediation" + - "reboot handling" + - "rollback strategy" + - "idempotent playbook" + - "CVE patch" +use_cases: + - "package_update_cve" + - "kernel_cve" + - "service_restart_cve" + - "config_file_cve" + - "selinux_cve" + - "batch_remediation" +related_docs: + - "rhel/package-management.md" + - "ansible/error-handling.md" + - "rhel/version-compatibility.md" +last_updated: 2026-02-24 +--- + +# CVE Remediation Playbook Templates + +This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. + +## Overview + +Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: + +- **Error handling**: Block/rescue/always patterns for safe execution +- **Rollback capability**: Automated rollback on failure +- **Audit logging**: Comprehensive remediation tracking +- **Reboot handling**: Safe reboot detection and execution +- **Idempotency**: Safe to re-run without side effects +- **Health checks**: Pre-flight and post-flight validation + +## Template Index + +1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) +2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs +3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications +4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot +5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs +6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet + +## When to Use Each Template + +| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | +|----------|-------------------|----------|------------------|-------------------| +| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | +| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | +| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | +| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | +| SELinux context | File contexts | Template 5 | No | Rare in containerized | +| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | + +--- + +## Template 1: Package Update (Most Common) + +### Use Case +Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. + +### When to Use +- CVE affects an installed package +- Fix available via package update (DNF/YUM) +- No kernel or system-critical components involved +- Service restart sufficient (no reboot needed) + +### Key Features +- RHEL version validation (7/8/9 compatibility) +- Automatic backup creation (RHEL 8/9 snapshots) +- Package update with cache refresh +- Reboot detection +- Conditional service restarts +- Audit logging +- Idempotent (safe to re-run) + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Package Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + vulnerable_packages: + - package-name-1 + - package-name-2 + affected_services: [] # Optional: services to restart after update + + pre_tasks: + - name: Gather package facts + package_facts: + manager: auto + + - name: Verify system is RHEL + assert: + that: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version in ["7", "8", "9"] + fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" + success_msg: "RHEL {{ ansible_distribution_major_version }} detected" + + - name: Check current vulnerable package versions + debug: + msg: > + Package {{ item }} current version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Create backup point (RHEL 8/9 with Boom) + command: > + boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: + - ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: backup_result + + - name: Log backup creation + debug: + msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" + + tasks: + - name: Update vulnerable packages + block: + - name: Update packages using DNF (RHEL 8/9) + dnf: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update packages using YUM (RHEL 7) + yum: + name: "{{ vulnerable_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package update success + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed" + success_msg: "Packages updated successfully" + + rescue: + - name: Log package update failure + debug: + msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" + + - name: Fail playbook on update error + fail: + msg: "CVE remediation failed - package update error" + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check if reboot is required (systemd method) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Gather updated package facts + package_facts: + manager: auto + + - name: Verify package versions after update + debug: + msg: > + Package {{ item }} updated version: + {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages + + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - needs_restarting.rc == 0 | default(true) + + - name: Wait for services to be active + systemd: + name: "{{ item }}" + state: started + loop: "{{ affected_services | default([]) }}" + when: + - affected_services is defined + - affected_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Notify if reboot required + debug: + msg: | + ⚠️ REBOOT REQUIRED ⚠️ + System: {{ inventory_hostname }} + Reason: Package update requires reboot + Action: Schedule maintenance window for reboot + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) + + - name: Create reboot notification file + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Packages Updated: {{ vulnerable_packages | join(', ') }} + Reason: Package update requires system reboot + dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" + mode: '0644' + when: > + reboot_required_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) +``` + +### Usage Example + +```bash +# Create inventory file +cat > inventory.ini < remediate-cve-2024-1234.yml < + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ service_name }} service configuration updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Service: {{ service_name }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate sshd CVE requiring config changes +- name: CVE-2024-5678 Remediation - SSH Weak Ciphers + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-5678" + service_name: "sshd" + config_file: "/etc/ssh/sshd_config" + + tasks: + - name: Disable weak SSH ciphers + lineinfile: + path: "{{ config_file }}" + regexp: '^#?Ciphers.*' + line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' + backup: true + + - name: Disable weak MACs + lineinfile: + path: "{{ config_file }}" + regexp: '^#?MACs.*' + line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' + backup: true +``` + +### Pitfalls to Avoid + +❌ **Don't skip configuration validation**: Always test config syntax before restart +❌ **Don't forget backups**: Configuration rollback impossible without backups +❌ **Don't skip health checks**: Verify service functionality after restart +❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes + +--- + +## Template 3: Configuration File Update + +### Use Case +CVEs requiring modifications to system configuration files (not service-specific configs). + +### When to Use +- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) +- Fix involves file content changes +- No package updates or service restarts required +- Configuration takes effect via sysctl reload or next boot + +### Key Features +- Configuration file backup +- Atomic file updates +- Configuration validation +- Sysctl reload for kernel parameters +- Audit trail + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - System Configuration Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. + config_backup_dir: "/var/backups/cve-remediation" + sysctl_reload: true # Set to true if sysctl configuration + + pre_tasks: + - name: Create backup directory + file: + path: "{{ config_backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current configuration + copy: + src: "{{ config_file }}" + dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" + remote_src: true + mode: preserve + register: backup_config + + - name: Record current configuration checksum + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: original_config_stat + + tasks: + - name: Update configuration file + block: + # Example: Disable source routing (CVE mitigation) + - name: Set kernel parameter - Disable source routing + sysctl: + name: net.ipv4.conf.all.accept_source_route + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + - name: Set kernel parameter - Disable ICMP redirects + sysctl: + name: net.ipv4.conf.all.accept_redirects + value: '0' + state: present + reload: "{{ sysctl_reload }}" + sysctl_file: "{{ config_file }}" + when: config_file contains 'sysctl' + + # For non-sysctl configs, use lineinfile or blockinfile + - name: Update non-sysctl configuration + lineinfile: + path: "{{ config_file }}" + regexp: '^#?ParameterName.*' + line: 'ParameterName value' + backup: true + when: config_file not contains 'sysctl' + register: config_update + + - name: Verify configuration change + stat: + path: "{{ config_file }}" + checksum_algorithm: sha256 + register: new_config_stat + + - name: Assert configuration was modified + assert: + that: + - new_config_stat.stat.checksum != original_config_stat.stat.checksum + fail_msg: "Configuration file was not modified" + success_msg: "Configuration successfully updated" + + rescue: + - name: Log configuration failure + debug: + msg: "Configuration update failed - initiating rollback" + + - name: Restore configuration from backup + copy: + src: "{{ backup_config.dest }}" + dest: "{{ config_file }}" + remote_src: true + mode: preserve + + - name: Reload original sysctl configuration + command: sysctl -p {{ config_file }} + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Fail playbook after rollback + fail: + msg: "CVE remediation failed - configuration rolled back" + + post_tasks: + - name: Verify sysctl parameters (if applicable) + command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects + register: sysctl_verify + when: sysctl_reload and (config_file contains 'sysctl') + changed_when: false + + - name: Display sysctl values + debug: + msg: "{{ sysctl_verify.stdout_lines }}" + when: sysctl_reload and (config_file contains 'sysctl') + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + {{ config_file }} updated on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document configuration changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + Configuration File: {{ config_file }} + Backup Location: {{ backup_config.dest }} + Changes: Kernel parameters hardened per CVE mitigation + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate kernel parameter CVE +- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening + hosts: all_servers + become: true + + vars: + cve_id: "CVE-2024-9012" + config_file: "/etc/sysctl.conf" + sysctl_reload: true + + tasks: + - name: Disable IP forwarding + sysctl: + name: net.ipv4.ip_forward + value: '0' + state: present + reload: true + sysctl_file: /etc/sysctl.conf + + - name: Enable SYN cookies + sysctl: + name: net.ipv4.tcp_syncookies + value: '1' + state: present + reload: true + sysctl_file: /etc/sysctl.conf +``` + +### Pitfalls to Avoid + +❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot +❌ **Don't modify production configs without backup**: Always create backups +❌ **Don't forget validation**: Verify sysctl values after reload +❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots + +--- + +## Template 4: Kernel Update with Reboot + +### Use Case +Critical kernel CVEs requiring kernel package updates and system reboots. + +### When to Use +- CVE affects the Linux kernel +- Fix available via kernel package update +- System reboot mandatory for remediation +- High-impact operation requiring maintenance window + +### Key Features +- Kernel version validation +- Grub configuration backup +- Safe reboot with timeout +- Kubernetes node draining (if applicable) +- Post-reboot verification +- Health checks after reboot + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot + hosts: affected_systems + become: true + gather_facts: true + serial: 1 # Update one host at a time for safety + + vars: + cve_id: "CVE-YYYY-NNNNN" + kernel_package: "kernel" + reboot_timeout: 600 # 10 minutes + post_reboot_delay: 30 # Wait 30 seconds after reboot + kubernetes_node: false # Set to true if K8s node + drain_timeout: 300 # 5 minutes for node drain + + pre_tasks: + - name: Record current kernel version + command: uname -r + register: current_kernel + changed_when: false + + - name: Display current kernel + debug: + msg: "Current kernel: {{ current_kernel.stdout }}" + + - name: Check if system is a Kubernetes node + stat: + path: /etc/kubernetes/kubelet.conf + register: k8s_check + + - name: Set kubernetes_node fact + set_fact: + kubernetes_node: "{{ k8s_check.stat.exists }}" + + - name: Drain Kubernetes node (if applicable) + command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s + delegate_to: localhost + when: kubernetes_node + register: drain_result + + - name: Log node drain + debug: + msg: "Node drained: {{ drain_result.stdout }}" + when: kubernetes_node + + - name: Create pre-update snapshot (RHEL 8/9) + command: > + boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + register: snapshot_result + + - name: Backup GRUB configuration + copy: + src: /etc/default/grub + dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak + remote_src: true + mode: preserve + + tasks: + - name: Update kernel package + block: + - name: Update kernel using DNF (RHEL 8/9) + dnf: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update kernel using YUM (RHEL 7) + yum: + name: "{{ kernel_package }}" + state: latest + update_cache: true + register: kernel_update + when: ansible_distribution_major_version == "7" + + - name: Verify kernel update + assert: + that: + - kernel_update is changed or kernel_update is success + fail_msg: "Kernel update failed" + success_msg: "Kernel updated successfully" + + rescue: + - name: Log kernel update failure + debug: + msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" + + - name: Uncordon Kubernetes node on failure + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + ignore_errors: true + + - name: Fail playbook + fail: + msg: "CVE remediation failed - kernel update error" + + - name: Create reboot notification + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Current Kernel: {{ current_kernel.stdout }} + Action: System will reboot to apply kernel update + dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + mode: '0644' + + - name: Log pre-reboot state + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - + Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Reboot system to apply kernel update + reboot: + reboot_timeout: "{{ reboot_timeout }}" + post_reboot_delay: "{{ post_reboot_delay }}" + msg: "Rebooting for CVE-{{ cve_id }} kernel update" + register: reboot_result + + post_tasks: + - name: Verify system is back online + wait_for_connection: + timeout: 300 + + - name: Gather facts after reboot + setup: + + - name: Record new kernel version + command: uname -r + register: new_kernel + changed_when: false + + - name: Verify kernel was updated + assert: + that: + - new_kernel.stdout != current_kernel.stdout + fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" + success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" + + - name: Verify all services are running + service_facts: + + - name: Check critical services + assert: + that: + - ansible_facts.services['sshd.service'].state == 'running' + fail_msg: "Critical services not running after reboot" + success_msg: "System health check passed" + + - name: Uncordon Kubernetes node + command: kubectl uncordon {{ inventory_hostname }} + delegate_to: localhost + when: kubernetes_node + register: uncordon_result + + - name: Wait for node to be ready + command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s + delegate_to: localhost + when: kubernetes_node + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document kernel update + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Previous Kernel: {{ current_kernel.stdout }} + New Kernel: {{ new_kernel.stdout }} + Reboot Duration: {{ reboot_result.elapsed }} seconds + Kubernetes Node: {{ kubernetes_node }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' + + - name: Clean up notification files + file: + path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" + state: absent +``` + +### Usage Example + +```bash +# Kernel CVE remediation with Kubernetes integration +ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ + --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ + --limit production-worker-nodes \ + --ask-become-pass + +# Serial execution (one node at a time) is built into the playbook +# with 'serial: 1' directive +``` + +### Kubernetes/OpenShift Considerations + +**Before running this playbook on K8s nodes**: + +1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain +2. **Check StatefulSets**: May require special handling +3. **Verify cluster capacity**: Ensure remaining nodes can handle workload +4. **Schedule maintenance window**: Kernel updates are high-impact + +### Pitfalls to Avoid + +❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety +❌ **Don't forget post-reboot verification**: Ensure new kernel is running +❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes + +--- + +## Template 5: SELinux Context Update + +### Use Case +CVEs affecting SELinux file contexts or requiring SELinux policy updates. + +### When to Use +- CVE involves SELinux context issues +- Fix requires `restorecon` or policy updates +- SELinux denials blocking remediation +- No package updates required (context-only fixes) + +### Key Features +- SELinux status validation +- Context backup +- Safe context restoration +- Policy module management +- AVC denial checking +- Rollback capability + +### Complete Playbook + +```yaml +--- +- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update + hosts: affected_systems + become: true + gather_facts: true + + vars: + cve_id: "CVE-YYYY-NNNNN" + affected_paths: + - /path/to/affected/file1 + - /path/to/affected/dir/ + selinux_type: "httpd_sys_content_t" # Example context + backup_dir: "/var/backups/selinux-contexts" + + pre_tasks: + - name: Check SELinux status + command: getenforce + register: selinux_status + changed_when: false + failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] + + - name: Verify SELinux is not disabled + assert: + that: + - selinux_status.stdout != 'Disabled' + fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" + success_msg: "SELinux mode: {{ selinux_status.stdout }}" + + - name: Create backup directory + file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + + - name: Backup current SELinux contexts + shell: > + ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context + loop: "{{ affected_paths }}" + changed_when: false + + - name: Check for recent AVC denials + command: ausearch -m avc -ts recent + register: avc_denials + failed_when: false + changed_when: false + + - name: Log AVC denials if present + debug: + msg: "Recent AVC denials detected - review /var/log/audit/audit.log" + when: avc_denials.rc == 0 + + tasks: + - name: Apply SELinux context fixes + block: + - name: Restore default SELinux contexts + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + register: restorecon_result + + - name: Set specific SELinux type (if needed) + sefcontext: + target: '{{ item }}(/.*)?' + setype: "{{ selinux_type }}" + state: present + loop: "{{ affected_paths }}" + when: selinux_type is defined + register: sefcontext_result + + - name: Apply new context after semanage + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + when: sefcontext_result is changed + + - name: Verify contexts were applied + shell: ls -Z {{ item }} + loop: "{{ affected_paths }}" + register: context_verify + changed_when: false + + - name: Display new contexts + debug: + msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" + + rescue: + - name: Log SELinux context failure + debug: + msg: "SELinux context update failed - review AVC denials" + + - name: Display backup location + debug: + msg: "Context backups available in {{ backup_dir }}" + + - name: Fail playbook + fail: + msg: "CVE remediation failed - SELinux context update error" + + post_tasks: + - name: Check for new AVC denials + command: ausearch -m avc -ts recent + register: new_avc_denials + failed_when: false + changed_when: false + + - name: Warn if new AVC denials + debug: + msg: "⚠️ New AVC denials detected - manual review required" + when: new_avc_denials.rc == 0 + + - name: Verify SELinux is still enforcing + command: getenforce + register: final_selinux_status + changed_when: false + + - name: Assert SELinux mode unchanged + assert: + that: + - final_selinux_status.stdout == selinux_status.stdout + fail_msg: "SELinux mode changed unexpectedly" + success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" + + - name: Log remediation success + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - + SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Document SELinux changes + copy: + content: | + CVE: {{ cve_id }} + Date: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + Affected Paths: {{ affected_paths | join(', ') }} + SELinux Type Applied: {{ selinux_type | default('default') }} + Backup Location: {{ backup_dir }} + Status: Remediation successful + dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" + mode: '0644' +``` + +### Usage Example + +```yaml +# Remediate httpd SELinux context CVE +- name: CVE-2024-7890 Remediation - httpd SELinux Context + hosts: web_servers + become: true + + vars: + cve_id: "CVE-2024-7890" + affected_paths: + - /var/www/html + - /var/www/cgi-bin + selinux_type: "httpd_sys_content_t" + + tasks: + - name: Restore default contexts for web content + command: restorecon -Rv {{ item }} + loop: "{{ affected_paths }}" + + - name: Allow httpd network connections (if needed) + seboolean: + name: httpd_can_network_connect + state: true + persistent: true +``` + +### Pitfalls to Avoid + +❌ **Don't disable SELinux as a "fix"**: This defeats security +❌ **Don't use `setenforce 0`**: Keep SELinux enforcing +❌ **Don't skip AVC denial review**: Understand why denials occurred +❌ **Don't forget `restorecon -R`**: Recursive is critical for directories +❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects + +--- + +## Template 6: Batch Remediation + +### Use Case +Remediating multiple CVEs across a fleet of systems efficiently. + +### When to Use +- Multiple CVEs need remediation +- Large number of affected systems +- Want to minimize maintenance windows +- Need comprehensive remediation reporting + +### Key Features +- Multi-CVE handling +- Consolidated package updates +- Batch service restarts +- Progress tracking +- Detailed reporting +- Failure isolation (continues on non-critical errors) + +### Complete Playbook + +```yaml +--- +- name: Batch CVE Remediation - Multiple CVEs + hosts: affected_systems + become: true + gather_facts: true + serial: "{{ batch_size | default(5) }}" # Configurable batch size + + vars: + batch_id: "batch-{{ ansible_date_time.epoch }}" + batch_size: 5 # Process 5 systems concurrently + + cves: + - cve_id: "CVE-2024-1111" + packages: ["httpd", "httpd-tools"] + services: ["httpd"] + priority: "critical" + + - cve_id: "CVE-2024-2222" + packages: ["openssl", "openssl-libs"] + services: [] + priority: "important" + + - cve_id: "CVE-2024-3333" + packages: ["glibc", "glibc-common"] + services: [] + priority: "important" + + all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" + all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" + + report_dir: "/var/log/cve-remediation/{{ batch_id }}" + + pre_tasks: + - name: Create report directory + file: + path: "{{ report_dir }}" + state: directory + mode: '0755' + + - name: Log batch remediation start + copy: + content: | + Batch ID: {{ batch_id }} + Start Time: {{ ansible_date_time.iso8601 }} + System: {{ inventory_hostname }} + CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} + Total Packages: {{ all_packages | length }} + Total Services: {{ all_services | length }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" + mode: '0644' + + - name: Gather current package versions + package_facts: + manager: auto + + - name: Record pre-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" + mode: '0644' + + - name: Create backup snapshot (RHEL 8/9) + command: > + boom create --title "batch-{{ batch_id }}" + when: ansible_distribution_major_version in ["8", "9"] + ignore_errors: true + + tasks: + - name: Batch update all vulnerable packages + block: + - name: Update all packages in one transaction (RHEL 8/9) + dnf: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + + - name: Update all packages in one transaction (RHEL 7) + yum: + name: "{{ all_packages }}" + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" + + - name: Verify package updates + package_facts: + manager: auto + + - name: Record post-update package versions + copy: + content: | + {% for pkg in all_packages %} + {% if pkg in ansible_facts.packages %} + {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} + {% else %} + {{ pkg }}: not installed + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" + mode: '0644' + + rescue: + - name: Log package update failures + copy: + content: | + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Status: FAILED + Error: {{ package_update.msg | default('Package update failed') }} + Failed Packages: {{ package_update.failures | default([]) | join(', ') }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + mode: '0644' + + - name: Continue despite package failures + debug: + msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" + + - name: Check if reboot required + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + post_tasks: + - name: Restart affected services (if no reboot needed) + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ all_services }}" + when: + - all_services | length > 0 + - not (reboot_required_file.stat.exists | default(false)) + - (needs_restarting.rc == 0) | default(true) + ignore_errors: true + register: service_restarts + + - name: Log service restart failures + copy: + content: | + {% for result in service_restarts.results | default([]) %} + {% if result.failed %} + Service: {{ result.item }} + Status: FAILED + Error: {{ result.msg | default('Unknown error') }} + {% endif %} + {% endfor %} + dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" + mode: '0644' + when: service_restarts.failed | default(false) + + - name: Generate CVE remediation summary + copy: + content: | + ======================================== + BATCH CVE REMEDIATION SUMMARY + ======================================== + Batch ID: {{ batch_id }} + System: {{ inventory_hostname }} + Completion Time: {{ ansible_date_time.iso8601 }} + + CVEs Remediated: + {% for cve in cves %} + - {{ cve.cve_id }} ({{ cve.priority }}) + Packages: {{ cve.packages | join(', ') }} + Services: {{ cve.services | join(', ') if cve.services else 'None' }} + {% endfor %} + + Package Updates: {{ all_packages | length }} packages + Service Restarts: {{ all_services | length }} services + + Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} + + {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} + ⚠️ REBOOT REQUIRED ⚠️ + Schedule maintenance window to reboot this system. + {% endif %} + + Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} + + Reports Location: {{ report_dir }} + dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" + mode: '0644' + + - name: Log to central remediation log + lineinfile: + path: /var/log/cve-remediation.log + line: > + {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - + Remediated {{ cves | length }} CVEs - + {{ all_packages | length }} packages updated - + {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - + {{ inventory_hostname }} + create: true + mode: '0644' + + - name: Display remediation summary + debug: + msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" +``` + +### Usage Example + +```bash +# Remediate multiple CVEs across production fleet +ansible-playbook -i production-inventory.ini batch-remediation.yml \ + --extra-vars "batch_size=10" \ + --limit web_servers + +# Generate consolidated report +grep "Batch batch-" /var/log/cve-remediation.log | \ + awk '{print $1, $2, $NF}' | \ + sort > batch-remediation-summary.txt +``` + +### Reporting + +After execution, find detailed reports in `/var/log/cve-remediation//`: + +``` +/ +├── host1_start.txt # Remediation start details +├── host1_pre_versions.txt # Package versions before update +├── host1_post_versions.txt # Package versions after update +├── host1_SUMMARY.txt # Remediation summary +├── host1_FAILED.txt # Failures (if any) +└── host1_service_failures.txt # Service restart failures (if any) +``` + +### Pitfalls to Avoid + +❌ **Don't set batch_size too high**: Limits blast radius if failures occur +❌ **Don't skip pre/post version recording**: Critical for audit trail +❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically +❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) +❌ **Don't forget consolidated reporting**: Management needs overall status + +--- + +## Cross-Reference Guide + +### For RHEL-Specific Operations +- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection +- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences +- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns +- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns + +### For Error Handling & Rollback +- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies +- **[idempotency.md](./idempotency.md)** - Safe re-run patterns + +### For Execution & Deployment +- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows + +### For Risk Assessment +- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology +- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation + +--- + +## Official Red Hat Sources + +This document is derived from: + +1. **Red Hat Lightspeed Remediations Guide** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index + +2. **Creating and Managing Remediation Plans** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide + +3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-02-24 + +--- + +## Quick Reference Table + +| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | +|----------|----------|---------|------------|----------|------------| +| 1. Package Update | User-space packages | Rare | Low | High | Low | +| 2. Service Restart | Service configs | No | Medium | Medium | Low | +| 3. Config Update | System configs | No | Low | Medium | Low | +| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | +| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | +| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | + +## Template Selection Decision Tree + +``` +Is CVE affecting kernel? +├─ YES → Use Template 4 (Kernel Update) +└─ NO → Is CVE affecting multiple systems? + ├─ YES → Use Template 6 (Batch Remediation) + └─ NO → What component is affected? + ├─ Package → Use Template 1 (Package Update) + ├─ Service config → Use Template 2 (Service Restart) + ├─ System config → Use Template 3 (Config Update) + └─ SELinux context → Use Template 5 (SELinux) +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-02-24 +**Maintained By**: Remediation Agent Knowledge Base +**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/playbook-integration-aap.md new file mode 100644 index 00000000..806841f5 --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/playbook-integration-aap.md @@ -0,0 +1,667 @@ +--- +title: Playbook Integration with AAP +category: ansible +sources: + - title: Red Hat Ansible Automation Platform Documentation + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 + date_accessed: 2026-02-24 + - title: AAP Projects + url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects + date_accessed: 2026-02-24 +tags: [aap, playbooks, git-integration, project-sync, version-control] +semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] +use_cases: [playbook-executor, job-template-creator] +related_docs: [aap-job-execution.md, cve-remediation-templates.md] +last_updated: 2026-02-24 +--- + +# Playbook Integration with AAP + +## Overview + +This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. + +## Workflow Overview + +```mermaid +graph LR + A[Generate Playbook] --> B[Add to Git Repo] + B --> C[Commit & Push] + C --> D[Sync AAP Project] + D --> E[Playbook Available in AAP] + E --> F[Create/Use Job Template] + F --> G[Execute Playbook] +``` + +## Git Repository Structure + +### Recommended Directory Layout + +``` +ansible-remediation-playbooks/ +├── README.md +├── .gitignore +├── playbooks/ +│ ├── remediation/ +│ │ ├── remediation-CVE-2025-49794.yml +│ │ ├── remediation-CVE-2025-50123.yml +│ │ └── remediation-template.yml +│ ├── verification/ +│ │ └── verify-remediation.yml +│ └── rollback/ +│ └── rollback-template.yml +├── roles/ +│ ├── common/ +│ ├── package-update/ +│ └── service-restart/ +├── inventories/ +│ ├── production.ini +│ ├── staging.ini +│ └── development.ini +├── group_vars/ +│ └── all.yml +└── host_vars/ +``` + +**Key Directories**: +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (optional if using AAP inventories) +- `group_vars/` and `host_vars/` - Variable files + +### .gitignore Configuration + +```gitignore +# Ansible +*.retry +.vault_pass +*.swp +*~ + +# Logs +*.log + +# Credentials (NEVER commit) +**/credentials.* +**/secrets.* +**/.env + +# Temporary files +/tmp/ +.DS_Store +``` + +## Adding Playbooks to Git Repository + +### Method 1: Existing Repository + +If you already have a Git repository configured in AAP: + +#### Step 1: Clone Repository (if not already local) + +```bash +# Clone the repository +git clone https://github.com/your-org/ansible-remediation-playbooks.git +cd ansible-remediation-playbooks +``` + +#### Step 2: Add Generated Playbook + +```bash +# Create remediation directory if it doesn't exist +mkdir -p playbooks/remediation + +# Add the playbook (replace with actual content) +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +--- +- name: Remediate CVE-2025-49794 + hosts: all + become: true + + tasks: + - name: Check disk space + # ... playbook content ... +EOF +``` + +#### Step 3: Commit Changes + +```bash +# Stage the new playbook +git add playbooks/remediation/remediation-CVE-2025-49794.yml + +# Create descriptive commit message +git commit -m "Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Requires: Brief service restart (~10s downtime) +" +``` + +#### Step 4: Push to Remote + +```bash +# Push to main branch (or your default branch) +git push origin main +``` + +#### Step 5: Sync AAP Project + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Find your project (e.g., "Remediation Playbooks") +3. Click the **Sync** button (🔄 icon) +4. Wait for status to change to "Successful" (green checkmark) +5. Verify playbook appears in project's playbook list + +**Via AAP API** (if available): +```bash +curl -X POST \ + "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ + -H "Authorization: Bearer ${AAP_API_TOKEN}" +``` + +### Method 2: New Repository + +If you need to create a new repository for remediation playbooks: + +#### Step 1: Initialize Local Repository + +```bash +# Create project directory +mkdir ansible-remediation-playbooks +cd ansible-remediation-playbooks + +# Initialize Git +git init +``` + +#### Step 2: Create Directory Structure + +```bash +# Create directory structure +mkdir -p playbooks/{remediation,verification,rollback} +mkdir -p roles +mkdir -p inventories +mkdir -p {group_vars,host_vars} +``` + +#### Step 3: Create README + +```bash +cat > README.md << 'EOF' +# Ansible Remediation Playbooks + +CVE remediation playbooks for Red Hat Enterprise Linux systems. + +## Directory Structure + +- `playbooks/remediation/` - CVE remediation playbooks +- `playbooks/verification/` - Post-remediation verification +- `playbooks/rollback/` - Rollback procedures +- `roles/` - Shared Ansible roles +- `inventories/` - Inventory files (if not using AAP inventories) + +## Naming Convention + +Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` + +## Usage + +Playbooks are executed via AAP job templates. See internal documentation +for execution procedures. + +## Best Practices + +1. Always test in non-production first +2. Review playbook in dry-run (check) mode +3. Backup systems before remediation +4. Verify remediation success after execution +5. Document changes in commit messages +EOF +``` + +#### Step 4: Create .gitignore + +```bash +cat > .gitignore << 'EOF' +*.retry +.vault_pass +*.swp +*~ +*.log +**/credentials.* +**/secrets.* +**/.env +/tmp/ +.DS_Store +EOF +``` + +#### Step 5: Add First Playbook + +```bash +# Add your generated playbook +cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' +# [Your playbook content here] +EOF +``` + +#### Step 6: Initial Commit + +```bash +# Stage all files +git add . + +# Create initial commit +git commit -m "Initial commit: Add remediation playbooks structure + +- Directory structure for remediation, verification, rollback +- README with project documentation +- .gitignore for security +- First remediation playbook: CVE-2025-49794 +" +``` + +#### Step 7: Create Remote Repository + +**On GitHub**: +1. Go to https://github.com/new +2. Enter repository name: `ansible-remediation-playbooks` +3. Choose visibility (Private recommended for security) +4. **Do NOT** initialize with README (you already have one) +5. Click "Create repository" +6. Copy the repository URL + +**On GitLab**: +1. Go to "New Project" +2. Enter project name +3. Choose visibility +4. **Uncheck** "Initialize with README" +5. Create project +6. Copy the repository URL + +#### Step 8: Connect and Push + +```bash +# Add remote +git remote add origin + +# Rename branch to main (if needed) +git branch -M main + +# Push to remote +git push -u origin main +``` + +#### Step 9: Add Project to AAP + +**Via AAP Web UI**: +1. Navigate to **Automation Execution** → **Projects** +2. Click **Add** button +3. Fill in project form: + - **Name**: "Remediation Playbooks" + - **Organization**: Select your organization + - **Source Control Type**: Git + - **Source Control URL**: `` + - **Source Control Branch**: `main` + - **Source Control Credential**: (if private repo) +4. Click **Save** +5. AAP will automatically sync +6. Wait for status "Successful" + +## Project Sync Process + +### Understanding Project Sync + +**What Happens During Sync**: +1. AAP connects to Git repository +2. Fetches latest commits from specified branch +3. Downloads playbooks and related files +4. Updates project playbook list +5. Makes playbooks available for job templates + +**Sync Triggers**: +- Manual: Click Sync button in AAP Web UI +- Automatic: Configured update interval (optional) +- Webhook: Git push triggers AAP sync (optional) +- Pre-launch: Job template can auto-sync before execution + +### Sync Verification + +**Check Sync Status**: +```bash +# Via MCP tool +projects_list(search="Remediation") + +# Look for: +# - status: "successful" +# - scm_revision: Latest commit SHA +# - last_update_failed: false +``` + +**Verify Playbook Available**: +1. In AAP Web UI, go to Projects +2. Click on your project +3. View "Playbooks" tab +4. Confirm new playbook appears in list + +### Troubleshooting Sync Issues + +**Sync Failed - Authentication**: +``` +Error: Authentication failed +``` +**Cause**: Invalid or missing Git credentials +**Fix**: +- Update Source Control Credential in project settings +- Verify credential has read access to repository +- For private repos, ensure SSH key or token is valid + +**Sync Failed - Network**: +``` +Error: Failed to connect to repository +``` +**Cause**: Network connectivity issues or firewall +**Fix**: +- Verify repository URL is correct +- Check AAP server can reach Git server +- Review firewall rules + +**Sync Failed - Branch Not Found**: +``` +Error: Branch 'main' not found +``` +**Cause**: Specified branch doesn't exist +**Fix**: +- Verify branch name in project settings +- Check repository has commits on that branch +- Update branch name to match repository + +**Playbook Not Appearing**: +``` +Sync successful but playbook not in list +``` +**Cause**: Playbook not in correct path or format +**Fix**: +- Verify playbook is in repository root or subdirectory +- Check playbook has .yml or .yaml extension +- Ensure playbook is valid Ansible syntax +- Re-sync project after fixing + +## Playbook Versioning Strategy + +### Semantic Versioning for Playbooks + +**Approach 1: Git Tags** +```bash +# Tag specific playbook versions +git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" +git push origin remediate-CVE-2025-49794-v1.0 + +# Update for new version +git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" +git push origin remediate-CVE-2025-49794-v1.1 +``` + +**Approach 2: Filename Versioning** +``` +playbooks/remediation/ +├── remediation-CVE-2025-49794-v1.yml +├── remediation-CVE-2025-49794-v2.yml +└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) +``` + +**Approach 3: Branch-Based** +```bash +# Create feature branch for new playbook +git checkout -b remediate-cve-2025-49794 + +# Develop and test +git add playbooks/remediation/remediation-CVE-2025-49794.yml +git commit -m "Add CVE-2025-49794 remediation" + +# Merge to main after testing +git checkout main +git merge remediate-cve-2025-49794 +git push origin main +``` + +### Recommended Versioning Approach + +**For Production**: +1. Use Git tags for major versions +2. Keep playbook filenames stable +3. Document changes in commit messages +4. Use branches for development/testing +5. Merge to main only after validation + +**Version Format**: +``` +CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} + +Examples: +- CVE-2025-49794-v1.0 (Initial release) +- CVE-2025-49794-v1.1 (Bug fix) +- CVE-2025-49794-v2.0 (Major changes) +``` + +## Best Practices + +### Commit Message Guidelines + +**Format**: +``` +: + + + + +``` + +**Example**: +``` +feat: Add remediation playbook for CVE-2025-49794 + +- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) +- Affected package: httpd +- Remediation: Update to httpd-2.4.57-8.el9 +- Target systems: Production web servers +- Impact: Brief service restart (~10s downtime) +- Tested on: RHEL 9.3, 9.4 +- Validation: Passed dry-run on 50 staging systems + +Refs: TICKET-12345 +``` + +**Commit Types**: +- `feat:` - New playbook +- `fix:` - Bug fix in existing playbook +- `refactor:` - Code restructuring without behavior change +- `docs:` - Documentation updates +- `test:` - Test-related changes +- `chore:` - Maintenance tasks + +### Security Best Practices + +1. **Never Commit Credentials**: + - Use AAP credential vault + - Reference credentials via AAP, not in playbooks + - Add credential files to .gitignore + +2. **Sensitive Variables**: + ```yaml + # Bad - hardcoded password + - name: Connect to database + vars: + db_password: "MyPassword123" + + # Good - reference AAP credential + - name: Connect to database + vars: + db_password: "{{ lookup('env', 'DB_PASSWORD') }}" + ``` + +3. **Audit Trail**: + - Descriptive commit messages + - Link to change tickets + - Document testing performed + - Tag production versions + +### Code Review Process + +**Before Merging to Main**: +1. **Syntax Validation**: + ```bash + ansible-playbook --syntax-check playbook.yml + ``` + +2. **Linting**: + ```bash + ansible-lint playbook.yml + ``` + +3. **Dry-Run Testing**: + - Test on staging systems first + - Run in check mode + - Review output for errors + +4. **Peer Review**: + - Create pull request + - Have colleague review changes + - Address feedback + - Approve and merge + +## AAP Project Configuration + +### Project Settings + +**Optimal Configuration**: +```yaml +Name: Remediation Playbooks +Organization: Default +Source Control Type: Git +Source Control URL: https://github.com/org/ansible-remediation-playbooks.git +Source Control Branch: main +Source Control Credential: Git-ReadOnly-Credential + +Options: + Clean: Yes (remove local modifications) + Delete: Yes (delete before sync) + Track submodules: No (unless needed) + Update Revision on Launch: Yes (auto-sync before jobs) + +Update Cache Timeout: 0 (always fetch latest) +``` + +**Update on Launch**: +- **Enabled**: AAP syncs project before each job launch +- **Pros**: Always uses latest playbook version +- **Cons**: Slight delay before job starts +- **Recommendation**: Enable for dynamic environments + +### Multiple Projects Strategy + +**Option 1: Single Project for All Playbooks** +``` +Project: "Remediation Playbooks" +Contains: All remediation, verification, rollback playbooks +Pros: Simple management, single sync point +Cons: All teams share same repository +``` + +**Option 2: Separate Projects by Purpose** +``` +Project: "CVE Remediation" + - playbooks/remediation/ + +Project: "Verification Playbooks" + - playbooks/verification/ + +Project: "Rollback Procedures" + - playbooks/rollback/ + +Pros: Clear separation, different access controls +Cons: More complex, multiple syncs needed +``` + +**Option 3: Separate Projects by Team/Environment** +``` +Project: "Production Remediation" + - Branch: main + +Project: "Staging Remediation" + - Branch: staging + +Project: "Development Remediation" + - Branch: develop + +Pros: Environment isolation, safe testing +Cons: Need to promote across branches +``` + +## Automation and CI/CD Integration + +### Automated Testing Pipeline + +**Example GitHub Actions**: +```yaml +name: Playbook Validation + +on: + push: + branches: [ main, develop ] + paths: + - 'playbooks/**/*.yml' + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Ansible + run: | + pip install ansible ansible-lint + + - name: Syntax Check + run: | + ansible-playbook --syntax-check playbooks/**/*.yml + + - name: Ansible Lint + run: | + ansible-lint playbooks/ + + - name: Check for Secrets + run: | + git secrets --scan +``` + +### Webhook Integration + +**Trigger AAP Sync on Git Push**: + +1. **Configure Webhook in Git**: + - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` + - Events: Push events + - Secret: Generate in AAP + +2. **Enable Webhook in AAP**: + - Project settings → Enable Webhook + - Copy webhook URL and secret + - Add to Git repository settings + +**Result**: Git push automatically triggers AAP project sync. + +## Related Documentation + +- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP +- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns +- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/insights/README.md new file mode 100644 index 00000000..661065fd --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/docs/insights/README.md @@ -0,0 +1,38 @@ +--- +title: Red Hat Lightspeed Documentation Overview +category: insights +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Documentation Overview + +This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. + +## Available Documentation + +### Current Documentation +- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk + - CVE identification and classification (Vulnerable vs Affected) + - Security Rules designation + - Red Hat severity ratings (Critical/Important/Moderate/Low) + - Risk assessment and prioritization methodology + - Priority decision matrix + - Integration with remediation workflows + +### Future Enhancements (P2 Priority) +- **remediation-workflow.md** - End-to-end remediation process (planned) +- **system-inventory.md** - Inventory management patterns (planned) + +## When to Use These Docs + +**Use vulnerability-logic.md when**: +- Performing CVE impact analysis +- Need to understand Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk assessments to management + +## Quick Links + +- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest +- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/insights/vulnerability-logic.md new file mode 100644 index 00000000..0edc1ebd --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/docs/insights/vulnerability-logic.md @@ -0,0 +1,568 @@ +--- +title: Red Hat Lightspeed Vulnerability Assessment Logic +category: insights +sources: + - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems + url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + sections: CVE identification, classification, threat intelligence + date_accessed: 2026-01-20 + - title: Generating Vulnerability Service Reports + url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + sections: Executive reports, CVE reports, data export + date_accessed: 2026-01-20 + - title: Red Hat CVE Database + url: https://access.redhat.com/security/security-updates/cve + sections: Official CVE entries, security updates + date_accessed: 2026-01-20 + - title: A Complete View of System Vulnerabilities + url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + sections: Vulnerability service overview, best practices + date_accessed: 2026-01-20 +tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "Red Hat Lightspeed" + - "CVE assessment" + - "vulnerability classification" + - "threat intelligence" + - "security rules" + - "affected but not vulnerable" + - "vulnerable status" + - "CVSS score" + - "severity rating" + - "remediation priority" +use_cases: + - "risk_assessment" + - "cve_impact_analysis" + - "remediation_prioritization" + - "vulnerability_reporting" +related_docs: + - "references/cvss-scoring.md" + - "ansible/cve-remediation-templates.md" + - "references/compliance-frameworks.md" +last_updated: 2026-01-20 +--- + +# Red Hat Lightspeed Vulnerability Assessment Logic + +This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. + +## Overview + +The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. + +**Key Capabilities**: +- Automated CVE identification on registered systems +- Threat intelligence integration +- Remediation playbook generation +- Executive and technical reporting +- Compliance tracking + +## CVE Classification + +### Vulnerable vs Affected But Not Vulnerable + +Red Hat Lightspeed categorizes CVEs into two primary classifications: + +#### Vulnerable +**Definition**: System is running flawed code with an **open path to exploitation**. + +**Characteristics**: +- CVE exists in installed package +- No mitigating factors present +- Exploit path is viable +- System is exposed to risk + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 +- Package is vulnerable +- No compensating controls +- **Status**: Vulnerable ✗ + +**Action Required**: Immediate remediation recommended + +#### Affected But Not Vulnerable +**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. + +**Characteristics**: +- CVE exists in installed package +- Mitigating factors prevent exploitation +- Configuration blocks exploit path +- Feature not enabled/used + +**Mitigating Factors**: +- SELinux blocking exploit path +- Firewall preventing network access +- Service not enabled/running +- Feature compiled out or disabled +- Red Hat backported fix without version number change + +**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE +- Package contains vulnerable code +- Firewall blocks external HTTP access +- **Status**: Affected but not vulnerable ⚠ + +**Action Required**: Lower priority, monitor for configuration changes + +### Security Rules + +**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. + +**Criteria for Security Rule Status**: +- Significant media coverage +- Active exploitation in the wild +- High CVSS score (typically ≥ 8.0) +- Affects critical infrastructure +- Red Hat Product Security team escalation + +**Example**: Log4Shell (CVE-2021-44228) +- CVSS 10.0 +- Widespread exploitation +- Affects many RHEL systems +- **Designated as Security Rule** + +**Impact**: +- Appears in executive dashboards +- Priority remediation recommended +- Enhanced reporting and tracking +- Compliance audit visibility + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** in addition to CVSS scores: + +| Severity | CVSS Range | Description | Response Time | +|----------|-----------|-------------|---------------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | +| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | + +**Red Hat Severity ≠ CVSS Score** + +Red Hat severity considers: +- **Exploitability**: How easily can it be exploited? +- **Impact**: What damage can it cause? +- **Affected configurations**: How common is the vulnerable configuration? +- **Compensating controls**: Are mitigations available? +- **Customer environment**: How are customers actually using the software? + +**Example**: +- **CVSS Score**: 8.5 (Important) +- **Red Hat Severity**: Moderate +- **Reason**: Requires non-default configuration rarely used in production + +## Vulnerability Assessment Workflow + +### Step 1: System Registration + +```bash +# Register system with Red Hat Lightspeed +insights-client --register + +# Verify registration +insights-client --status +``` + +**What Lightspeed Collects**: +- Installed packages and versions +- Running services +- System configuration +- SELinux status +- Firewall rules (high-level) +- Subscription data + +**Privacy**: Lightspeed collects system metadata only, not application data or files. + +### Step 2: CVE Identification + +Lightspeed automatically: +1. Scans installed packages against Red Hat CVE database +2. Compares package versions to affected ranges +3. Identifies all applicable CVEs +4. Updates daily as new CVEs published + +**CVE Sources**: +- Red Hat Product Security team +- NVD (National Vulnerability Database) +- Red Hat engineering analysis +- Customer-reported vulnerabilities + +### Step 3: Exploitability Analysis + +For each identified CVE, Lightspeed determines exploitability: + +**Factors Analyzed**: +- SELinux policies (can SELinux block the exploit?) +- Service status (is the vulnerable service running?) +- Network exposure (is the service accessible remotely?) +- Feature enablement (is the vulnerable feature enabled?) +- Configuration (does config prevent exploitation?) + +**Example Analysis**: +``` +CVE-2024-1234: httpd remote code execution +Package: httpd-2.4.37-1.el8 (vulnerable version) + +Exploitability Check: +✓ Service running: YES (httpd.service active) +✓ Network accessible: YES (port 80/443 open) +✓ SELinux blocking: NO (httpd_can_network_connect enabled) +✓ Feature enabled: YES (mod_cgi loaded) + +Conclusion: VULNERABLE +``` + +### Step 4: Threat Intelligence Integration + +Lightspeed integrates external threat intelligence: + +**Data Sources**: +- Known exploits in the wild +- CISA Known Exploited Vulnerabilities (KEV) catalog +- Security researcher disclosures +- Red Hat threat intelligence feeds + +**Impact on Priority**: +- **Active exploitation** → Escalate to Critical +- **Proof-of-concept available** → Increase priority +- **Theoretical only** → Standard priority + +### Step 5: Remediation Recommendation + +Lightspeed generates remediation recommendations: + +**Automated Playbook Available**: +``` +CVE-2024-1234 Remediation +───────────────────────── +Status: Automated remediation available +Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 +Reboot Required: No +Estimated Time: 5 minutes +Ansible Playbook: ✓ Available + +Actions: +1. Download playbook from Lightspeed console +2. Review and customize for your environment +3. Execute via Ansible Automation Platform +4. Verify remediation success +``` + +**Manual Remediation Required**: +``` +CVE-2024-5678 Remediation +───────────────────────── +Status: Manual remediation required +Package: custom-app-1.0 (not in Red Hat repos) +Guidance: Contact vendor for patch + +Actions: +1. Review vendor security advisory +2. Test vendor-provided patch in staging +3. Schedule maintenance window +4. Apply patch manually +5. Re-scan with Lightspeed to verify +``` + +## Remediation Prioritization + +### Priority Decision Matrix + +Lightspeed prioritizes CVEs based on multiple factors: + +| Factor | Weight | Description | +|--------|--------|-------------| +| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | +| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | +| **Business Impact** | 20% | System criticality (production vs dev/test) | +| **Active Threats** | 10% | Known exploitation in the wild | + +**Example Calculation**: +``` +CVE-2024-1234: +- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 +- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 +- Business Impact: Production → Score: 10 × 0.20 = 2.0 +- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 + +Total Priority Score: 8.5 / 10 → P1 (High Priority) +``` + +### Recommended Response Times + +| Priority | Response Time | Typical Scenarios | +|----------|---------------|-------------------| +| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | +| **P1** | 7 days | Important + Vulnerable + Production | +| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | +| **P3** | 90 days | Low severity or non-production systems | + +## Reporting Capabilities + +### Executive Reports (PDF) + +**Purpose**: High-level summary for leadership +**Contents**: +- Total CVE count by severity +- Trend analysis (improving/worsening) +- Top 10 critical CVEs +- Remediation progress metrics +- Compliance posture + +**Generated via Lightspeed Console or API** + +### CVE Reports (PDF) + +**Purpose**: Detailed technical analysis +**Contents**: +- Filtered CVE list (by severity, date, system) +- Per-CVE details (CVSS, description, affected systems) +- Remediation guidance +- Timeline for resolution + +### Vulnerability Data Export (CSV/JSON) + +**Purpose**: Integration with other tools (SIEM, ticketing) +**Contents**: +- Complete CVE dataset +- System-to-CVE mapping +- Remediation status +- Custom fields + +**Example Export**: +```csv +CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status +CVE-2024-1234,Important,7.5,15,Yes,Vulnerable +CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable +``` + +## Integration with Remediation Agent + +When using the remediation-agent plugin: + +### Step 1: Risk Assessment (Use This Doc) + +```yaml +# CVE Impact Skill uses vulnerability-logic.md +- Retrieve CVE data from Lightspeed +- Understand Red Hat severity rating +- Check if "vulnerable" or "affected but not vulnerable" +- Assess business impact +- Determine priority (P0/P1/P2/P3) +``` + +### Step 2: Remediation (Use Templates) + +```yaml +# Remediator Agent uses cve-remediation-templates.md +- If priority = P0/P1: Generate playbook immediately +- If priority = P2: Schedule for maintenance window +- If priority = P3: Add to backlog + +# Use appropriate template based on CVE type +- Package CVE → Template 1 (Package Update) +- Kernel CVE → Template 4 (Kernel Update) +- Service CVE → Template 2 (Service Restart) +``` + +## API Integration + +### Lightspeed API Endpoints + +**Get CVE Information**: +```bash +# Via lightspeed-mcp tool +vulnerability_get_cve_info(cve_id="CVE-2024-1234") + +# Returns: +{ + "cve_id": "CVE-2024-1234", + "severity": "Important", + "cvss_score": 7.5, + "publish_date": "2024-01-15", + "description": "...", + "affected_packages": ["httpd-2.4.37"], + "remediation_available": true +} +``` + +**List Affected Systems**: +```bash +vulnerability_list_cves(cve_id="CVE-2024-1234") + +# Returns: +{ + "systems": [ + { + "uuid": "abc-123", + "hostname": "web-server-01", + "status": "vulnerable", + "package": "httpd-2.4.37-1.el8" + } + ] +} +``` + +## Best Practices + +### 1. Regular Scanning + +- **Daily scans**: Automated via `insights-client` systemd timer +- **On-demand scans**: After patching to verify remediation +- **Post-change scans**: After system configuration changes + +```bash +# Force immediate scan +insights-client --force-reregister +``` + +### 2. Understand Context + +Don't remediate solely based on CVSS: +- ✅ Check Red Hat severity rating +- ✅ Verify "vulnerable" vs "affected but not vulnerable" +- ✅ Consider system criticality +- ✅ Review threat intelligence +- ❌ Don't auto-patch based on CVSS alone + +### 3. Prioritize Production + +``` +Production Vulnerable > Production Affected > Non-Prod Vulnerable +``` + +### 4. Track Remediation Progress + +- Use Lightspeed dashboards +- Export metrics for management +- Set SLAs per priority level +- Audit remediation completion + +### 5. Compliance Integration + +Map CVE remediation to compliance frameworks: +- **PCI-DSS**: High/Critical CVEs must be remediated per timeline +- **SOC 2**: Vulnerability management process documented +- **NIST**: CVE scoring aligns with NIST guidelines + +See: [compliance-frameworks.md](../references/compliance-frameworks.md) + +## Common Scenarios + +### Scenario 1: Security Rule Announced + +``` +Notification: CVE-2024-9999 designated as Security Rule +Severity: Critical (CVSS 9.8) +Affected Systems: 127 systems + +Workflow: +1. Read vulnerability-logic.md to understand Security Rule criteria +2. Assess: 127 systems × Critical severity = P0 priority +3. Generate batch remediation playbook (Template 6) +4. Execute rolling updates (serial: 5 for safety) +5. Verify all systems remediated within 24 hours +``` + +### Scenario 2: Affected But Not Vulnerable + +``` +CVE: CVE-2024-1111 +Status: Affected but not vulnerable +Reason: SELinux prevents exploitation + +Workflow: +1. Read vulnerability-logic.md to understand classification +2. Document why not vulnerable (SELinux policy blocks exploit) +3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) +4. Remediate during next maintenance window (not urgent) +5. Update documentation for audit purposes +``` + +### Scenario 3: No Automated Remediation + +``` +CVE: CVE-2024-2222 +Package: third-party-app-1.0 +Remediation: Manual steps required + +Workflow: +1. Check Red Hat CVE database for guidance +2. Contact vendor for security advisory +3. Create custom remediation playbook +4. Test in staging environment +5. Document manual steps for audit trail +``` + +## Troubleshooting + +### Issue: System Not Showing CVEs + +**Possible Causes**: +- System not registered with Lightspeed +- `insights-client` service not running +- Subscription not active +- Network connectivity issues + +**Resolution**: +```bash +# Check registration +insights-client --status + +# Re-register if needed +insights-client --register + +# Force update +insights-client --force-reregister +``` + +### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't + +**Possible Causes**: +- Lightspeed detected mitigating factor (SELinux, firewall) +- Configuration analysis outdated +- False positive + +**Resolution**: +1. Review system configuration +2. Verify mitigating factors actually present +3. If Lightspeed wrong: Contact Red Hat support +4. Override classification if necessary (document reason) + +### Issue: Remediation Playbook Failed + +**Possible Causes**: +- Package dependency conflicts +- Repository not accessible +- System requires reboot (not detected) + +**Resolution**: +1. Review playbook execution logs +2. Check `package_update.results` for errors +3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting +4. Re-run with increased verbosity: `ansible-playbook -vvv` + +## Related Documentation + +- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation +- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity + +## Official Red Hat Sources + +This document is derived from: + +1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** + https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview + +2. **Generating Vulnerability Service Reports** + https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index + +3. **Red Hat CVE Database** + https://access.redhat.com/security/security-updates/cve + +4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** + https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/references/README.md new file mode 100644 index 00000000..4c6481e1 --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/docs/references/README.md @@ -0,0 +1,39 @@ +--- +title: Reference Documentation Overview +category: references +last_updated: 2026-01-20 +--- + +# Reference Documentation Overview + +This directory contains reference materials supporting CVE remediation decisions. + +## Available Documentation + +### Current Documentation +- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings + - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) + - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) + - Why Red Hat severity ≠ CVSS score + - Priority decision matrix + - Real-world CVE examples (Log4Shell, Spectre, Sudo) + - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) + +### Future Enhancements (P2 Priority) +- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) +- **glossary.md** - Red Hat terminology reference (planned) + +## When to Use These Docs + +**Use cvss-scoring.md when**: +- Interpreting CVSS scores and vector strings +- Understanding Red Hat severity ratings +- Prioritizing CVEs for remediation +- Explaining risk to stakeholders +- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) + +## Quick Links + +- Red Hat Security: https://access.redhat.com/security/ +- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/references/cvss-scoring.md new file mode 100644 index 00000000..e0ccd3dc --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/docs/references/cvss-scoring.md @@ -0,0 +1,636 @@ +--- +title: CVSS Scoring and Red Hat Severity Mappings +category: references +sources: + - title: Severity Ratings (Red Hat Customer Portal) + url: https://access.redhat.com/security/updates/classification + sections: Red Hat severity ratings, CVSS usage + date_accessed: 2026-01-20 + - title: How We Classify Security Severity Levels + url: https://access.redhat.com/solutions/725593 + sections: Severity classification methodology + date_accessed: 2026-01-20 + - title: Security Update Policy + url: https://access.redhat.com/security/lifecycle-security-update-policy + sections: Security lifecycle, update policies + date_accessed: 2026-01-20 + - title: Product Security Center + url: https://access.redhat.com/security/ + sections: Security advisories, bulletins, CVSS data + date_accessed: 2026-01-20 +tags: [cvss, severity, scoring, risk-assessment, priority] +applies_to: [rhel6, rhel7, rhel8, rhel9] +semantic_keywords: + - "CVSS score" + - "severity rating" + - "Red Hat severity" + - "vulnerability scoring" + - "risk assessment" + - "priority matrix" + - "CVSS v3.1" + - "attack vector" + - "exploitability" + - "impact metrics" +use_cases: + - "risk_assessment" + - "cve_prioritization" + - "compliance_reporting" + - "stakeholder_communication" +related_docs: + - "insights/vulnerability-logic.md" + - "references/compliance-frameworks.md" + - "ansible/cve-remediation-templates.md" +last_updated: 2026-01-20 +--- + +# CVSS Scoring and Red Hat Severity Mappings + +This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. + +## Overview + +**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. + +**Key Concepts**: +- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics +- **CVSS Temporal Score**: Adjusts base score for current exploit availability +- **CVSS Environmental Score**: Organization-specific adjustments +- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products + +**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. + +## CVSS v3.1 Metrics + +CVSS base scores are calculated from 8 metrics across 3 categories: + +### Exploitability Metrics (How Easy to Exploit) + +#### 1. Attack Vector (AV) +**Question**: How is the vulnerability exploited? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | +| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | +| **Local (L)** | Medium | Requires local access to system | Privilege escalation | +| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | + +**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). + +#### 2. Attack Complexity (AC) +**Question**: How difficult is the attack to execute? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | +| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | + +**Impact on Priority**: Low complexity = easier exploitation = higher priority. + +#### 3. Privileges Required (PR) +**Question**: What privileges must attacker have? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | +| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | +| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | + +**Impact on Priority**: None required = widest attack surface = highest priority. + +#### 4. User Interaction (UI) +**Question**: Does exploitation require user action? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **None (N)** | Higher | Fully automated | Wormable exploit | +| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | + +**Impact on Priority**: No interaction = self-propagating = highest priority. + +### Scope Metric + +#### 5. Scope (S) +**Question**: Can the exploit impact resources beyond the vulnerable component? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | +| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | + +**Impact on Priority**: Changed scope = broader impact = higher priority. + +### Impact Metrics (What Damage) + +#### 6. Confidentiality (C) +**Question**: How much data can be disclosed? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | +| **Low (L)** | 0.22 | Limited disclosure | Single file read | +| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | + +#### 7. Integrity (I) +**Question**: How much can data/systems be modified? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | +| **Low (L)** | 0.22 | Limited modification | Single file write | +| **None (N)** | 0.00 | No integrity impact | Read-only exploit | + +#### 8. Availability (A) +**Question**: How much is service availability impacted? + +| Value | Score | Description | Example | +|-------|-------|-------------|---------| +| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | +| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | +| **None (N)** | 0.00 | No availability impact | Information disclosure only | + +## CVSS Score Calculation + +### Base Score Formula + +The CVSS v3.1 base score is calculated using a complex formula: + +``` +Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] + +If Scope Unchanged: + BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) + +If Scope Changed: + BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) + +Where: +Exploitability = 8.22 × AV × AC × PR × UI +``` + +**You don't need to calculate manually** - use CVSS calculators: +- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 +- Red Hat CVE pages include calculated scores + +### CVSS Severity Ratings (FIRST Standard) + +| CVSS Score | Severity | Description | +|------------|----------|-------------| +| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | +| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | +| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | +| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | +| 0.0 | **None** | No impact | + +## Red Hat Severity Ratings + +Red Hat uses a **4-point severity scale** that may differ from CVSS: + +### Red Hat Severity Levels + +| Severity | Typical CVSS Range | Description | Response Time | Example | +|----------|-------------------|-------------|---------------|---------| +| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | +| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | +| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | +| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | + +### Why Red Hat Severity ≠ CVSS Score + +Red Hat Product Security considers additional factors: + +**Exploitability in Real World**: +- Is the vulnerable code path actually used in typical RHEL deployments? +- Are default RHEL configurations vulnerable? +- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? + +**Example 1: CVSS Higher Than Red Hat Severity** +``` +CVE: CVE-2024-XXXX +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate + +Reason: +- Vulnerability requires non-default configuration +- Feature rarely enabled in production RHEL systems +- SELinux blocks exploitation in enforcing mode +- Red Hat downgrades to Moderate based on real-world risk +``` + +**Example 2: Red Hat Severity Higher Than CVSS** +``` +CVE: CVE-2024-YYYY +CVSS Score: 6.5 (Medium) +Red Hat Severity: Important + +Reason: +- CVSS calculated for generic software +- In RHEL context, vulnerability is more exploitable +- Commonly used RHEL service affected +- Red Hat elevates to Important based on customer environment +``` + +## Priority Decision Matrix + +Combine Red Hat severity with other factors to determine priority: + +### Priority Calculation + +| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | +|------------------|------------------|-------------------|---------------|----------|----------| +| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | +| Critical | Vulnerable | Yes | No | **P0** | 48 hours | +| Critical | Vulnerable | No | - | **P1** | 7 days | +| Important | Vulnerable | Yes | Yes | **P1** | 7 days | +| Important | Vulnerable | Yes | No | **P1** | 14 days | +| Important | Vulnerable | No | - | **P2** | 30 days | +| Moderate | Vulnerable | Yes | - | **P2** | 30 days | +| Moderate | Vulnerable | No | - | **P3** | 90 days | +| Low | Vulnerable | - | - | **P3** | 90 days | +| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | + +### Decision Tree + +``` +1. What is Red Hat Severity? + ├─ Critical → Continue to 2 + ├─ Important → Continue to 2 + ├─ Moderate → P2 (unless production + exploit = P2) + └─ Low → P3 + +2. Is system Vulnerable or Affected but not vulnerable? + ├─ Vulnerable → Continue to 3 + └─ Affected but not vulnerable → P3 + +3. Is this a production system? + ├─ Yes → Continue to 4 + └─ No → P1 (Critical) or P2 (Important) + +4. Is exploit known/available? + ├─ Yes → P0 + └─ No → P0 (Critical) or P1 (Important) +``` + +## CVSS Vector String + +CVSS scores include a **vector string** encoding all metrics: + +**Example Vector String**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Decoded**: +- **AV:N** - Attack Vector: Network +- **AC:L** - Attack Complexity: Low +- **PR:N** - Privileges Required: None +- **UI:N** - User Interaction: None +- **S:C** - Scope: Changed +- **C:H** - Confidentiality Impact: High +- **I:H** - Integrity Impact: High +- **A:H** - Availability Impact: High + +**Score**: 10.0 (Critical) + +**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. + +## Common CVE Patterns + +### Pattern 1: Critical Remote Code Execution + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact) + +**CVSS Score**: 9.0-10.0 +**Red Hat Severity**: Critical +**Example**: Heartbleed (CVE-2014-0160) + +**Priority**: P0 - Immediate remediation + +### Pattern 2: Privilege Escalation + +**Typical Metrics**: +- AV:L (Local) +- AC:L (Low complexity) +- PR:L (Low privileges) +- UI:N (No interaction) +- C:H/I:H/A:H (Full impact after escalation) + +**CVSS Score**: 7.8 +**Red Hat Severity**: Important +**Example**: Dirty COW (CVE-2016-5195) + +**Priority**: P1 - 7 day remediation window + +### Pattern 3: Information Disclosure + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:H/I:N/A:N (Confidentiality only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Important or Moderate (depends on data sensitivity) + +**Priority**: P1-P2 depending on data criticality + +### Pattern 4: Denial of Service + +**Typical Metrics**: +- AV:N (Network) +- AC:L (Low complexity) +- PR:N (No privileges) +- UI:N (No interaction) +- C:N/I:N/A:H (Availability only) + +**CVSS Score**: 7.5 +**Red Hat Severity**: Moderate (unless critical service) + +**Priority**: P2 - 30 days (unless high-availability requirement) + +## Compliance Framework Mappings + +### PCI-DSS Requirements + +**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes + +| CVSS Score | PCI-DSS Timeline | Remediation Deadline | +|------------|------------------|----------------------| +| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | +| 7.0-8.9 (High) | Within 30 days | **30 days max** | +| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | +| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | + +**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. + +### SOC 2 Trust Service Criteria + +**CC7.1**: Detect and respond to security incidents + +- **All CVSS 7.0+**: Must be tracked, remediated, documented +- **Remediation process**: Must demonstrate timely response +- **Audit trail**: Document priority decisions and remediation timeline + +### NIST 800-53 + +**SI-2**: Flaw Remediation + +- **High-impact systems**: Remediate high/critical within 30 days +- **Moderate-impact**: Remediate within 60 days +- **Low-impact**: Remediate within 90 days + +Map CVSS to NIST impact: +- CVSS 9.0-10.0 → High impact +- CVSS 7.0-8.9 → Moderate impact +- CVSS < 7.0 → Low impact + +See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings + +## Real-World Examples + +### Example 1: Log4Shell (CVE-2021-44228) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Network (remotely exploitable) +- Attack Complexity: Low (trivial to exploit) +- Privileges Required: None (unauthenticated) +- User Interaction: None (fully automated) +- Scope: Changed (RCE impacts entire system) +- Impact: H/H/H (full compromise) + +**Score**: 10.0 +**Red Hat Severity**: Critical +**Response**: P0 - Immediate remediation (24 hours) + +**Why P0**: +- Remotely exploitable +- No authentication +- Widespread usage (logging library) +- Active exploitation in wild +- Full system compromise possible + +### Example 2: Spectre Variant 1 (CVE-2017-5753) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N +``` + +**Metrics**: +- Attack Vector: Local (requires local access) +- Attack Complexity: High (timing attack, difficult) +- Privileges Required: Low (unprivileged user) +- Scope: Changed (can read across privilege boundaries) +- Impact: C:H (information disclosure), I:N, A:N + +**Score**: 5.6 (Medium by CVSS) +**Red Hat Severity**: Important (elevated due to CPU-level impact) +**Response**: P1 - 7 day remediation window + +**Why Red Hat Elevated**: +- Affects all modern CPUs +- Hardware-level vulnerability +- Difficult to fully mitigate +- Widespread impact across cloud/virtualization + +### Example 3: Sudo Heap Overflow (CVE-2021-3156) + +**CVSS v3.1 Vector**: +``` +CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +``` + +**Metrics**: +- Attack Vector: Local +- Attack Complexity: Low (reliable exploit available) +- Privileges Required: Low (any local user) +- Scope: Unchanged +- Impact: H/H/H (root access) + +**Score**: 7.8 (High) +**Red Hat Severity**: Important +**Response**: P1 - 7 days + +**Why Important**: +- Local privilege escalation to root +- sudo installed by default on all RHEL systems +- Any local user can exploit +- Reliable exploits available + +## Integration with Remediation Agent + +### CVE Impact Skill Workflow + +```yaml +# Step 1: Retrieve CVE data (use vulnerability-logic.md) +- Get CVSS score from Red Hat Lightspeed +- Get Red Hat severity rating +- Check vulnerable vs affected status + +# Step 2: Interpret CVSS (use THIS document) +- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) +- Map CVSS to Red Hat severity +- Check for discrepancies (explain to user) + +# Step 3: Calculate priority (use THIS document) +- Apply priority decision matrix +- Consider: severity + vulnerable status + production + exploit +- Output: P0/P1/P2/P3 with reasoning + +# Step 4: Recommend action +- P0: Immediate remediation (use cve-remediation-templates.md) +- P1: Schedule within 7 days +- P2: Next maintenance window +- P3: Backlog +``` + +### Remediator Agent Integration + +```yaml +# Input from CVE Impact Skill: +priority: P0 +cvss_score: 9.8 +red_hat_severity: Critical +reasoning: "Network RCE, no auth required, production systems affected" + +# Remediator Agent Decision: +if priority == "P0": + - Generate playbook immediately (Template 1 or 4) + - Recommend emergency change process + - Execute with minimal delay + +if priority == "P1": + - Generate playbook + - Schedule maintenance window + - Allow time for testing in staging + +if priority == "P2" or "P3": + - Add to remediation backlog + - Batch with other low-priority CVEs + - Include in next quarterly patching cycle +``` + +## Best Practices + +### 1. Trust Red Hat Severity Over Raw CVSS + +Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: +- Default RHEL configurations +- SELinux policies +- Firewall defaults +- Real-world usage patterns + +✅ **Use Red Hat severity for prioritization decisions** +⚠️ CVSS is a helpful reference but not the sole factor + +### 2. Document Prioritization Decisions + +Create an audit trail: +```markdown +CVE-2024-XXXX Prioritization Decision +Date: 2024-01-15 +CVSS Score: 8.5 (High) +Red Hat Severity: Moderate +Assigned Priority: P2 + +Rationale: +- CVSS high due to network vector +- Red Hat downgraded to Moderate (non-default config required) +- No active exploitation detected +- Non-production systems only affected +- Decision: P2 (30-day remediation window) + +Approved by: [Security Team Lead] +``` + +### 3. Re-evaluate on New Information + +Priorities can change: +- **Exploit published**: P2 → P1 +- **Active exploitation**: Any → P0 +- **Configuration change**: Affected but not vulnerable → Vulnerable + +Set up alerts for CVE status changes. + +### 4. Communicate in Business Terms + +Translate CVSS for stakeholders: + +**For Technical Teams**: +``` +CVE-2024-XXXX: CVSS 9.8 (Critical) +Attack Vector: Network, No Auth Required +Impact: Remote Code Execution +Priority: P0 - Patch within 24 hours +``` + +**For Management**: +``` +CVE-2024-XXXX: Critical Severity +Risk: Attackers can remotely compromise our web servers +Business Impact: Customer data exposure, service outage +Action Required: Emergency patching tonight +Estimated Downtime: 30 minutes per server +``` + +## Quick Reference + +### CVSS to Red Hat Severity (Typical Mapping) + +| CVSS Range | Red Hat Severity | Priority | Response Time | +|------------|------------------|----------|---------------| +| 9.0-10.0 | Critical | P0 | 24-48 hours | +| 7.0-8.9 | Important | P1 | 7 days | +| 4.0-6.9 | Moderate | P2 | 30 days | +| 0.1-3.9 | Low | P3 | 90 days | + +**Note**: Actual Red Hat severity may differ - always check CVE page. + +### Priority Override Conditions + +Escalate priority if: +- **Active exploitation** detected (any severity → P0) +- **Production critical system** affected (+1 priority level) +- **Compliance deadline** approaching (adjust to meet deadline) +- **Wormable vulnerability** (self-propagating → P0) + +Downgrade priority if: +- **Affected but not vulnerable** (-1 priority level) +- **Non-production only** (-1 priority level) +- **Compensating controls** in place (-1 priority level) + +## Related Documentation + +- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation +- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements + +## Official Red Hat Sources + +This document is derived from: + +1. **Severity Ratings** + https://access.redhat.com/security/updates/classification + +2. **How We Classify Security Severity Levels** + https://access.redhat.com/solutions/725593 + +3. **Security Update Policy** + https://access.redhat.com/security/lifecycle-security-update-policy + +4. **Product Security Center** + https://access.redhat.com/security/ + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-parameters.md new file mode 100644 index 00000000..b1465f4f --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-parameters.md @@ -0,0 +1,89 @@ +--- +title: Red Hat Lightspeed MCP - Parameter Reference +category: references +sources: + - title: Red Hat Lightspeed MCP + url: https://github.com/redhat/lightspeed-mcp + date_accessed: 2026-02-26 +tags: [lightspeed, mcp, parameters, inventory] +last_updated: 2026-02-26 +--- + +# Lightspeed MCP Parameter Reference + +Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** + +## inventory__list_hosts + +**Purpose**: List hosts with filtering and sorting options. + +**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. + +| Parameter | Type | Required | Example | Notes | +|-----------|------|----------|---------|-------| +| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | +| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | +| `page` | integer | No | `1` | Pagination page number. | + +**Correct**: +``` +inventory__list_hosts(per_page=10, display_name="") +``` + +**Wrong** (causes "Unexpected keyword argument" error): +``` +inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size +inventory__list_hosts(per_page="100") # ❌ Use integer, not string +``` + +## AAP MCP vs Lightspeed MCP + +| Server | Pagination Parameter | Type | +|--------|---------------------|------| +| lightspeed-mcp (inventory) | `per_page` | integer | +| aap-mcp-job-management | `page_size` | integer | +| aap-mcp-inventory-management | `page_size` | integer | + +Do not mix parameter names between servers. + +## vulnerability__get_cves + +**Purpose**: List CVEs affecting the account with filtering. + +**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | +| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | +| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | +| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | + +**For remediatable CVEs** (user asks "which CVEs can I remediate?"): +``` +vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") +``` + +**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). + +## vulnerability__get_system_cves + +**Purpose**: List CVEs affecting a specific system. Supports pagination. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | +| `limit` | integer | `100` | Records per page (default 10) | +| `offset` | integer | `0`, `100`, `200` | Pagination offset | +| `sort` | string | `"-public_date"` | Use `-` for descending | + +**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. + +## vulnerability__get_cve_systems + +**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. + +| Parameter | Type | Example | Notes | +|-----------|------|---------|-------| +| `cve` | string | `"CVE-2024-1234"` | Required | +| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-tool-failures.md new file mode 100644 index 00000000..f2c661b4 --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-tool-failures.md @@ -0,0 +1,69 @@ +--- +title: Lightspeed MCP Tool Failures — Handling and Workarounds +category: references +tags: [lightspeed, mcp, troubleshooting, errors] +last_updated: 2026-03-02 +--- + +# Lightspeed MCP Tool Failures + +When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. + +## Generic Pattern + +1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) +2. **Show** a user-friendly message explaining what happened and what we know +3. **Use** alternative tools to achieve the same goal when possible +4. **Do NOT** retry the failing tool—backend errors typically persist for the same request + +## Known Failures and Workarounds + +### vulnerability__get_cves — `limit_` Unexpected keyword argument + +**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` + +**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). + +**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: +``` +vulnerability__get_cves() +``` +Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. + +**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). + +### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) + +**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` + +**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. + +**User-friendly message**: +``` +⚠️ CVE explanation unavailable for this system + +The detailed "why this CVE affects your system" explanation could not be retrieved. +This sometimes happens when the system profile is missing module data. + +**What we know** (from other sources): +- CVE: [CVE-ID] +- Affected system: [hostname] +- Severity: [from get_cve] +- Affected packages: [from get_cve] + +**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. +``` + +**Workaround**: Synthesize from `get_cve` + `get_host_details`: +1. `get_cve(cve_id)` → affected_packages, severity, advisory +2. `get_host_details(system_id)` → installed_packages +3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." + +**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. + +### Other Tool Failures + +When a different tool fails with a similar cryptic error: +1. Apply the generic pattern (no raw error, user-friendly message) +2. Identify alternative tools that provide equivalent data +3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/references/skill-invocation.md new file mode 100644 index 00000000..132854c8 --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/docs/references/skill-invocation.md @@ -0,0 +1,35 @@ +--- +title: Skill Invocation Reference +category: references +tags: [skills, invocation, troubleshooting] +last_updated: 2026-03-02 +--- + +# Skill Invocation Reference + +Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). + +## Invoking Skills (All Sub-Skills) + +When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: + +- **Use the Skill tool** with the skill name. Format may vary by host: + - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar + - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` +- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. +- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. + +## If Validator Invocation Fails + +If validator invocation returns "No task found" or similar: + +1. **Do NOT block the workflow.** Proceed with a warning. +2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." +3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. +4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. + +The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. + +## Validation Freshness + +If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/rhel/README.md new file mode 100644 index 00000000..3f8526e9 --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/docs/rhel/README.md @@ -0,0 +1,40 @@ +--- +title: RHEL Documentation Overview +category: rhel +last_updated: 2026-01-20 +--- + +# RHEL Documentation Overview + +This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. + +## Available Documentation + +### Priority P0 (Core) +- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts + - RHEL 7/8/9 compatibility + - Package update patterns + - Repository management + - Subscription Manager integration + +### Future Enhancements (P1-P2 Priority) +- **selinux-context.md** - SELinux remediation patterns (planned) +- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) +- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) +- **systemd-services.md** - Service management patterns (planned) + +## When to Use These Docs + +**Use package-management.md when**: +- Creating playbooks that update packages +- Need to detect if reboot is required (needs-restarting) +- Working across multiple RHEL versions (7/8/9) +- Handling DNF/YUM differences +- Managing service restarts after package updates +- Troubleshooting repository or subscription issues + +## Quick Links + +- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 +- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index +- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/rhel/package-management.md new file mode 100644 index 00000000..f7e4252e --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/docs/rhel/package-management.md @@ -0,0 +1,738 @@ +--- +title: RHEL Package Management for CVE Remediation +category: rhel +sources: + - title: Managing Software with the DNF Tool (RHEL 9) + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + sections: DNF commands, updating packages, repository management + date_accessed: 2026-01-20 + - title: Software Management in RHEL 9 Adoption Guide + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + sections: RHEL 7/8/9 compatibility, migration considerations + date_accessed: 2026-01-20 + - title: Updating RHEL 9 Content + url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + sections: Package update procedures, reboot detection + date_accessed: 2026-01-20 +tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] +applies_to: [rhel7, rhel8, rhel9] +semantic_keywords: + - "DNF package manager" + - "YUM package manager" + - "package update" + - "repository management" + - "reboot detection" + - "systemd service management" + - "needs-restarting" + - "subscription manager" +use_cases: + - "package_update_cve" + - "rhel_version_compatibility" + - "reboot_detection" + - "service_restart_after_update" +related_docs: + - "ansible/cve-remediation-templates.md" + - "rhel/version-compatibility.md" + - "rhel/systemd-services.md" +last_updated: 2026-01-20 +--- + +# RHEL Package Management for CVE Remediation + +This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. + +## Overview + +Red Hat Enterprise Linux uses different package managers across versions: +- **RHEL 7**: YUM (Yellowdog Updater Modified) +- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias +- **RHEL 9**: DNF with `yum` as an alias + +**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. + +## DNF vs YUM Command Compatibility + +### Command Equivalence Table + +| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | +|-----------|--------------|----------------|-------| +| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | +| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | +| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | +| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | +| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | +| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | +| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | + +### Ansible Module Compatibility + +```yaml +# RHEL 7 - Use yum module +- name: Update packages (RHEL 7) + yum: + name: httpd + state: latest + when: ansible_distribution_major_version == "7" + +# RHEL 8/9 - Use dnf module (preferred) +- name: Update packages (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] + +# Universal approach - yum module works on all versions +- name: Update packages (All RHEL versions) + package: + name: httpd + state: latest + # Uses appropriate package manager automatically +``` + +## Package Update Patterns for CVE Remediation + +### Pattern 1: Single Package Update + +**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) + +```yaml +- name: Update vulnerable package + dnf: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version in ["8", "9"] + +- name: Update vulnerable package (RHEL 7) + yum: + name: httpd + state: latest + update_cache: true + register: package_update + when: ansible_distribution_major_version == "7" +``` + +**Key Options**: +- `state: latest` - Updates to newest available version +- `update_cache: true` - Refreshes repository metadata before update +- `register: package_update` - Captures update results for verification + +### Pattern 2: Multiple Related Packages + +**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) + +```yaml +- name: Update vulnerable packages and dependencies + dnf: + name: + - openssl + - openssl-libs + - openssl-devel + state: latest + update_cache: true + register: package_update +``` + +**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. + +### Pattern 3: Kernel Package Updates + +**Use Case**: Kernel CVEs requiring reboot + +```yaml +- name: Update kernel package + dnf: + name: kernel + state: latest + update_cache: true + register: kernel_update + +- name: Record current kernel before reboot + command: uname -r + register: current_kernel + changed_when: false + +# Reboot will be handled separately +# See: Template 4 in cve-remediation-templates.md +``` + +**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. + +### Pattern 4: Security-Only Updates + +**Use Case**: Apply only security updates, not all available updates + +```bash +# RHEL 8/9 - Security updates only +dnf update --security + +# RHEL 7 - Requires yum-plugin-security +yum update --security +``` + +**Ansible Equivalent**: +```yaml +- name: Apply security updates only (RHEL 8/9) + command: dnf update -y --security + register: security_updates + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Repository Management + +### Enabling/Disabling Repositories + +```yaml +- name: Enable repository for specific package + command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms + when: ansible_distribution_major_version == "9" + +- name: Update package from specific repo + dnf: + name: httpd + state: latest + enablerepo: rhel-9-for-x86_64-appstream-rpms +``` + +### Repository List (RHEL 9) + +Common repositories for CVE remediation: +- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages +- `rhel-9-for-x86_64-appstream-rpms` - Application streams +- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages + +### Verifying Repository Configuration + +```yaml +- name: List enabled repositories + command: subscription-manager repos --list-enabled + register: enabled_repos + changed_when: false + +- name: Display enabled repos + debug: + msg: "{{ enabled_repos.stdout_lines }}" +``` + +## Reboot Detection Patterns + +### Method 1: Check for Reboot-Required File + +```yaml +- name: Check if reboot is required (file-based) + stat: + path: /var/run/reboot-required + register: reboot_required_file + +- name: Notify if reboot needed + debug: + msg: "⚠️ System reboot required" + when: reboot_required_file.stat.exists +``` + +**Note**: Not all RHEL systems create this file. More reliable method below. + +### Method 2: needs-restarting Command (RHEL 8/9) + +**Most Reliable Method for RHEL 8/9** + +```yaml +- name: Check if reboot is required (needs-restarting) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Determine reboot requirement + set_fact: + reboot_required: "{{ needs_restarting.rc != 0 }}" + when: ansible_distribution_major_version in ["8", "9"] + +- name: Display reboot status + debug: + msg: "Reboot required: {{ reboot_required }}" +``` + +**Exit Codes**: +- `0` - No reboot required +- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) + +### Method 3: Check Specific Package Updates + +```yaml +- name: Check if kernel was updated + shell: | + LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') + RUNNING_KERNEL=$(uname -r) + if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then + echo "reboot_needed" + fi + register: kernel_check + changed_when: false + +- name: Set reboot flag if kernel changed + set_fact: + reboot_required: true + when: "'reboot_needed' in kernel_check.stdout" +``` + +### Comprehensive Reboot Detection + +**Recommended Pattern for CVE Remediation**: + +```yaml +- name: Comprehensive reboot detection + block: + - name: Check needs-restarting (RHEL 8/9) + command: needs-restarting -r + register: needs_restarting + failed_when: false + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + + - name: Check reboot-required file + stat: + path: /var/run/reboot-required + register: reboot_file + + - name: Check if kernel was updated + shell: | + rpm -q --last kernel | head -1 | \ + grep -q "$(uname -r)" || echo "kernel_updated" + register: kernel_check + changed_when: false + failed_when: false + + - name: Determine final reboot requirement + set_fact: + reboot_required: > + {{ + reboot_file.stat.exists | default(false) or + (needs_restarting.rc != 0 | default(false)) or + ('kernel_updated' in kernel_check.stdout) + }} + + - name: Display reboot requirement + debug: + msg: | + Reboot Required: {{ reboot_required }} + Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} +``` + +## Service Restart After Package Updates + +### Pattern 1: Restart Specific Services + +```yaml +- name: Restart httpd after package update + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required + +- name: Wait for service to be active + systemd: + name: httpd + state: started + retries: 3 + delay: 5 +``` + +### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) + +```yaml +- name: Find services that need restarting + command: needs-restarting -s + register: services_to_restart + changed_when: false + when: ansible_distribution_major_version in ["8", "9"] + +- name: Parse service names + set_fact: + service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" + when: services_to_restart.stdout_lines | length > 0 + +- name: Restart services that need it + systemd: + name: "{{ item }}" + state: restarted + loop: "{{ service_list }}" + when: + - service_list is defined + - not reboot_required + ignore_errors: true +``` + +**`needs-restarting -s` Output Example**: +``` +httpd.service +NetworkManager.service +sshd.service +``` + +### Pattern 3: Conditional Service Restart Based on Package + +```yaml +- name: Map packages to services + set_fact: + package_service_map: + httpd: httpd + nginx: nginx + sshd: sshd + openssl: [httpd, nginx, sshd] # Multiple services may use openssl + +- name: Restart services for updated packages + systemd: + name: "{{ package_service_map[item] }}" + state: restarted + loop: "{{ package_update.results | map(attribute='item') | list }}" + when: + - package_update is changed + - item in package_service_map + - not reboot_required +``` + +## Package Version Verification + +### Pre/Post Update Version Comparison + +```yaml +- name: Gather package facts before update + package_facts: + manager: auto + +- name: Record pre-update versions + set_fact: + pre_update_versions: "{{ ansible_facts.packages }}" + +- name: Update packages + dnf: + name: "{{ vulnerable_packages }}" + state: latest + register: package_update + +- name: Gather package facts after update + package_facts: + manager: auto + +- name: Compare versions + debug: + msg: | + Package: {{ item }} + Before: {{ pre_update_versions[item][0].version | default('not installed') }} + After: {{ ansible_facts.packages[item][0].version | default('not installed') }} + loop: "{{ vulnerable_packages }}" + when: item in ansible_facts.packages +``` + +### Verify Specific Package Version + +```yaml +- name: Verify package is at required version + shell: | + rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' + register: package_version + changed_when: false + +- name: Assert minimum version + assert: + that: + - package_version.stdout is version(minimum_version, '>=') + fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" + success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" +``` + +## Rollback and Backup Strategies + +### RHEL 8/9 Snapshot with Boom + +```yaml +- name: Install boom-boot (if not present) + dnf: + name: boom-boot + state: present + when: ansible_distribution_major_version in ["8", "9"] + +- name: Create pre-update snapshot + command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" + register: snapshot_result + ignore_errors: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Log snapshot creation + debug: + msg: "Snapshot created: {{ snapshot_result.stdout }}" + when: snapshot_result is success +``` + +### Package Downgrade (Emergency Rollback) + +```yaml +- name: Downgrade package to previous version + dnf: + name: httpd-2.4.37-1.el8 + state: present + allow_downgrade: true + when: ansible_distribution_major_version in ["8", "9"] + +- name: Downgrade package (RHEL 7) + yum: + name: httpd-2.4.37-1.el7 + state: present + when: ansible_distribution_major_version == "7" +``` + +**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. + +## Subscription Manager Integration + +### Verify System Registration + +```yaml +- name: Check subscription status + command: subscription-manager status + register: subscription_status + changed_when: false + failed_when: false + +- name: Assert system is registered + assert: + that: + - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" + fail_msg: "System is not properly subscribed to Red Hat repositories" + success_msg: "System subscription is current" +``` + +### Refresh Subscription + +```yaml +- name: Refresh subscription data + command: subscription-manager refresh + when: subscription_status.rc != 0 + +- name: Update repository metadata + command: dnf clean all && dnf makecache + when: ansible_distribution_major_version in ["8", "9"] +``` + +## RHEL Version-Specific Considerations + +### RHEL 7 + +- **Package Manager**: YUM (Python 2.7-based) +- **Systemd Version**: 219 +- **Reboot Detection**: No `needs-restarting -r`, use alternative methods +- **Security Updates**: Requires `yum-plugin-security` package + +```yaml +- name: Install security plugin (RHEL 7) + yum: + name: yum-plugin-security + state: present + when: ansible_distribution_major_version == "7" +``` + +### RHEL 8 + +- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias +- **Systemd Version**: 239 +- **Reboot Detection**: `needs-restarting -r` available +- **Module Streams**: AppStream concept introduced + +```yaml +- name: Enable module stream (RHEL 8) + command: dnf module enable httpd:2.4 -y + when: ansible_distribution_major_version == "8" +``` + +### RHEL 9 + +- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias +- **Systemd Version**: 252 +- **Reboot Detection**: `needs-restarting -r` available +- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification + +```yaml +- name: Install multisig plugin (RHEL 9.7+) + dnf: + name: python3-dnf-plugin-multisig + state: present + when: + - ansible_distribution_major_version == "9" + - ansible_distribution_version is version('9.7', '>=') +``` + +## Common Pitfalls and Solutions + +### Pitfall 1: Not Refreshing Repository Cache + +**Problem**: Updates fail or don't detect new packages +**Solution**: Always use `update_cache: true` + +```yaml +# ❌ Bad - may miss new package versions +- dnf: + name: httpd + state: latest + +# ✅ Good - ensures latest metadata +- dnf: + name: httpd + state: latest + update_cache: true +``` + +### Pitfall 2: Ignoring Reboot Requirements + +**Problem**: CVE remains exploitable after "update" +**Solution**: Always check and handle reboots + +```yaml +# ✅ Complete pattern +- name: Update package + dnf: + name: kernel + state: latest + +- name: Check reboot requirement + command: needs-restarting -r + register: needs_reboot + failed_when: false + +- name: Notify if reboot needed + debug: + msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" + when: needs_reboot.rc != 0 +``` + +### Pitfall 3: Not Verifying Package Update Success + +**Problem**: Package update silently fails, CVE remains +**Solution**: Use `register` and verify changes + +```yaml +- name: Update package + dnf: + name: httpd + state: latest + register: package_update + +- name: Verify update occurred + assert: + that: + - package_update is changed or package_update is success + fail_msg: "Package update failed - CVE remediation incomplete" +``` + +### Pitfall 4: Restarting Services When Reboot Required + +**Problem**: Wasted effort, service restart won't apply kernel updates +**Solution**: Conditional service restarts + +```yaml +- name: Restart service only if no reboot needed + systemd: + name: httpd + state: restarted + when: + - package_update is changed + - not reboot_required # Don't restart if rebooting anyway +``` + +### Pitfall 5: Using Wrong Package Manager Module + +**Problem**: Playbook fails on different RHEL versions +**Solution**: Use version-conditional tasks or `package` module + +```yaml +# ✅ Best - works on all RHEL versions +- name: Update package (universal) + package: + name: httpd + state: latest + +# ✅ Also good - version-specific +- name: Update package (RHEL 8/9) + dnf: + name: httpd + state: latest + when: ansible_distribution_major_version in ["8", "9"] +``` + +## Quick Reference Commands + +### Package Operations +```bash +# Update single package +dnf update httpd + +# Update all security patches +dnf update --security + +# Update multiple packages +dnf update httpd httpd-tools + +# Check for available updates +dnf check-update + +# List installed packages +dnf list installed + +# Show package info +dnf info httpd + +# Search for package +dnf search webserver +``` + +### Reboot Detection +```bash +# Check if reboot needed (RHEL 8/9) +needs-restarting -r + +# List services needing restart +needs-restarting -s + +# Check current kernel vs installed +uname -r +rpm -q kernel --last | head -1 +``` + +### Repository Management +```bash +# List enabled repos +subscription-manager repos --list-enabled + +# Enable specific repo +subscription-manager repos --enable=repo-name + +# Refresh repo metadata +dnf clean all && dnf makecache +``` + +## Related Documentation + +- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns +- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences +- **[Systemd Services](systemd-services.md)** - Service management patterns +- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates + +## Official Red Hat Sources + +This document is derived from: + +1. **Managing Software with the DNF Tool (RHEL 9)** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index + +2. **Software Management in RHEL 9 Adoption Guide** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 + +3. **Updating RHEL 9 Content** + https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool + +**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 +**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/testing/aap-integration-test-guide.md new file mode 100644 index 00000000..6c122770 --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/docs/testing/aap-integration-test-guide.md @@ -0,0 +1,649 @@ +--- +title: AAP Integration Test Guide +category: testing +sources: + - title: Internal Testing Documentation + date_accessed: 2026-02-24 +tags: [testing, aap-integration, workflow-verification, remediation-testing] +semantic_keywords: [aap integration testing, workflow verification, remediation test] +use_cases: [remediation, playbook-executor] +related_docs: [aap-job-execution.md, playbook-integration-aap.md] +last_updated: 2026-02-24 +--- + +# AAP Integration Test Guide + +## Overview + +This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. + +## Prerequisites for Testing + +### Required Setup + +1. **AAP Environment**: + - AAP 2.4+ instance accessible + - Valid API token with appropriate permissions + - At least one project configured + - At least one inventory with test systems + - At least one job template (or ability to create one) + +2. **Environment Variables**: + ```bash + export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" + export AAP_API_TOKEN="your-api-token" + ``` + +3. **Test Systems**: + - At least 2-3 RHEL systems in AAP inventory + - Systems registered with Red Hat Lightspeed + - Systems have known CVEs for testing + - SSH access configured with credentials in AAP + +4. **MCP Configuration**: + - `rh-sre/.mcp.json` configured with AAP MCP servers + - `lightspeed-mcp` configured and working + - All environment variables set + +### Verification Checklist + +Before starting tests, verify: + +- [ ] AAP Web UI accessible (your AAP instance URL) +- [ ] Can log in with your credentials +- [ ] API token has been generated +- [ ] Environment variables are set (run: `env | grep AAP`) +- [ ] Test systems visible in AAP inventory +- [ ] Test systems have CVEs in Red Hat Lightspeed +- [ ] Git repository available for playbook storage + +## Test Plan Structure + +``` +Test Phase 1: Component Testing +├─ Test 1.1: AAP MCP Validator +├─ Test 1.2: Job Template Lister +├─ Test 1.3: Playbook Generator +└─ Test 1.4: Inventory Access + +Test Phase 2: Integration Testing +├─ Test 2.1: Template Selection Workflow +├─ Test 2.2: Dry-Run Execution +├─ Test 2.3: Production Execution +└─ Test 2.4: Error Handling + +Test Phase 3: End-to-End Testing +├─ Test 3.1: Full Remediator Workflow +├─ Test 3.2: Multi-CVE Remediation +└─ Test 3.3: Partial Failure Recovery + +Test Phase 4: Performance Testing +└─ Test 4.1: Large-Scale Execution +``` + +## Test Phase 1: Component Testing + +### Test 1.1: AAP MCP Validator + +**Objective**: Verify AAP MCP server connectivity and resource availability. + +**Steps**: +1. Invoke the mcp-aap-validator skill +2. Observe validation checks +3. Confirm all checks pass + +**Expected Results**: +``` +✓ AAP MCP Validation: PASSED + +Configuration: +✓ MCP server aap-mcp-job-management configured +✓ MCP server aap-mcp-inventory-management configured +✓ Environment variable AAP_MCP_SERVER is set +✓ Environment variable AAP_API_TOKEN is set +✓ Job management server connectivity verified +✓ Inventory management server connectivity verified + +Resources: +✓ Found N job template(s) available +✓ Found M inventory/inventories available + +Ready to execute AAP operations. +``` + +**Pass Criteria**: +- All configuration checks pass +- Both MCP servers connect successfully +- At least 1 job template found +- At least 1 inventory found + +**Troubleshooting**: +- If fails: Review error message and fix configuration +- If partial: Note warnings but may proceed if resources exist +- If connection fails: Check AAP server status and credentials + +### Test 1.2: Job Template Lister + +**Objective**: Verify ability to list and filter job templates. + +**Test Command**: Use `job_templates_list` MCP tool via skill + +**Steps**: +1. Request list of all job templates +2. Verify response contains expected templates +3. Note template IDs for later tests + +**Expected Results**: +- List of templates with IDs, names, projects, inventories +- At least 1 template suitable for remediation + +**Pass Criteria**: +- Tool returns valid response +- Template data includes required fields +- Can identify suitable template for testing + +### Test 1.3: Playbook Generator + +**Objective**: Verify playbook generation from CVE data. + +**Steps**: +1. Invoke playbook-generator skill with a known CVE +2. Review generated playbook +3. Verify playbook has required sections + +**Test Input**: +- CVE ID: Use a real CVE affecting your test systems +- Target systems: Your test system UUIDs + +**Expected Results**: +- Valid Ansible YAML playbook generated +- Includes: pre-flight checks, package updates, service restarts +- Follows Red Hat best practices +- Has proper error handling + +**Pass Criteria**: +- Playbook is syntactically valid YAML +- Contains all remediation tasks +- Includes backup/rollback steps +- Has audit logging + +### Test 1.4: Inventory Access + +**Objective**: Verify ability to query AAP inventories and hosts. + +**Test Command**: Use `inventories_list` and `hosts_list` MCP tools + +**Steps**: +1. List all inventories +2. Select test inventory +3. List hosts in that inventory +4. Verify test systems are present + +**Expected Results**: +- Inventory list returned +- Can query hosts within inventory +- Test systems visible with correct metadata + +**Pass Criteria**: +- At least 1 inventory returned +- Hosts query succeeds +- Test systems found in inventory + +## Test Phase 2: Integration Testing + +### Test 2.1: Template Selection Workflow + +**Objective**: Test the template selection and creation workflow. + +**Scenario A: Existing Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Skill lists available templates +3. Select an existing compatible template +4. Verify selection is accepted + +**Expected Results**: +``` +Found N compatible job template(s): + +1. "CVE Remediation Template" (ID: 10) + - Inventory: Production Servers (1) + - Project: Remediation Playbooks (5) + - Credentials: ✓ Configured + +Select template number or "create" for new: 1 + +✓ Using template: CVE Remediation Template (ID: 10) +``` + +**Pass Criteria**: +- Templates listed successfully +- User can select a template +- Selection is confirmed + +**Scenario B: Create New Template** + +**Steps**: +1. Invoke playbook-executor skill +2. Choose "create" option +3. Follow template creation guidance +4. Verify template appears in AAP + +**Expected Results**: +- User guided through Web UI creation +- Template created with correct settings +- Template visible in `job_templates_list` + +**Pass Criteria**: +- Guidance is clear and actionable +- Template created successfully +- Template has required configuration + +### Test 2.2: Dry-Run Execution + +**Objective**: Test check mode (dry-run) execution. + +**Steps**: +1. Generate a remediation playbook +2. Select job template +3. Choose "yes" when asked about dry-run +4. Wait for dry-run to complete +5. Review dry-run results + +**Expected Results**: +``` +⏳ Dry-run in progress... + +Job ID: 1234 +Status: running + +# Dry-Run Results + +## Job Summary +**Job ID**: 1234 +**Status**: ✓ Successful (Check Mode) +**Duration**: 2m 15s + +## Simulated Changes +| Host | Would Change | OK | Failed | Status | +|------|--------------|-----|--------|--------| +| test-01 | 2 | 6 | 0 | ✓ Ready | +| test-02 | 2 | 6 | 0 | ✓ Ready | + +✓ No errors detected in dry-run +``` + +**Pass Criteria**: +- Job launches with `job_type: "check"` +- Execution completes successfully +- Results show "would change" counts +- No actual changes made to systems +- User asked to proceed with actual execution + +### Test 2.3: Production Execution + +**Objective**: Test actual playbook execution (run mode). + +**Steps**: +1. After successful dry-run, approve actual execution +2. Monitor execution progress +3. Wait for completion +4. Review execution report + +**Expected Results**: +``` +⏳ Execution in progress... + +Job ID: 1235 +Status: running + +# Playbook Execution Report + +## Job Summary +**Job ID**: 1235 +**Status**: ✅ Successful +**Duration**: 3m 45s + +## Per-Host Results +| Host | OK | Changed | Failed | Unreachable | Status | +|------|-----|---------|--------|-------------|--------| +| test-01 | 6 | 2 | 0 | 0 | ✅ Success | +| test-02 | 6 | 2 | 0 | 0 | ✅ Success | + +**Summary**: 2 of 2 hosts successfully remediated + +## Next Steps +☐ Verify remediation with remediation-verifier skill +``` + +**Pass Criteria**: +- Job launches with `job_type: "run"` +- Real-time progress displayed +- Execution completes successfully +- All hosts show success status +- Comprehensive report generated +- AAP URL provided for detailed view + +### Test 2.4: Error Handling + +**Objective**: Test error handling and recovery. + +**Scenario A: Partial Host Failure** + +**Setup**: +- Use 3 test systems +- Cause failure on 1 system (e.g., remove package, stop service) + +**Steps**: +1. Execute remediation playbook +2. Observe partial failure +3. Review error report +4. Choose to relaunch for failed host + +**Expected Results**: +``` +⚠️ Playbook Execution Completed with Failures + +Job ID: 1236 +Systems Remediated: 2 of 3 +Failed Systems: test-03 + +## Failed Tasks Details +**Host**: test-03 +**Task**: Update package httpd +**Error**: "No package httpd available" +**Recommendation**: Check repository configuration + +Would you like to: +1. Relaunch for failed host only +2. Fix issues manually and relaunch +``` + +**Pass Criteria**: +- Failure detected and reported +- Specific error message provided +- Troubleshooting guidance given +- Relaunch option offered +- Can successfully relaunch for failed host only + +**Scenario B: Connection Failure** + +**Setup**: +- Block SSH to one test system (firewall rule) + +**Steps**: +1. Execute remediation playbook +2. Observe connection failure +3. Review error categorization + +**Expected Results**: +``` +❌ Host test-02: unreachable + +**Error Category**: Connection Failure + +**Troubleshooting**: +1. Check SSH service: systemctl status sshd +2. Verify firewall: firewall-cmd --list-all +3. Test connectivity: ping test-02 +``` + +**Pass Criteria**: +- Connection failure detected +- Categorized as connection error +- Specific troubleshooting provided + +## Test Phase 3: End-to-End Testing + +### Test 3.1: Full Remediator Workflow + +**Objective**: Test complete CVE remediation from analysis to verification. + +**Steps**: +1. **Invoke remediation skill** with a known CVE +2. **Impact Analysis**: Review CVE risk assessment +3. **CVE Validation**: Confirm CVE is valid and has remediation +4. **System Context**: Review affected systems and strategy +5. **Playbook Generation**: Review generated playbook, approve +6. **Dry-Run**: Run check mode, review results, approve production +7. **Execution**: Monitor real execution, review report +8. **Verification**: Verify CVE status updated in Lightspeed + +**Test Input**: +``` +User: "Remediate CVE-YYYY-NNNNN on my test systems" +``` + +**Expected Flow**: +1. Agent analyzes CVE impact +2. Agent validates CVE exists +3. Agent gathers system context +4. Agent generates playbook +5. Agent offers dry-run → User approves +6. Agent shows dry-run results +7. Agent asks for production execution → User approves +8. Agent executes playbook +9. Agent reports success +10. Agent suggests verification +11. User invokes remediation-verifier +12. Verifier confirms CVE resolved + +**Pass Criteria**: +- All steps complete without errors +- User prompted at appropriate points +- Dry-run shows simulated changes +- Production execution succeeds +- CVE status updated in Lightspeed +- Comprehensive report at each stage + +**Timeline**: ~10-15 minutes for full workflow + +### Test 3.2: Multi-CVE Remediation + +**Objective**: Test batch remediation of multiple CVEs. + +**Steps**: +1. Invoke remediation skill with 2-3 CVEs +2. Verify agent handles batch processing +3. Confirm single consolidated playbook generated +4. Execute remediation +5. Verify all CVEs resolved + +**Test Input**: +``` +User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" +``` + +**Expected Results**: +- Agent processes all CVEs +- Consolidated playbook with all fixes +- Single job execution covering all changes +- Report shows results per CVE + +**Pass Criteria**: +- Batch processing works correctly +- Playbook includes all remediation tasks +- Execution handles multiple changes +- Verification confirms all CVEs resolved + +### Test 3.3: Partial Failure Recovery + +**Objective**: Test recovery from partial failures. + +**Scenario**: 5 test systems, 2 fail during execution + +**Steps**: +1. Execute remediation on 5 systems +2. Observe 2 failures +3. Review error analysis +4. Fix issues on failed systems +5. Relaunch for failed systems only +6. Verify all systems eventually succeed + +**Expected Results**: +- Partial success reported (3 of 5) +- Failed systems identified +- Relaunch targets only failed systems +- Second execution succeeds +- Final report shows 5 of 5 success + +**Pass Criteria**: +- Partial failure handled gracefully +- Relaunch doesn't re-run successful hosts +- Ultimate success achieved +- Audit trail shows full history + +## Test Phase 4: Performance Testing + +### Test 4.1: Large-Scale Execution + +**Objective**: Test performance with larger number of systems. + +**Setup**: +- Use 20+ systems in inventory +- Single CVE affecting all systems + +**Steps**: +1. Execute remediation targeting 20+ systems +2. Monitor execution time +3. Review AAP resource usage +4. Verify all systems succeed + +**Expected Results**: +- Execution completes in reasonable time +- Progress monitoring works at scale +- All systems remediated successfully +- Report generated efficiently + +**Pass Criteria**: +- Job completes within expected timeframe +- No timeouts or performance degradation +- Monitoring provides useful progress updates +- Final report is comprehensive + +**Performance Benchmarks**: +- 10 systems: ~5-10 minutes +- 20 systems: ~10-20 minutes +- 50 systems: ~20-40 minutes +(Times vary based on package size and network) + +## Test Reporting Template + +### Test Execution Report + +```markdown +# AAP Integration Test Report + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [AAP Server URL] +**Test Phase**: [1-4] + +## Summary +- Tests Run: N +- Tests Passed: N +- Tests Failed: N +- Pass Rate: NN% + +## Phase 1: Component Testing +- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL +- [ ] Test 1.2: Job Template Lister - PASS/FAIL +- [ ] Test 1.3: Playbook Generator - PASS/FAIL +- [ ] Test 1.4: Inventory Access - PASS/FAIL + +## Phase 2: Integration Testing +- [ ] Test 2.1: Template Selection - PASS/FAIL +- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL +- [ ] Test 2.3: Production Execution - PASS/FAIL +- [ ] Test 2.4: Error Handling - PASS/FAIL + +## Phase 3: End-to-End Testing +- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL +- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL +- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL + +## Phase 4: Performance Testing +- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL + +## Issues Found +1. [Issue description] - Severity: High/Medium/Low +2. [Issue description] - Severity: High/Medium/Low + +## Recommendations +1. [Recommendation] +2. [Recommendation] + +## Sign-Off +Tested by: [Name] +Approved by: [Name] +Date: YYYY-MM-DD +``` + +## Common Issues and Solutions + +### Issue: "AAP MCP Validation Failed" + +**Symptoms**: Validation fails with connection errors + +**Solutions**: +1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) +2. Check API token is valid and not expired +3. Ensure AAP server is accessible from your network +4. Review AAP MCP server logs for errors + +### Issue: "No Job Templates Found" + +**Symptoms**: Validation passes but no templates available + +**Solutions**: +1. Create job template via AAP Web UI +2. Ensure project is synced and contains playbooks +3. Verify inventory is configured +4. Check credentials are attached to template + +### Issue: "Dry-Run Shows No Changes" + +**Symptoms**: Dry-run completes but reports 0 changes + +**Solutions**: +1. Verify systems actually need remediation +2. Check playbook targets correct hosts +3. Ensure package names are correct +4. Review playbook conditionals (when clauses) + +### Issue: "Execution Hangs" + +**Symptoms**: Job starts but never completes + +**Solutions**: +1. Check AAP Web UI for job status +2. Review job output for stuck tasks +3. Verify systems are reachable +4. Increase job timeout in template settings + +## Sign-Off Criteria + +Before considering AAP integration complete, verify: + +- [ ] All Phase 1 tests pass +- [ ] All Phase 2 tests pass +- [ ] At least Test 3.1 passes (full workflow) +- [ ] No critical issues remain +- [ ] Documentation is accurate +- [ ] Examples work as described +- [ ] Performance is acceptable + +## Next Steps After Testing + +1. **Document Results**: Complete test report template +2. **Fix Issues**: Address any failures found +3. **Update Documentation**: Correct any inaccuracies +4. **User Acceptance**: Have users test workflow +5. **Production Rollout**: Enable for production use + +## Related Documentation + +- [AAP Job Execution Guide](../ansible/aap-job-execution.md) +- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) +- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__system-context/environment/mcp-servers/mock-lightspeed-mcp.py b/evaluation/without_skills/rh-sre__system-context/environment/mcp-servers/mock-lightspeed-mcp.py new file mode 100644 index 00000000..e826c96e --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/environment/mcp-servers/mock-lightspeed-mcp.py @@ -0,0 +1,759 @@ +#!/usr/bin/env python3 +""" +Mock Lightspeed MCP Server + +Simulates the Red Hat Lightspeed MCP server for the rh-sre-fleet-inventory +benchmark task. Implements the MCP protocol via FastMCP so that agents can +call get_host_details, get_cve_systems, get_cves, get_cve, and +create_vulnerability_playbook as real MCP tools. + +Fleet composition (63 systems total): + - 30 production (web, db, app, lb, monitoring, cache) + - 15 staging + - 10 development + - 5 QA + - 3 legacy (ambiguous tags — no explicit environment) + +CVE data (5 CVEs): + - CVE-2024-12345 Critical 9.8 RCE in HTTP processing + - CVE-2024-54321 Important 7.5 SQL injection in DB parser + - CVE-2024-11111 Moderate 5.3 Info disclosure in logging + - CVE-2024-98765 Important 8.1 DoS in load balancer + - CVE-2024-22222 Low 3.1 Info disclosure in monitoring +""" + +import os +import random +from datetime import datetime, timedelta +from typing import Optional + +from fastmcp import FastMCP + +random.seed(42) + +REFERENCE_TIME = datetime(2026, 2, 15, 12, 0, 0) + + +# --------------------------------------------------------------------------- +# Mock fleet data +# --------------------------------------------------------------------------- + +def _ts(delta: timedelta) -> str: + """Return an ISO timestamp offset from REFERENCE_TIME.""" + return (REFERENCE_TIME - delta).isoformat() + "Z" + + +def _system_profile_for_host(host_type: str, rhel_version: str, sid: int) -> dict: + """Generate system_profile fields for a host based on type and RHEL version.""" + el = "el9" if rhel_version.startswith("9") else "el8" + kernel = f"5.14.0-362.24.1.{el}_3.x86_64" if "9" in rhel_version else f"4.18.0-477.27.1.{el}.x86_64" + base_pkgs = [ + {"name": "kernel-core", "version": f"5.14.0-362.24.1.{el}.x86_64"}, + {"name": "httpd", "version": f"2.4.57-5.{el}"}, + {"name": "sshd", "version": f"8.9p1-23.{el}"}, + {"name": "firewalld", "version": f"1.2.5-4.{el}"}, + {"name": "systemd", "version": f"250-19.{el}"}, + ] + if "web" in host_type or "lb" in host_type: + base_pkgs.extend([ + {"name": "nginx", "version": f"1.24.1-3.{el}"}, + {"name": "openssl", "version": f"3.0.7-24.{el}"}, + ]) + elif "db" in host_type: + base_pkgs.extend([ + {"name": "postgresql", "version": f"15.4-1.{el}"}, + {"name": "openssl", "version": f"3.0.7-24.{el}"}, + ]) + elif "mon" in host_type: + base_pkgs.extend([ + {"name": "prometheus", "version": f"2.45.0-1.{el}"}, + {"name": "node_exporter", "version": f"1.6.1-2.{el}"}, + ]) + else: + base_pkgs.extend([ + {"name": "java-17-openjdk", "version": f"17.0.8-4.{el}"}, + {"name": "openssl", "version": f"3.0.7-24.{el}"}, + ]) + services = ["sshd.service", "firewalld.service", "chronyd.service"] + if "web" in host_type or "lb" in host_type: + services.append("httpd.service") + elif "db" in host_type: + services.extend(["postgresql.service", "postgresql-15.service"]) + elif "mon" in host_type: + services.extend(["prometheus.service", "node_exporter.service"]) + else: + services.append("httpd.service") + ip_octet = 10 + (sid % 245) + mac_hex = f"{(sid % 256):02x}" + return { + "installed_packages": base_pkgs[:8], + "running_services": services, + "network_interfaces": [ + {"name": "eth0", "ipv4": [f"10.0.1.{ip_octet}"], "mac": f"52:54:00:a1:b2:{mac_hex}"}, + {"name": "lo", "ipv4": ["127.0.0.1"], "mac": "00:00:00:00:00:00"}, + ], + "kernel_version": kernel, + } + + +def generate_mock_systems() -> list[dict]: + """Generate 63 mock systems with realistic distribution.""" + systems: list[dict] = [] + sid = 1 + + # --- Production (30) --------------------------------------------------- + + # Web servers (8) + for i in range(1, 9): + rhel = "9.3" if i <= 5 else ("9.2" if i <= 7 else "8.9") + stale = i == 7 + tags = ["production", "web-tier"] + if i <= 4: + tags.extend(["customer-facing", "pci-compliant", "high-availability"]) + if i <= 2: + tags.append("critical") + systems.append({ + "id": f"sys-{sid:03d}-web-prod-{i:02d}", + "display_name": f"web-server-{i:02d}.prod.example.com", + "fqdn": f"web-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=9) if stale else timedelta(hours=random.randint(1, 20))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Database servers (6) + for i in range(1, 7): + rhel = "9.3" if i <= 4 else "9.2" + stale = i == 5 + tags = ["production", "database-tier", "critical"] + if i <= 4: + tags.extend(["pci-compliant", "soc2-compliant", "high-availability"]) + if i <= 2: + tags.append("customer-data") + systems.append({ + "id": f"sys-{sid:03d}-db-prod-{i:02d}", + "display_name": f"db-server-{i:02d}.prod.example.com", + "fqdn": f"db-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=10) if stale else timedelta(hours=random.randint(1, 18))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Application servers (10) + for i in range(1, 11): + rhel = "8.9" if i <= 6 else ("9.2" if i <= 8 else "9.3") + stale = i == 9 + tags = ["production", "app-tier"] + if i <= 3: + tags.extend(["customer-facing", "pci-compliant", "soc2-compliant"]) + elif i <= 6: + tags.extend(["hipaa-compliant", "soc2-compliant"]) + if i <= 5: + tags.append("high-availability") + systems.append({ + "id": f"sys-{sid:03d}-app-prod-{i:02d}", + "display_name": f"app-server-{i:02d}.prod.example.com", + "fqdn": f"app-server-{i:02d}.prod.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=8) if stale else timedelta(hours=random.randint(1, 22))), + "tags": tags, + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + # Load balancers (3) + for i in range(1, 4): + tags = ["production", "loadbalancer", "critical", "high-availability"] + if i <= 2: + tags.append("customer-facing") + systems.append({ + "id": f"sys-{sid:03d}-lb-prod-{i:02d}", + "display_name": f"lb-server-{i:02d}.prod.example.com", + "fqdn": f"lb-server-{i:02d}.prod.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(1, 12))), + "tags": tags, + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Monitoring (2) + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-mon-prod-{i:02d}", + "display_name": f"monitor-server-{i:02d}.prod.example.com", + "fqdn": f"monitor-server-{i:02d}.prod.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=random.randint(1, 6))), + "tags": ["production", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # Cache (1) — stale + systems.append({ + "id": f"sys-{sid:03d}-cache-prod-01", + "display_name": "cache-server-01.prod.example.com", + "fqdn": "cache-server-01.prod.example.com", + "rhel_version": "8.9", + "last_seen": _ts(timedelta(days=11)), + "tags": ["production", "cache-tier"], + "stale": True, + "satellite_managed": False, + }) + sid += 1 + + # --- Staging (15) ------------------------------------------------------ + + for i in range(1, 5): + rhel = "9.3" if i <= 2 else "9.2" + stale = i == 3 + systems.append({ + "id": f"sys-{sid:03d}-web-stg-{i:02d}", + "display_name": f"web-server-{i:02d}.staging.example.com", + "fqdn": f"web-server-{i:02d}.staging.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=12) if stale else timedelta(hours=random.randint(2, 20))), + "tags": ["staging", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 4): + systems.append({ + "id": f"sys-{sid:03d}-db-stg-{i:02d}", + "display_name": f"db-server-{i:02d}.staging.example.com", + "fqdn": f"db-server-{i:02d}.staging.example.com", + "rhel_version": "9.3" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(3, 18))), + "tags": ["staging", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 6): + systems.append({ + "id": f"sys-{sid:03d}-app-stg-{i:02d}", + "display_name": f"app-server-{i:02d}.staging.example.com", + "fqdn": f"app-server-{i:02d}.staging.example.com", + "rhel_version": "8.9" if i <= 3 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(4, 22))), + "tags": ["staging", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-lb-stg-{i:02d}", + "display_name": f"lb-server-{i:02d}.staging.example.com", + "fqdn": f"lb-server-{i:02d}.staging.example.com", + "rhel_version": "8.8", + "last_seen": _ts(timedelta(hours=random.randint(2, 16))), + "tags": ["staging", "loadbalancer"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-stg-01", + "display_name": "monitor-server-01.staging.example.com", + "fqdn": "monitor-server-01.staging.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(hours=8)), + "tags": ["staging", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Development (10) -------------------------------------------------- + + for i in range(1, 4): + rhel = "9.2" if i == 1 else "8.9" + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-dev-{i:02d}", + "display_name": f"web-server-{i:02d}.dev.example.com", + "fqdn": f"web-server-{i:02d}.dev.example.com", + "rhel_version": rhel, + "last_seen": _ts(timedelta(days=15) if stale else timedelta(hours=random.randint(5, 23))), + "tags": ["development", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-db-dev-{i:02d}", + "display_name": f"db-server-{i:02d}.dev.example.com", + "fqdn": f"db-server-{i:02d}.dev.example.com", + "rhel_version": "9.3" if i == 1 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(6, 20))), + "tags": ["development", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 5): + systems.append({ + "id": f"sys-{sid:03d}-app-dev-{i:02d}", + "display_name": f"app-server-{i:02d}.dev.example.com", + "fqdn": f"app-server-{i:02d}.dev.example.com", + "rhel_version": "8.9" if i <= 2 else "9.2", + "last_seen": _ts(timedelta(hours=random.randint(8, 22))), + "tags": ["development", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-mon-dev-01", + "display_name": "monitor-server-01.dev.example.com", + "fqdn": "monitor-server-01.dev.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=10)), + "tags": ["development", "monitoring"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- QA (5) ------------------------------------------------------------ + + for i in range(1, 3): + stale = i == 2 + systems.append({ + "id": f"sys-{sid:03d}-web-qa-{i:02d}", + "display_name": f"web-server-{i:02d}.qa.example.com", + "fqdn": f"web-server-{i:02d}.qa.example.com", + "rhel_version": "9.3", + "last_seen": _ts(timedelta(days=14) if stale else timedelta(hours=random.randint(4, 18))), + "tags": ["qa", "web-tier"], + "stale": stale, + "satellite_managed": False, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-db-qa-01", + "display_name": "db-server-01.qa.example.com", + "fqdn": "db-server-01.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=12)), + "tags": ["qa", "database-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + for i in range(1, 3): + systems.append({ + "id": f"sys-{sid:03d}-app-qa-{i:02d}", + "display_name": f"app-server-{i:02d}.qa.example.com", + "fqdn": f"app-server-{i:02d}.qa.example.com", + "rhel_version": "9.2", + "last_seen": _ts(timedelta(hours=random.randint(5, 19))), + "tags": ["qa", "app-tier"], + "stale": False, + "satellite_managed": False, + }) + sid += 1 + + # --- Legacy (3) — ambiguous tags, no explicit environment -------------- + + systems.append({ + "id": f"sys-{sid:03d}-legacy-payment-01", + "display_name": "legacy-payment-gw.example.com", + "fqdn": "legacy-payment-gw.example.com", + "rhel_version": "8.7", + "last_seen": _ts(timedelta(hours=3)), + "tags": ["legacy-system", "payment-gateway", "critical"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-reports-01", + "display_name": "reports-legacy.example.com", + "fqdn": "reports-legacy.example.com", + "rhel_version": "8.6", + "last_seen": _ts(timedelta(days=6)), + "tags": ["legacy-system", "reporting", "financial-data"], + "stale": False, + "satellite_managed": True, + }) + sid += 1 + + systems.append({ + "id": f"sys-{sid:03d}-legacy-archive-01", + "display_name": "archive-01.legacy.example.com", + "fqdn": "archive-01.legacy.example.com", + "rhel_version": "8.5", + "last_seen": _ts(timedelta(days=30)), + "tags": ["legacy-system", "archive", "read-only"], + "stale": True, + "satellite_managed": True, + }) + sid += 1 + + # Add system_profile to each host + for idx, s in enumerate(systems): + host_type = "app" # default + for ht in ["web", "db", "app", "lb", "mon", "cache"]: + if ht in s["id"]: + host_type = ht + break + s["system_profile"] = _system_profile_for_host( + host_type, s["rhel_version"], idx + 1 + ) + + return systems + + +MOCK_SYSTEMS = generate_mock_systems() + +# --------------------------------------------------------------------------- +# Mock CVE data +# --------------------------------------------------------------------------- + +MOCK_CVE_DATA = { + "CVE-2024-12345": { + "cve_id": "CVE-2024-12345", + "severity": "Critical", + "cvss_score": 9.8, + "description": "Remote code execution vulnerability in HTTP request processing", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-03-15", + "compliance_notes": ( + "PCI-DSS 6.2 requires critical vulnerabilities be patched within " + "30 days for systems handling cardholder data" + ), + "affected_systems": [ + {"system_id": "sys-001-web-prod-01", "display_name": "web-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-002-web-prod-02", "display_name": "web-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-003-web-prod-03", "display_name": "web-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-004-web-prod-04", "display_name": "web-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-031-web-stg-01", "display_name": "web-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-032-web-stg-02", "display_name": "web-server-02.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 6, + "total_remediated": 2, + "total_vulnerable": 4, + }, + "CVE-2024-54321": { + "cve_id": "CVE-2024-54321", + "severity": "Important", + "cvss_score": 7.5, + "description": "SQL injection vulnerability in database query parser", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-04-30", + "compliance_notes": ( + "PCI-DSS 6.2 requires high-risk vulnerabilities be patched within " + "90 days. Affects systems storing cardholder data." + ), + "affected_systems": [ + {"system_id": "sys-009-db-prod-01", "display_name": "db-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-010-db-prod-02", "display_name": "db-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-011-db-prod-03", "display_name": "db-server-03.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-012-db-prod-04", "display_name": "db-server-04.prod.example.com", "status": "Patched", "remediation_available": True}, + {"system_id": "sys-035-db-stg-01", "display_name": "db-server-01.staging.example.com", "status": "Vulnerable", "remediation_available": True}, + ], + "total_affected": 5, + "total_remediated": 2, + "total_vulnerable": 3, + }, + "CVE-2024-11111": { + "cve_id": "CVE-2024-11111", + "severity": "Moderate", + "cvss_score": 5.3, + "description": "Information disclosure in application logging", + "pci_impact": True, + "soc2_impact": True, + "hipaa_impact": True, + "compliance_deadline": "2024-06-30", + "compliance_notes": ( + "HIPAA requires remediation of vulnerabilities exposing PHI. " + "PCI-DSS allows longer timelines for moderate risks." + ), + "affected_systems": [ + # 6 vulnerable production app servers + {"system_id": f"sys-{15+i:03d}-app-prod-{i:02d}", "display_name": f"app-server-{i:02d}.prod.example.com", + "status": "Vulnerable", "remediation_available": True} + for i in range(1, 7) + ] + [ + # 2 affected-but-not-vulnerable production app servers + {"system_id": "sys-022-app-prod-07", "display_name": "app-server-07.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "SELinux policy prevents exploitation of logging vulnerability"}, + {"system_id": "sys-023-app-prod-08", "display_name": "app-server-08.prod.example.com", + "status": "Affected but not vulnerable", "remediation_available": True, + "mitigation_reason": "Application logging feature is disabled in configuration"}, + ] + [ + # 3 patched staging app servers + {"system_id": f"sys-{40+i:03d}-app-stg-{i:02d}", "display_name": f"app-server-{i:02d}.staging.example.com", + "status": "Patched", "remediation_available": True} + for i in range(1, 4) + ], + "total_affected": 11, + "total_remediated": 3, + "total_vulnerable": 6, + }, + "CVE-2024-98765": { + "cve_id": "CVE-2024-98765", + "severity": "Important", + "cvss_score": 8.1, + "description": "Denial of service vulnerability in load balancer traffic handling", + "pci_impact": False, + "soc2_impact": True, + "hipaa_impact": False, + "compliance_deadline": "2024-05-15", + "compliance_notes": ( + "SOC2 CC7.1 requires protection of system availability. " + "Critical infrastructure should be patched urgently." + ), + "affected_systems": [ + {"system_id": "sys-025-lb-prod-01", "display_name": "lb-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-026-lb-prod-02", "display_name": "lb-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-027-lb-prod-03", "display_name": "lb-server-03.prod.example.com", "status": "Vulnerable", "remediation_available": True}, + {"system_id": "sys-045-lb-stg-01", "display_name": "lb-server-01.staging.example.com", "status": "Patched", "remediation_available": True}, + ], + "total_affected": 4, + "total_remediated": 1, + "total_vulnerable": 3, + }, + "CVE-2024-22222": { + "cve_id": "CVE-2024-22222", + "severity": "Low", + "cvss_score": 3.1, + "description": "Minor information disclosure in monitoring agent error messages", + "pci_impact": False, + "soc2_impact": False, + "hipaa_impact": False, + "compliance_deadline": None, + "compliance_notes": "Low severity, no immediate compliance impact. Patch during regular maintenance window.", + "affected_systems": [ + {"system_id": "sys-028-mon-prod-01", "display_name": "monitor-server-01.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + {"system_id": "sys-029-mon-prod-02", "display_name": "monitor-server-02.prod.example.com", "status": "Vulnerable", "remediation_available": False}, + ], + "total_affected": 2, + "total_remediated": 0, + "total_vulnerable": 2, + }, +} + + +# --------------------------------------------------------------------------- +# MCP server +# --------------------------------------------------------------------------- + +mcp = FastMCP("lightspeed-mcp") + + +@mcp.tool +def get_host_details( + system_id: Optional[str] = None, + hostname_pattern: Optional[str] = None, + tags: Optional[list[str]] = None, + rhel_version_prefix: Optional[str] = None, +) -> dict: + """Retrieve registered system inventory from Red Hat Lightspeed. + + Returns all systems when called with no arguments. Supports filtering + by system ID, hostname pattern, tags, and RHEL version prefix. + + Args: + system_id: Return only the system matching this ID. + hostname_pattern: Filter by hostname (supports * wildcards). + tags: Filter to systems having at least one of these tags. + rhel_version_prefix: Filter by RHEL version prefix (e.g. "8" or "9.3"). + """ + filtered = list(MOCK_SYSTEMS) + + if system_id: + filtered = [s for s in filtered if s["id"] == system_id] + + if hostname_pattern: + pattern = hostname_pattern.replace("*", "") + filtered = [s for s in filtered if pattern in s["fqdn"]] + + if tags: + filtered = [ + s for s in filtered + if any(t in s.get("tags", []) for t in tags) + ] + + if rhel_version_prefix: + filtered = [ + s for s in filtered + if s["rhel_version"].startswith(rhel_version_prefix) + ] + + return { + "systems": filtered, + "total": len(MOCK_SYSTEMS), + "count": len(filtered), + } + + +@mcp.tool +def get_cve_systems(cve_id: str) -> dict: + """Find systems affected by a specific CVE. + + Returns affected systems with their vulnerability status + (Vulnerable, Patched, or Affected but not vulnerable) and + whether automated remediation is available. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id in MOCK_CVE_DATA: + return MOCK_CVE_DATA[cve_id] + + return { + "cve_id": cve_id, + "affected_systems": [], + "total_affected": 0, + "total_remediated": 0, + } + + +@mcp.tool +def get_cves() -> dict: + """List all known CVEs affecting the fleet. + + Returns summary information for every CVE including severity, + CVSS score, affected/vulnerable counts, and compliance impact. + """ + summaries = [] + for cve in MOCK_CVE_DATA.values(): + entry = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + "remediation_available": any( + s.get("remediation_available", False) + for s in cve["affected_systems"] + ), + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + } + if "total_vulnerable" in cve: + entry["total_vulnerable"] = cve["total_vulnerable"] + summaries.append(entry) + return {"cves": summaries, "total": len(summaries)} + + +@mcp.tool +def get_cve(cve_id: str) -> dict: + """Get detailed information about a specific CVE. + + Returns full CVE metadata including severity, CVSS score, description, + compliance impact, and deadline — but not the per-system breakdown. + Use get_cve_systems for that. + + Args: + cve_id: CVE identifier in CVE-YYYY-NNNNN format. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + result = { + "cve_id": cve["cve_id"], + "severity": cve["severity"], + "cvss_score": cve["cvss_score"], + "description": cve["description"], + "pci_impact": cve["pci_impact"], + "soc2_impact": cve["soc2_impact"], + "hipaa_impact": cve["hipaa_impact"], + "compliance_deadline": cve["compliance_deadline"], + "compliance_notes": cve["compliance_notes"], + "total_affected": cve["total_affected"], + "total_remediated": cve["total_remediated"], + } + if "total_vulnerable" in cve: + result["total_vulnerable"] = cve["total_vulnerable"] + return result + + +@mcp.tool +def create_vulnerability_playbook( + cve_id: str, + system_ids: Optional[list[str]] = None, +) -> dict: + """Generate an Ansible remediation playbook for a CVE. + + Creates a playbook targeting the specified systems (or all vulnerable + systems if none specified). Returns the playbook content and metadata. + + Args: + cve_id: CVE identifier to remediate. + system_ids: Specific system IDs to target. Omit for all vulnerable. + """ + if cve_id not in MOCK_CVE_DATA: + return {"error": f"CVE {cve_id} not found"} + + cve = MOCK_CVE_DATA[cve_id] + if not any(s.get("remediation_available") for s in cve["affected_systems"]): + return { + "error": "No automated remediation available for this CVE", + "cve_id": cve_id, + } + + targets = system_ids or [ + s["system_id"] + for s in cve["affected_systems"] + if s["status"] == "Vulnerable" + ] + + return { + "cve_id": cve_id, + "playbook_id": f"playbook-{cve_id.lower()}-mock", + "target_systems": targets, + "target_count": len(targets), + "status": "generated", + "playbook_content": ( + f"# Auto-generated remediation playbook for {cve_id}\n" + f"# Targets: {len(targets)} systems\n" + f"---\n" + f"- hosts: targeted_systems\n" + f" become: true\n" + f" tasks:\n" + f" - name: Apply patch for {cve_id}\n" + f" dnf:\n" + f" name: '*'\n" + f" state: latest\n" + f" security: true\n" + ), + } + + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-sre__system-context/instruction.md b/evaluation/without_skills/rh-sre__system-context/instruction.md new file mode 100644 index 00000000..95d0540e --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/instruction.md @@ -0,0 +1,16 @@ +# System Context Task + +You are a Red Hat SRE. Before rolling out a remediation for a critical vulnerability, you need to gather comprehensive context about the affected systems to make safe remediation decisions. + +## Scenario +A high-severity advisory has been identified that affects multiple systems in your fleet. Before applying any patches, you need to understand each affected system's role, current health, installed packages, running services, and any special constraints (maintenance windows, compliance requirements, dependencies). + +## Requirements +- Use MCP tools to query systems in the fleet and identify those affected by the advisory +- For each affected system, gather: system role and criticality, current health and uptime, installed package versions relevant to the advisory, running services that may be impacted, and any compliance or scheduling constraints +- Assess which systems can be patched immediately vs. which need coordination +- Identify dependencies between systems that affect remediation ordering + +Document your system context analysis and remediation readiness assessment in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-sre__system-context/solution/solve.sh b/evaluation/without_skills/rh-sre__system-context/solution/solve.sh new file mode 100644 index 00000000..94c4eb6d --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/solution/solve.sh @@ -0,0 +1,19 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# System Context Report + +## Affected Systems +| System | RHEL | Environment | Infrastructure | Tags | +|--------|------|-------------|----------------|------| +| web-01 | 9.3 | Production | bare_metal | pci-compliant | +| db-01 | 8.9 | Staging | virtualized | - | + +## Data Source +get_cve_systems + get_host_details with include_system_profile=true. system_profile: rhel_version, infrastructure_type, installed_packages. + +## Remediation Strategy (Decision Matrix) +- Deployment type: Batch (multiple systems) +- Infrastructure: Bare metal, virtualized +- Maintenance window: Required for production +- Kubernetes: Rolling update with pod eviction if K8s nodes +REPORT_EOF diff --git a/evaluation/without_skills/rh-sre__system-context/task.toml b/evaluation/without_skills/rh-sre__system-context/task.toml new file mode 100644 index 00000000..d060c445 --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-sre__system-context" +name = "rh-sre System Context Gathering Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-sre", "system-context", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-sre__system-context/tests/llm_judge.py b/evaluation/without_skills/rh-sre__system-context/tests/llm_judge.py new file mode 100644 index 00000000..c2970b3d --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "infrastructure_classification", "file": "/root/report.md", "question": "Does the report classify systems by infrastructure_type (bare_metal/virtualized/container) and infrastructure_vendor?", "reference": "A skilled report uses infrastructure classification fields. An unskilled report doesn't distinguish infrastructure types."}, + {"id": "kubernetes_safety_context", "file": "/root/report.md", "question": "Does the report consider Kubernetes context (PDBs, daemonsets) for safe remediation planning?", "reference": "A skilled report checks hasPdbs and daemonsets for safety. An unskilled report ignores K8s workload context."}, + {"id": "staged_rollout", "file": "/root/report.md", "question": "Does the report recommend staged rollout (staging first, then production batches) based on environment criticality?", "reference": "A skilled report follows staged rollout pattern. An unskilled report patches all systems simultaneously."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-sre__system-context/tests/test.sh b/evaluation/without_skills/rh-sre__system-context/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-sre__system-context/tests/test_outputs.py b/evaluation/without_skills/rh-sre__system-context/tests/test_outputs.py new file mode 100644 index 00000000..ff39869d --- /dev/null +++ b/evaluation/without_skills/rh-sre__system-context/tests/test_outputs.py @@ -0,0 +1,84 @@ +""" +Tests for rh-sre__system-context per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_topic(self): + content = read_report().lower() + assert any(t in content for t in ['system', 'context', 'environment']), ( + "report should mention key topic" + ) + + def test_report_has_structure(self): + content = read_report() + assert len(content) > 150, "report should have substantial content" + + +class TestSkillDependent: + def test_remediation_strategy_by_context(self): + """Skill: Determine strategy from context: batch vs rolling, maintenance window, pod eviction for K8s.""" + c = read_report().lower() + has_strategy = any(t in c for t in ["strategy", "approach", "rolling", "batch"]) + has_context = any(t in c for t in ["maintenance", "pod eviction", "kubernetes", "staging first"]) + assert has_strategy and has_context, ( + "should derive strategy from context (skill: Decision Matrix)" + ) + + def test_rhel_version_distribution(self): + """Skill: Report RHEL version distribution (playbook must support multiple versions).""" + c = read_report().lower() + assert any(t in c for t in ['rhel', 'version', 'distribution', 'el7', 'el8', 'el9']), ( + "Should report RHEL version distribution (skill: conditional dnf/yum)" + ) + + def test_environment_and_criticality(self): + """Skill: Classify by environment (prod/staging/dev) and criticality for rollout order.""" + c = read_report().lower() + has_env = any(t in c for t in ["staging", "development", "rollout_order", "rollout order"]) + has_crit = any(t in c for t in ["critical", "criticality", "priority", "high", "rollout"]) + assert has_env and has_crit, ( + "should classify by environment and criticality (skill: rollout_order)" + ) + + def test_infrastructure_classification(self): + """Skill: infrastructure_type (bare_metal/virtualized/container) and infrastructure_vendor (kvm) fields.""" + c = read_report().lower() + has_type = any(t in c for t in ["infrastructure_type", "infrastructure_vendor", "virtualized"]) + has_bare = "bare_metal" in c or "bare metal" in c + assert has_type or has_bare, ( + "should reference infrastructure classification (skill: bare_metal/virtualized/container)" + ) + + def test_kubernetes_context_fields(self): + """Skill: hasPdbs and daemonsets_present for safety planning in K8s context.""" + c = read_report().lower() + has_k8s = any(t in c for t in ["pdb", "daemonset"]) + has_safety = any(t in c for t in ["safety", "eviction"]) + assert has_k8s and has_safety, ( + "should reference PDB/daemonset for K8s safety (skill)" + ) + + def test_needs_restarting_check(self): + """Docs teach needs-restarting -r (exit code 0=no reboot, 1=reboot needed) + and -s for services needing restart. Without docs, agents skip this check.""" + c = read_report().lower() + assert any(t in c for t in [ + "needs-restarting", "needs_restarting", "reboot", "restart service", + ]), "should use needs-restarting for reboot/service restart assessment" diff --git a/evaluation/without_skills/rh-virt__vm-clone/environment/Dockerfile b/evaluation/without_skills/rh-virt__vm-clone/environment/Dockerfile new file mode 100644 index 00000000..a76f03e8 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-clone/environment/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-virt__vm-clone/environment/mcp-servers/mock-virt-mcp.py b/evaluation/without_skills/rh-virt__vm-clone/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..70ce07d7 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-clone/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1465 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), + ("prod-vms", {"env": "production"}), + ("test-env", {"env": "testing"}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── prod-vms (instruction-specific) ────────────────────────────────── + _vm("production-db", "prod-vms", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true"}, + 8, 16, "Running", True, 1), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-virt__vm-clone/instruction.md b/evaluation/without_skills/rh-virt__vm-clone/instruction.md new file mode 100644 index 00000000..922cf2fb --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-clone/instruction.md @@ -0,0 +1,13 @@ +# VM Cloning Task + +You are an OpenShift Virtualization administrator. The QA team needs an exact copy of the production database VM to test a schema migration. Plan the clone so it can run safely in the test environment without affecting production. + +## Requirements +- Inspect the source VM (`production-db` in `prod-vms`) to understand its current state, storage configuration, and whether it needs to be stopped for cloning +- Determine the right cloning approach: whether the storage backend supports efficient cloning, and whether the VM needs to be offline +- Plan the clone target (`test-db-clone` in `test-env`) ensuring it has no network or storage conflicts with the source +- Verify the clone will be fully independent: separate disks, no shared PVCs, and safe to modify without affecting production + +Document your cloning plan in `/root/report.md`. + +Use MCP tools to examine the cluster. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-virt__vm-clone/solution/solve.sh b/evaluation/without_skills/rh-virt__vm-clone/solution/solve.sh new file mode 100644 index 00000000..17534b7f --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-clone/solution/solve.sh @@ -0,0 +1,34 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Cloning Plan + +## Source: production-db (prod-vms) → Target: test-db-clone (test-env) + +### Cloning Strategy +Using **Clone Storage** (full copy) strategy. Three strategies available: +1. **Clone Storage** (selected) — Full copy of all DataVolumes/PVCs. Independent clone. +2. **Reference Existing** — Shared disk. NOT safe for database workloads. +3. **New Empty Storage** — Fresh disk. Loses data. + +Full copy ensures test-db-clone is completely independent from production-db. + +### Spec Modifications for Clone +- Set `runStrategy: Halted` (don't auto-start the clone) +- Regenerate `domain.firmware.uuid` and `domain.firmware.serial` to avoid conflicts +- Update metadata.name to `test-db-clone` +- Update metadata.namespace to `test-env` +- Update DataVolume names to avoid collision + +### Storage Cloning +- Discover source DataVolumes via label: `vm.kubevirt.io/name=production-db` +- CSI volume cloning support required on the StorageClass +- Create new DataVolume with `source.pvc` referencing the original +- **Reference Existing** = shared disk — data corruption risk if both VMs run +- Use `resources_create_or_update` to create cloned VM and DataVolume + +### Verification +- Check target name `test-db-clone` doesn't exist in `test-env` +- Verify CSI driver supports volume cloning +- Monitor DataVolume clone progress + +REPORT_EOF diff --git a/evaluation/without_skills/rh-virt__vm-clone/task.toml b/evaluation/without_skills/rh-virt__vm-clone/task.toml new file mode 100644 index 00000000..46cd3410 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-clone/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-clone" +name = "rh-virt VM Cloning Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-clone", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-virt__vm-clone/tests/llm_judge.py b/evaluation/without_skills/rh-virt__vm-clone/tests/llm_judge.py new file mode 100644 index 00000000..749897f2 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-clone/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "firmware_uuid_regeneration", "file": "/root/report.md", "question": "Does the report address firmware UUID/serial regeneration to avoid identity conflicts between source and clone?", "reference": "A skilled report regenerates domain.firmware.uuid and serial in the clone spec. An unskilled report clones without addressing firmware identity."}, + {"id": "storage_clone_strategy", "file": "/root/report.md", "question": "Does the report discuss DataVolume clone strategy using source.pvc and StorageClass considerations?", "reference": "A skilled report uses DataVolume with source.pvc and considers CSI clone support. An unskilled report copies data manually."}, + {"id": "halted_run_strategy", "file": "/root/report.md", "question": "Does the report set runStrategy: Halted for the cloned VM to start in Stopped state?", "reference": "A skilled report ensures the clone starts halted. An unskilled report starts the clone immediately, risking conflicts."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-virt__vm-clone/tests/test.sh b/evaluation/without_skills/rh-virt__vm-clone/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-clone/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-virt__vm-clone/tests/test_outputs.py b/evaluation/without_skills/rh-virt__vm-clone/tests/test_outputs.py new file mode 100644 index 00000000..1638de54 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-clone/tests/test_outputs.py @@ -0,0 +1,90 @@ +""" +Tests for rh-virt__vm-clone per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_source_and_target(self): + content = read_report().lower() + has_source = any(t in content for t in ["source", "original", "production"]) + has_target = any(t in content for t in ["clone", "target", "copy", "destination"]) + assert has_source and has_target, "report should identify both a source VM and a clone target" + + +class TestSkillDependent: + def test_storage_class_cloning(self): + """Skill: StorageClass/CSI for PVC cloning strategy.""" + c = read_report().lower() + assert any(t in c for t in ["storageclass", "storage class", "csi", "volume cloning", "pvc clone", "clone support"]), ( + "should mention StorageClass or CSI cloning for clone strategy" + ) + + def test_identity_conflict(self): + """Skill: hostname, cloud-init, SSH key, firmware UUID conflicts between source and clone.""" + c = read_report().lower() + assert any(t in c for t in ["hostname", "cloud-init", "cloud init", "ssh key", "firmware", "uuid", "mac address", "identity conflict"]), ( + "should address identity conflicts (hostname, cloud-init, UUID) between source and clone" + ) + + def test_cross_namespace_rbac(self): + """Skill: RBAC/permissions for cross-namespace cloning.""" + c = read_report().lower() + assert any(t in c for t in ["rbac", "permission", "cross-namespace", "cross namespace", "target namespace", "create virtualmachine"]), ( + "should address RBAC or permissions for cross-namespace cloning" + ) + + def test_data_volume_cloning(self): + """Skill: DataVolume with source PVC for clone provisioning.""" + c = read_report().lower() + assert any(t in c for t in ["datavolume", "data volume", "source.pvc", "source pvc", "pvc datasource", "clone storage"]), ( + "should discuss DataVolume or PVC cloning for clone storage" + ) + + def test_datavolume_progress(self): + """Skill: Monitor DataVolume phase (Pending/Succeeded) during clone.""" + c = read_report().lower() + assert any(t in c for t in ["datavolume", "phase", "pending", "succeeded", "cloning progress", "status.phase"]), ( + "should mention monitoring DataVolume phase during clone" + ) + + def test_firmware_uuid_regeneration(self): + """Skill teaches domain.firmware.uuid and domain.firmware.serial must be + regenerated in clone spec to avoid identity conflicts. Without skill, + agents clone without regenerating firmware identifiers.""" + c = read_report().lower() + assert "firmware" in c and ("uuid" in c or "serial" in c), ( + "should address firmware UUID/serial regeneration for clone" + ) + + def test_run_strategy_halted_for_clone(self): + """Skill teaches runStrategy: Halted ensures cloned VM starts in Stopped state. + Without skill, agents start clone immediately.""" + c = read_report().lower() + assert any(t in c for t in ["halted", "runstrategy", "run strategy"]) and ( + "clone" in c or "stop" in c + ), "should set runStrategy: Halted for cloned VM" + + def test_source_pvc_bound(self): + """Docs teach CSI clone prerequisite: source PVC must be in Bound state. + Without docs, agents attempt cloning from unbound PVCs.""" + c = read_report().lower() + assert any(t in c for t in [ + "bound", "pvc status", "source pvc", "prerequisite", + ]) and ("pvc" in c or "storage" in c), ( + "should verify source PVC is Bound before cloning" + ) diff --git a/evaluation/without_skills/rh-virt__vm-create/environment/Dockerfile b/evaluation/without_skills/rh-virt__vm-create/environment/Dockerfile new file mode 100644 index 00000000..f77e513d --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-create/environment/Dockerfile @@ -0,0 +1,63 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/.ai-index/semantic-index.json new file mode 100644 index 00000000..2512c2a3 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/.ai-index/semantic-index.json @@ -0,0 +1,148 @@ +{ + "index_version": "1.0.0", + "last_updated": "2026-02-17", + "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", + + "documents": [ + { + "file": "INDEX.md", + "title": "VM Troubleshooting Guide - Index", + "category": "navigation", + "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], + "use_cases": ["navigation", "tool-reference", "general-guidance"], + "token_estimate": 450, + "skills": ["all"] + }, + { + "file": "scheduling-errors.md", + "title": "VM Scheduling Errors", + "category": "scheduling", + "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], + "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], + "errors_covered": [ + "ErrorUnschedulable - Node Taints", + "ErrorUnschedulable - Insufficient Resources", + "ErrorUnschedulable - Node Selector Mismatch" + ], + "token_estimate": 820, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "storage-errors.md", + "title": "VM Storage Errors", + "category": "storage", + "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], + "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], + "errors_covered": [ + "ErrorDataVolumeNotReady - DataVolume Still Provisioning", + "ErrorDataVolumeNotReady - Storage Class Not Found", + "ErrorDataVolumeNotReady - Insufficient Storage Quota", + "ErrorPvcNotFound", + "Storage Deletion Failures", + "DataVolume Cloning Failures" + ], + "token_estimate": 1350, + "skills": ["vm-creator", "vm-delete", "vm-clone"] + }, + { + "file": "lifecycle-errors.md", + "title": "VM Lifecycle Errors", + "category": "lifecycle", + "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], + "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], + "errors_covered": [ + "VM Stuck in Terminating State", + "VM Won't Start (Non-Scheduling Issues)", + "VM Won't Stop" + ], + "token_estimate": 1150, + "skills": ["vm-delete", "vm-lifecycle-manager"] + }, + { + "file": "runtime-errors.md", + "title": "VM Runtime Errors", + "category": "runtime", + "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], + "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], + "errors_covered": [ + "CrashLoopBackOff" + ], + "token_estimate": 850, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "network-errors.md", + "title": "VM Network Errors", + "category": "networking", + "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], + "use_cases": ["vm-creation", "network-troubleshooting"], + "errors_covered": [ + "Network Attachment Failures" + ], + "token_estimate": 580, + "skills": ["vm-creator"] + } + ], + + "skill_to_docs_mapping": { + "vm-creator": { + "primary_docs": ["scheduling-errors.md", "storage-errors.md"], + "secondary_docs": ["runtime-errors.md", "network-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3650, + "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" + }, + "vm-delete": { + "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 2950, + "description": "VM deletion failures - stuck terminating, storage cleanup" + }, + "vm-clone": { + "primary_docs": ["storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 1800, + "description": "VM cloning failures - DataVolume cloning, storage provisioning" + }, + "vm-lifecycle-manager": { + "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], + "secondary_docs": ["runtime-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3270, + "description": "VM start/stop failures - lifecycle management, scheduling" + }, + "vm-inventory": { + "primary_docs": [], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 450, + "description": "General guidance for status interpretation" + } + }, + + "error_to_docs_mapping": { + "ErrorUnschedulable": "scheduling-errors.md", + "ErrorDataVolumeNotReady": "storage-errors.md", + "ErrorPvcNotFound": "storage-errors.md", + "Terminating": "lifecycle-errors.md", + "CrashLoopBackOff": "runtime-errors.md", + "NetworkAttachmentFailed": "network-errors.md" + }, + + "cross_references": { + "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], + "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], + "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], + "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], + "network-errors.md": ["scheduling-errors.md"] + }, + + "progressive_disclosure_strategy": { + "description": "Load only the minimum documentation needed for current task", + "pattern": "error_type → primary_doc → cross_references (if needed)", + "token_savings": "29% average reduction vs single-file (based on rh-sre data)", + "navigation_overhead_reduction": "85% vs single-file navigation" + } +} diff --git a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/INDEX.md b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/INDEX.md new file mode 100644 index 00000000..a2e76963 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/INDEX.md @@ -0,0 +1,332 @@ +--- +title: VM Troubleshooting Guide - Index +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 + - title: OpenShift Virtualization - Virtual Machine Status + url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html + date_accessed: 2026-02-06 + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] +semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] +use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] +last_updated: 2026-02-17 +--- + +# VM Troubleshooting Guide - Index + +## Overview + +This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: + +- **Creation**: Scheduling failures, storage provisioning issues +- **Lifecycle**: Start/stop failures, stuck states +- **Deletion**: Resources stuck in Terminating, storage cleanup failures +- **Cloning**: DataVolume cloning errors, cross-namespace issues +- **Networking**: Secondary network attachment failures +- **Runtime**: CrashLoopBackOff, guest OS boot failures + +This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. + +--- + +## 🗂️ Troubleshooting Categories + +The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: + +### 1. [Scheduling Errors](scheduling-errors.md) +**When to use**: VM fails to schedule on any node + +**Errors covered**: +- ErrorUnschedulable - Node Taints +- ErrorUnschedulable - Insufficient Resources +- ErrorUnschedulable - Node Selector Mismatch + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 2. [Storage Errors](storage-errors.md) +**When to use**: VM has storage provisioning, deletion, or cloning issues + +**Errors covered**: +- ErrorDataVolumeNotReady (all 3 subsections) +- ErrorPvcNotFound +- Storage Deletion Failures +- DataVolume Cloning Failures + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### 3. [Lifecycle Errors](lifecycle-errors.md) +**When to use**: VM has start/stop/termination issues + +**Errors covered**: +- VM Stuck in Terminating State +- VM Won't Start (Non-Scheduling Issues) +- VM Won't Stop + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### 4. [Runtime Errors](runtime-errors.md) +**When to use**: VM repeatedly crashes or fails at runtime + +**Errors covered**: +- CrashLoopBackOff + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 5. [Network Errors](network-errors.md) +**When to use**: VM has secondary network attachment failures + +**Errors covered**: +- Network Attachment Failures (Multus, SR-IOV) + +**Skills that use this**: vm-create + +--- + +## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last + +### Critical Principle: MCP-First Approach + +**All diagnostic procedures in this guide follow the MCP-first pattern**: + +``` +1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) +2. ⚠️ IF FAILS: Ask user permission to use CLI command +3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval +``` + +### Why MCP Tools First? + +- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources +- **Consistency**: Same interface across all operations +- **Better Error Handling**: MCP tools return structured errors +- **Audit Trail**: MCP tool usage is logged and trackable + +### Command Pattern Examples + +Throughout this guide, you'll see diagnostic steps formatted like this: + +**✅ CORRECT Pattern (MCP First)**: + +```markdown +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Extract**: `.metadata.finalizers` from returned JSON + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` +``` + +### Available MCP Tools + +The `openshift-virtualization` MCP server provides these tools: + +**Resource Operations**: +- `resources_get` - Get specific resource (replaces `oc get `) +- `resources_list` - List resources (replaces `oc get `) +- `resources_delete` - Delete resource (replaces `oc delete`) +- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) + +**Pod Operations**: +- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) +- `pods_get` - Get pod details (replaces `oc get pod`) +- `pods_log` - Get pod logs (replaces `oc logs`) +- `pods_exec` - Execute in pod (replaces `oc exec`) +- `pods_delete` - Delete pod (replaces `oc delete pod`) +- `pods_top` - Pod resource usage (replaces `oc top pods`) + +**Events & Monitoring**: +- `events_list` - List events (replaces `oc get events`) +- `nodes_top` - Node resource usage (replaces `oc top nodes`) +- `nodes_log` - Node logs (replaces `oc adm node-logs`) +- `nodes_stats_summary` - Detailed node stats + +**VM Operations** (KubeVirt toolset): +- `vm_create` - Create VMs +- `vm_lifecycle` - Start/stop/restart VMs + +**Namespaces**: +- `namespaces_list` - List namespaces (replaces `oc get namespaces`) +- `projects_list` - List OpenShift projects (replaces `oc get projects`) + +### When CLI Commands Are Required + +Some operations have **NO MCP equivalent** and require CLI: + +- `virtctl` commands (console, VNC access) +- `oc debug node` (node debugging) +- `oc auth can-i` (permission checks) +- `oc adm taint` (node taint management) +- Complex JSON patch operations + +For these, the guide will note: **"CLI Only - No MCP equivalent"** + +### Quick Reference: CLI → MCP Mapping + +| CLI Command | MCP Tool Equivalent | +|-------------|---------------------| +| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | +| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | +| `oc logs -n ` | `pods_log` with name="", namespace="" | +| `oc get events -n ` | `events_list` with namespace="" | +| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | + +**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. + +### How to Read Diagnostic Sections + +Each error section includes: +1. **Symptom** - What you observe +2. **Description** - What's happening +3. **Possible Causes** - Why it's happening +4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback +5. **Solutions** - **MCP tools first**, then CLI fallback +6. **Verification** - **MCP tools first**, then CLI fallback + +**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. + +--- + +## 🔍 Quick Navigation by Skill + +**vm-create**: +- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics +- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning +- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff +- [Network Errors](network-errors.md) - Network attachment failures + +**vm-delete**: +- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state +- [Storage Errors](storage-errors.md) - Storage deletion failures + +**vm-clone**: +- [Storage Errors](storage-errors.md) - DataVolume cloning failures + +**vm-lifecycle-manager**: +- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop +- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling + +**vm-inventory**: +- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed + +--- + +## 📊 Documentation Coverage & Maintenance + +### Current Coverage + +This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: + +- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) +- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures +- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop +- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) +- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems + +**Total errors documented**: 12 error types across 6 categories + +--- + +### Encountering Undocumented Errors + +#### For AI Agents (Claude Code) + +If you encounter an error **not documented** in the categories above: + +1. **Report to user** with all available details (error message, affected resources, namespace) +2. **Provide best-effort diagnostics** using MCP tools: + - `resources_get` to inspect resource status + - `pods_log` to check virt-launcher or compute container logs + - `events_list` to view Kubernetes events +3. **Suggest documentation update**: + ``` + ⚠️ This error is not yet documented in the troubleshooting guide. + + **Error**: + **Affected resource**: / + + I recommend adding this error to the troubleshooting guide: + - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file + - If it's a new error class → Create new category file in docs/troubleshooting/ + + Would you like me to help document this error for future reference? + ``` + +4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs + +--- + +#### For Human Contributors + +To document a new error: + +1. **Determine the category**: + - Scheduling issue → `scheduling-errors.md` + - Storage problem → `storage-errors.md` + - Lifecycle/start/stop → `lifecycle-errors.md` + - Runtime crash → `runtime-errors.md` + - Network issue → `network-errors.md` + - New error class → Create new category file (e.g., `security-errors.md`) + +2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): + - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback + - **Solutions**: MCP Tool approach → Implementation → CLI Fallback + - **Verification**: MCP Tool checks → Expected results → CLI Fallback + +3. **Update semantic index**: + - Edit `.ai-index/semantic-index.json` + - Add error to `error_to_docs_mapping` + - Update relevant skill's `primary_docs` or `secondary_docs` + - Increment token estimates if needed + +4. **Reference in skill files**: + - Update skill's Reference Documentation section + - Ensure skill's Document Consultation steps point to the right category + +5. **Update this INDEX.md**: + - Add error to the appropriate category section (lines 50-120) + - Update "Current Coverage" list above + +--- + +## 📚 Additional Resources + +- [rh-virt README](../../README.md) - MCP server setup and skill documentation +- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources +- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/lifecycle-errors.md new file mode 100644 index 00000000..6167fcc8 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/lifecycle-errors.md @@ -0,0 +1,869 @@ +--- +title: VM Lifecycle Errors +category: kubevirt +sources: + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 +tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] +semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] +use_cases: [vm-deletion, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Lifecycle Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM lifecycle issues including start/stop failures and stuck termination states. + +**When to use this document**: +- VM stuck in `Terminating` state +- VM won't start (runStrategy is Always but VM never reaches Running) +- VM won't stop (runStrategy is Halted but VM never reaches Stopped) + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### VM Stuck in Terminating State + +**Symptom**: VM shows status `Terminating` but deletion never completes + +**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). + +**Possible Causes**: +- Finalizers blocking deletion +- PVC/DataVolume still attached and preventing cleanup +- VirtualMachineInstance (VMI) not terminating properly +- Custom controllers or operators blocking deletion +- Stuck virt-launcher pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check finalizers on the VM**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` + +**2. Check if VMI still exists**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name containing "virt-launcher-". + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check events for deletion issues**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results where `involvedObject.name` == "" and sort by timestamp. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' +``` + +**5. Check VM deletion timestamp**: + +Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. + +If present, VM is in deletion process. If null, VM is not being deleted. + +**Common Finalizer Patterns**: +- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer +- `foregroundDeletion` - Waits for dependent objects to be deleted +- Custom finalizers from operators + +**Solutions** (Use MCP Tools First): + +1. **Wait for dependent resources** (safest approach): + - Use `resources_list` to check if PVCs, DataVolumes are still deleting + - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) + +2. **Force delete VMI** (if VMI is stuck): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable - requires explicit user permission): + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +3. **Force delete virt-launcher pod** (if pod is stuck): + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + # Find the virt-launcher pod name first + oc get pods -n | grep virt-launcher- + + # Then delete it + oc delete pod virt-launcher--xxx -n --force --grace-period=0 + ``` + +4. **Remove finalizers** (⚠️ dangerous - use only as last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` + 2. Remove items from `.metadata.finalizers` array + 3. Update VM using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch not easily done via MCP): + ```bash + # This operation is complex for MCP - may need CLI + oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Check for protection labels** (vm-delete skill specific): + + Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. + + If value is "true", the vm-delete skill refuses deletion (this is expected behavior). + +**Verification**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error if deletion successful. + +**CLI Fallback**: +```bash +oc get vm -n +# Should return: Error from server (NotFound) +``` + +--- + + +--- + +### VM Won't Start (Non-Scheduling Issues) + +**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state + +**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. + +**Possible Causes**: +- Guest OS kernel panic or boot failure +- Cloud-init configuration errors +- Missing or corrupted disk image +- Insufficient memory for guest OS to boot +- QEMU/KVM virtualization errors +- VirtualMachineInstance (VMI) creation failures +- virt-launcher pod crashes + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VMI (VirtualMachineInstance) status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and overall VMI state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**2. Check VMI conditions for errors**: + +Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. View virt-launcher pod logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +First, get pod name from step 3, then: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +Look for QEMU errors, memory allocation failures, device errors. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**5. Check virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. + +Alternatively, use `pods_get` to get full pod details: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**6. Access VM console to see guest OS boot messages**: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +Look for kernel panic, initramfs errors, filesystem errors. + +**7. Check VM events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep -A 20 "Events:" +``` + +**Common Error Patterns**: + +1. **Guest OS Boot Failure**: + - Console shows kernel panic + - Guest hangs at GRUB or boot loader + - Cloud-init errors during first boot + +2. **Insufficient Memory**: + - Guest OS kills processes due to OOM + - VMI logs show memory allocation errors + +3. **QEMU Crashes**: + - virt-launcher logs show QEMU segfaults + - VMI repeatedly restarts + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for kernel panic, initramfs errors, filesystem errors. + +2. **Check virt-launcher pod logs for QEMU errors**: + + Use `pods_log` from diagnostic step 4 to view logs. + + Look for: + - "qemu-system-x86_64: ..." errors + - Memory allocation failures + - Device errors + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx + ``` + +3. **Increase memory if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check VMI memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +4. **Verify disk image integrity**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check PVC is bound: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + If using container disk, verify image exists and is accessible (check VMI spec). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get pvc -n + ``` + +5. **Check cloud-init configuration** (if applicable): + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. + + Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq + ``` + +6. **Restart VMI** (soft reset): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete VMI (VM controller will recreate it): + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for new VMI to start (use `resources_get` to check status). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n -w + ``` + +7. **Check virtualization extensions** (KVM): + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + + # In debug shell: + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should eventually return `Running`). + +Check VMI is running: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should show `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should eventually return: Running + +oc get vmi -n +# Should show: Running +``` + +--- + +### VM Won't Stop + +**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state + +**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. + +**Possible Causes**: +- Guest OS not responding to ACPI shutdown signal +- virt-launcher pod stuck and not terminating +- VirtualMachineInstance (VMI) deletion blocked by finalizers +- Guest shutdown scripts hanging +- Filesystem sync issues in guest OS + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (might show `Stopping`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**2. Check VMI status and deletion timestamp**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check VMI events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vmi -n | grep -A 10 "Events:" +``` + +**5. Check VMI finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.metadata.finalizers}' +``` + +**6. Check if guest is responsive** (if VMI still exists): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Patterns**: +- VMI shows `deletionTimestamp` but never actually deletes +- virt-launcher pod in `Terminating` state +- VM runStrategy is `Halted` but printableStatus shows `Stopping` + +**Solutions** (Use MCP Tools First): + +1. **Wait for graceful shutdown** (default: 30 seconds): + + Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. + +2. **Force stop by deleting VMI**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + This is the standard way to force-stop a VM. + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n + # Should return: Error from server (NotFound) + ``` + +3. **Force delete VMI with grace period** (if VMI won't delete): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +4. **Force delete virt-launcher pod**: + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find the pod using `pods_list_in_namespace` from diagnostic step 3. + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + ⚠️ **Note**: For force deletion with grace period, use CLI fallback. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') + oc delete pod $POD_NAME -n --force --grace-period=0 + ``` + +5. **Remove VMI finalizers** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VMI using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update VMI using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Patch VM runStrategy directly** (ensure consistency): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` (diagnostic step 1) + 2. Set `.spec.runStrategy` to `"Halted"` + 3. Update VM using `resources_create_or_update` with modified JSON + + **CLI Fallback** (merge patch easier via CLI): + Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Stopped` or `Halted`). + +Verify VMI is gone: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Verify virt-launcher pod is gone: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for pods matching `virt-launcher-`. Should return no results. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Stopped or Halted + +oc get vmi -n +# Should return: Error from server (NotFound) + +oc get pods -n | grep virt-launcher- +# Should return: No resources found +``` + +**Prevention**: +- Ensure guest OS has ACPI support enabled +- Use proper shutdown commands in guest OS +- Avoid forceful stops unless necessary (can corrupt guest filesystem) + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/network-errors.md b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/network-errors.md new file mode 100644 index 00000000..82fa313c --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/network-errors.md @@ -0,0 +1,429 @@ +--- +title: VM Network Errors +category: kubevirt +sources: + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] +semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] +use_cases: [vm-creation, network-troubleshooting] +related_docs: [INDEX.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Network Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. + +**When to use this document**: +- VM created successfully but secondary networks not attached +- NetworkAttachmentDefinition not found errors +- Multus CNI failures +- SR-IOV device attachment issues + +**Skills that use this**: vm-create + +--- + +### Network Attachment Failures + +**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working + +**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). + +**Possible Causes**: +- NetworkAttachmentDefinition doesn't exist in the namespace +- Multus CNI not installed or not configured on cluster +- Namespace mismatch (NAD in different namespace than VM) +- Interface name conflicts in VM spec +- Bridge/network configuration errors in NAD +- SR-IOV device not available (if using SR-IOV) + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List NetworkAttachmentDefinitions in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "" +} +``` + +Review `.items[].metadata.name` for available NADs. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -n +``` + +**2. List NetworkAttachmentDefinitions in all namespaces**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition" +} +``` + +Omit `namespace` parameter to list across all namespaces. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -A +``` + +**3. Check specific NetworkAttachmentDefinition**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "", + "name": "" +} +``` + +Review `.spec.config` for CNI configuration. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definition -n -o yaml +``` + +**4. Check VM network configuration**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.networks` to see network references. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq +``` + +**5. Check VM domain interfaces**: + +Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq +``` + +**6. Check virt-launcher pod network annotations** (shows actual attachments): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**7. Check Multus is installed**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "openshift-multus" +} +``` + +Should show Multus CNI pods running. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n openshift-multus +``` + +**8. Check for errors in virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n +``` + +**Common Error Messages**: +- `"network-attachment-definition not found"` - NAD doesn't exist in namespace +- `"multus CNI not configured"` - Multus not installed or misconfigured +- `"interface name conflict"` - Duplicate interface names in VM spec +- `"failed to add network"` - CNI plugin error (check NAD config) +- `"no available devices"` - SR-IOV device not available (if using SR-IOV) + +**Solutions** (Use MCP Tools First): + +1. **Verify NetworkAttachmentDefinition exists in correct namespace**: + + Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. + + If NAD is in different namespace, copy it to VM namespace: + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get NAD from source namespace using `resources_get` + 2. Modify `.metadata.namespace` to target namespace + 3. Create NAD in target namespace using `resources_create_or_update` + + **CLI Fallback** (stream processing easier via CLI): + Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" + ```bash + oc get network-attachment-definition -n -o yaml | \ + sed "s/namespace: /namespace: /" | \ + oc apply -f - + ``` + +2. **Create missing NetworkAttachmentDefinition**: + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + Example: Linux bridge network + + **Parameters**: + ```json + { + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": { + "name": "vlan100", + "namespace": "" + }, + "spec": { + "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" + ```bash + cat < + spec: + config: '{ + "cniVersion": "0.3.1", + "type": "bridge", + "bridge": "br1", + "vlan": 100, + "ipam": { + "type": "host-local", + "subnet": "192.168.100.0/24" + } + }' + EOF + ``` + +3. **Check Multus CNI installation**: + + Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. + + To check cluster network operator: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterOperator" + } + ``` + + Filter for `network` operator. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n openshift-multus + oc get clusteroperators network + ``` + +4. **Fix interface name conflicts** (if VM has duplicate names): + + Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. + + Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. + + **CLI Fallback** (interactive edit easier via CLI): + Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' + oc edit vm -n + ``` + +5. **Validate NAD configuration syntax**: + + Use `resources_get` from diagnostic step 3, extract `.spec.config`. + + Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get network-attachment-definition -n -o jsonpath='{.spec.config}' + ``` + +6. **Check SR-IOV device availability** (if using SR-IOV networks): + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + List SR-IOV network node policies: + + **Parameters**: + ```json + { + "apiVersion": "sriovnetwork.openshift.io/v1", + "kind": "SriovNetworkNodePolicy", + "namespace": "openshift-sriov-network-operator" + } + ``` + + Check SR-IOV device plugin pods: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "openshift-sriov-network-operator" + } + ``` + + Filter for pods with "device-plugin" in name. + + Check available SR-IOV devices on node: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "Node", + "name": "" + } + ``` + + Review `.status.allocatable` for SR-IOV resources. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get sriovnetworknodepolicy -n openshift-sriov-network-operator + oc get pods -n openshift-sriov-network-operator | grep device-plugin + oc describe node | grep -A 10 "Allocatable:" + ``` + +7. **Recreate VM with corrected network configuration** (if needed): + + If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +After remediation, check virt-launcher pod network status: + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +Should show all attached networks with status. Example output: +```json +[ + { + "name": "openshift-sdn", + "interface": "eth0", + "ips": ["10.128.2.10"], + "default": true + }, + { + "name": "vlan100", + "interface": "net1", + "ips": ["192.168.100.5"] + } +] +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**Check from inside VM** (via console): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# In guest OS: +ip addr show +# Should show all network interfaces (eth0, net1, etc.) +``` + +**Common Network Types**: +- **Linux Bridge**: Layer 2 bridge for VLAN networks +- **SR-IOV**: High-performance direct device assignment +- **macvlan**: MAC-based VLAN for container networks +- **OVN-Kubernetes**: OpenShift native overlay network + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/runtime-errors.md b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/runtime-errors.md new file mode 100644 index 00000000..804d3cbc --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/runtime-errors.md @@ -0,0 +1,616 @@ +--- +title: VM Runtime Errors +category: kubevirt +sources: + - title: Kubernetes Pod Lifecycle + url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ + date_accessed: 2026-02-17 +tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] +semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] +use_cases: [vm-creation, vm-lifecycle, diagnostics] +related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Runtime Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. + +**When to use this document**: +- VM or virt-launcher pod shows `CrashLoopBackOff` status +- virt-launcher pod repeatedly restarting +- Guest OS kernel panics on boot + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### CrashLoopBackOff + +**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting + +**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. + +**Possible Causes**: +- Guest OS kernel panic on boot +- Insufficient resources (memory/CPU) for guest OS +- Corrupted disk image or filesystem +- QEMU/libvirt crashes due to configuration errors +- Missing or incompatible device drivers in guest +- Resource limits too low for virt-launcher pod +- Virtualization features (KVM) not available on node + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check virt-launcher pod restart count**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# Look at RESTARTS column +``` + +**2. View recent crash logs** (previous container instance): + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "previous": true, + "tail": 100 +} +``` + +Look for QEMU errors, kernel panics, or segfaults. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n virt-launcher--xxx --previous +``` + +**3. Check current virt-launcher logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. + +**CLI Fallback** (if MCP unavailable or all containers needed): +```bash +oc logs -n virt-launcher--xxx --all-containers +``` + +**4. Check VMI conditions for crash details**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Extract `.status.conditions` for crash details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**5. Check pod events for crash reasons**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" +``` + +**6. Check pod resource limits**: + +**MCP Tool**: `pods_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +Extract `.spec.containers[0].resources` for resource limits. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq +``` + +**7. Check node kubelet logs for OOM kills**: + +⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +oc adm node-logs -u kubelet | grep -i oom +``` + +**8. Access guest console** (if VM briefly starts): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Crash Patterns**: + +1. **Guest Kernel Panic**: + - Console logs show kernel panic messages + - Guest crashes immediately after boot + - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" + +2. **OOM (Out of Memory)**: + - Pod killed with reason: `OOMKilled` + - Guest runs out of memory during boot or operation + - virt-launcher logs show memory allocation failures + +3. **QEMU Crash**: + - virt-launcher logs show QEMU segmentation fault + - Symptoms: "qemu-system-x86_64: terminated by signal" + - Configuration incompatibility or QEMU bug + +4. **Disk Image Corruption**: + - Guest cannot boot from disk + - Filesystem errors in guest console + - DataVolume import failed + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for kernel panic or boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for: + - Kernel panic messages + - Initramfs errors + - Filesystem mounting failures + - Missing device errors + +2. **Review virt-launcher crash logs**: + + Use `pods_log` with `previous: true` from diagnostic step 2. + + Look for: + - QEMU command line errors + - Device initialization failures + - Memory allocation errors + - Signal termination (SIGSEGV, SIGABRT) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx --previous + ``` + +3. **Check for OOM (Out of Memory) kills**: + + **MCP Tool**: `pods_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + Extract `.status.containerStatuses[0].lastState.terminated.reason`. + + If returns `"OOMKilled"`: + - Option 1: Increase virt-launcher memory limits + - Option 2: Decrease guest memory allocation + - Option 3: Use smaller instance type + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' + ``` + +4. **Increase resources if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check current memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +5. **Verify disk image integrity**: + + **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters for list**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" + } + ``` + + **Parameters for specific DV**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check `.status.phase` (should be `Succeeded`). + + If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get datavolume -n -o jsonpath='{.status.phase}' + ``` + +6. **Check virtualization (KVM) availability**: + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +7. **Simplify VM configuration** (eliminate variables): + + Try creating minimal VM using vm-create skill with: + - Small instance type + - No secondary networks + - Simple container disk (e.g., Fedora) + - No cloud-init + + If minimal VM works, add features back one by one. + +8. **Recreate VM with different workload** (test disk image): + + If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) + +After remediation, check pod restart count stops increasing: + +**Parameters for pods**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). + +Check VM reaches Running state: + +**Parameters for VM**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# RESTARTS should stabilize + +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Running +``` + +Verify guest is responsive: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# Should show login prompt or OS console +``` + +**Advanced Debugging**: + +**MCP Tool**: `pods_exec` (from openshift-virtualization) + +Check libvirt domain XML: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["virsh", "dumpxml", "1"] +} +``` + +Check QEMU process: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["ps", "aux"] +} +``` + +Filter output for "qemu" process. + +**CLI Fallback** (if MCP unavailable): +```bash +oc exec -n virt-launcher--xxx -- virsh dumpxml 1 +oc exec -n virt-launcher--xxx -- ps aux | grep qemu +``` + +**Prevention**: +- Start with minimal VM configuration and add complexity gradually +- Use recommended instance types for your workload +- Test disk images before deploying to production +- Ensure nodes have adequate resources and KVM support +- Monitor resource usage and set appropriate limits + +--- + +## Workaround Patterns for MCP Tool Limitations + +### General Pattern: Diagnose → Propose → Confirm → Execute + +When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): + +**1. Diagnose** the root cause using Kubernetes commands +```bash +oc describe vm -n +oc get events -n --field-selector involvedObject.name= +``` + +**2. Propose** a manual workaround to the user with clear explanation +- Explain why the issue occurred +- Show what will be changed +- List alternative options + +**3. Confirm** with user before executing (Human-in-the-Loop) +- Wait for explicit "yes" or "apply workaround" +- Never auto-execute modifications + +**4. Execute** the workaround using oc/kubectl +```bash +oc patch vm -n --type=merge -p '...' +``` + +**5. Verify** the fix was successful +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**6. Document** the limitation and suggest filing enhancement request +- Note this is temporary until MCP tool is enhanced +- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues + +### Example: Adding Tolerations Workaround + +**Diagnostic Output**: +```markdown +## ⚠️ VM Scheduling Issue Detected + +**Root Cause**: Node taints prevent VM scheduling + +**Details**: +- Found 3 nodes with taint: `virtualization=true:NoSchedule` +- VM spec does not include matching tolerations +- This prevents VM scheduling on virtualization-dedicated nodes +``` + +**Proposed Workaround**: +```bash +oc patch vm web-server -n vms --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' +``` + +**User Confirmation Required**: +``` +How would you like to proceed? +- "apply workaround" - I'll patch the VM with tolerations +- "manual" - I'll provide instructions for you to apply manually +- "cancel" - Delete the VM and abort creation +``` + +**After Execution**: +```markdown +## ✓ Workaround Applied Successfully + +**Action**: Added tolerations for taint `virtualization=true:NoSchedule` +**New Status**: Stopped (VM can now be scheduled) + +**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. +Future VMs in this cluster will need the same fix until the tool is enhanced. +``` + +--- + +## VM Status Reference + +### Status Values + +| Status | Meaning | Action Required | +|--------|---------|-----------------| +| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | +| `Running` | VM is running | Normal | +| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | +| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | +| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | +| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | +| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | +| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | +| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | +| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | + +### Checking VM Status + +```bash +# Get printable status +oc get vm -n -o jsonpath='{.status.printableStatus}' + +# Get detailed status and conditions +oc get vm -n -o jsonpath='{.status}' | jq + +# Watch status changes in real-time +oc get vm -n -w +``` + +--- + +## Best Practices for Agents + +When implementing diagnostic workflows: + +1. **Always verify VM status** after creation (wait 5-10 seconds first) +2. **Consult this document** when encountering error status values +3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) +4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) +5. **Respect human-in-the-loop** for all VM modifications +6. **Document temporary workarounds** and their limitations clearly +7. **Suggest filing issues** for missing MCP tool features + +### Document Consultation Pattern + +```markdown +**Document Consultation** (REQUIRED): +1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes +2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." +``` + +--- + +## Known MCP Tool Limitations + +### vm_create tool + +**Currently Supported**: +- ✓ Namespace, name (required) +- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) +- ✓ Size hints (small, medium, large) +- ✓ Storage size +- ✓ Autostart flag +- ✓ Networks (Multus NetworkAttachmentDefinitions) +- ✓ Performance family (u1, o1, c1, m1) +- ✓ Instance type, preference + +**Not Currently Supported** (requires workarounds): +- ✗ Tolerations (for node taints) +- ✗ Node selectors +- ✗ Affinity/anti-affinity rules +- ✗ Resource requests/limits (beyond instance type) +- ✗ Custom labels/annotations +- ✗ SSH keys injection +- ✗ Cloud-init user data + +**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. + +**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues + +--- + +## Additional Resources + +- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) +- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) +- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) +- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) + +--- + +[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/scheduling-errors.md new file mode 100644 index 00000000..e47c8c4f --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/scheduling-errors.md @@ -0,0 +1,417 @@ +--- +title: VM Scheduling Errors +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 +tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] +semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] +use_cases: [vm-creation, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Scheduling Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**When to use this document**: +- VM shows status `ErrorUnschedulable` after creation or start attempt +- VM events mention scheduling failures, taints, resources, or node selectors + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +## ErrorUnschedulable + +**Symptom**: VM shows status `ErrorUnschedulable` after creation + +**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**Possible Causes**: + +### 1. Node Taints (Most Common) + +Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM events for scheduling failures**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `involvedObject.name` == "" and look for messages like: +- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= +``` + +**2. Check node taints in the cluster**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' +``` + +**Common Taint Patterns**: +- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule +- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes +- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads + +**Solution - Add Tolerations to VM**: + +The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. + +**Workaround (post-creation using MCP Tools)**: + +**Step 1**: Get current VM spec + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Step 2**: Modify the returned JSON to add tolerations + +Add to `.spec.template.spec.tolerations`: +```json +{ + "tolerations": [ + { + "key": "virtualization", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule" + } + ] +} +``` + +**Step 3**: Update VM with modified spec + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +```json +{ + "resource": "" +} +``` + +Pass the complete modified VM resource as YAML or JSON string. + +**Step 4**: Verify tolerations were added + +Use `resources_get` again and check `.spec.template.spec.tolerations` in response. + +**Step 5**: Check if VM status improved + +Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. + +**CLI Fallback** (if MCP patch is too complex): +```bash +# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' + +# Verify tolerations +oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq + +# Check status +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**Example - Multiple Tolerations**: +```bash +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: "dedicated" + operator: "Equal" + value: "virt-workloads" + effect: "NoSchedule" +' +``` + +**Toleration Operators**: +- `Equal` - Key and value must match exactly +- `Exists` - Only key must exist (ignores value) + +**Toleration Effects**: +- `NoSchedule` - Don't schedule new pods (existing pods continue) +- `PreferNoSchedule` - Avoid scheduling if possible +- `NoExecute` - Don't schedule AND evict existing pods + +**Alternative Solutions**: +1. **Remove node taints** (if you have cluster-admin access): + ```bash + oc adm taint nodes virtualization=true:NoSchedule- + ``` + +2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits + +3. **File enhancement request** - Request tolerations support in openshift-mcp-server: + https://github.com/openshift/openshift-mcp-server/issues + +--- + +### 2. Insufficient Resources + +Not enough CPU, memory, or storage available on any node. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM resource requests**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' +``` + +**2. Check node resource availability**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. + +Alternatively, use `nodes_top` MCP tool for current resource usage. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe nodes | grep -A 5 "Allocated resources" +``` + +**3. Look for VM events mentioning "Insufficient"**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep "Insufficient" +``` + +**Example Event**: +``` +0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. +``` + +**Solutions** (Use MCP Tools First): + +1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) +2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill +3. **Delete unused VMs** - Use vm-delete skill to free up resources +4. **Check resource quotas**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters for quota**: + ```json + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" + } + ``` + + **Parameters for limit range**: + ```json + { + "apiVersion": "v1", + "kind": "LimitRange", + "namespace": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + oc describe limitrange -n + ``` + +--- + +### 3. Node Selector Mismatch + +VM requires specific node labels that don't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM node selector requirements**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.nodeSelector` to see required node labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' +``` + +**2. List available node labels**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.metadata.labels` for available labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes --show-labels +``` + +**3. Check if any nodes match the selector**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node", + "labelSelector": "=" +} +``` + +Should return at least one node with matching labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -l = +``` + +**Solutions** (Use MCP Tools First): + +**Option 1: Remove node selector from VM** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get current VM using `resources_get` (diagnostic step 1) +2. Remove `.spec.template.spec.nodeSelector` field +3. Update VM using `resources_create_or_update` with modified JSON + +**CLI Fallback** (JSON patch easier via CLI): +Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" +```bash +oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' +``` + +**Option 2: Add label to nodes** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get node using `resources_get` +2. Add label to `.metadata.labels` +3. Update node using `resources_create_or_update` + +⚠️ **Note**: Node labeling typically requires cluster admin privileges. + +**CLI Fallback** (simpler via CLI): +Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" +```bash +oc label node = +``` + +--- + +[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/storage-errors.md b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/storage-errors.md new file mode 100644 index 00000000..16ba584d --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/storage-errors.md @@ -0,0 +1,1011 @@ +--- +title: VM Storage Errors +category: kubevirt +sources: + - title: KubeVirt CDI - DataVolumes + url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ + date_accessed: 2026-02-17 + - title: Kubernetes Persistent Volumes + url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ + date_accessed: 2026-02-17 +tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] +semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] +use_cases: [vm-creation, vm-deletion, vm-cloning] +related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] +last_updated: 2026-02-17 +--- + +# VM Storage Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. + +**When to use this document**: +- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` +- Storage deletion fails after VM deletion +- DataVolume cloning operations fail +- PVC provisioning issues + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### ErrorDataVolumeNotReady + +**Symptom**: VM shows status `ErrorDataVolumeNotReady` + +**Description**: The DataVolume (persistent storage) backing the VM is not ready. + +**Possible Causes**: + +#### 1. DataVolume Still Provisioning + +Storage provisioning takes time, especially for large disks or when importing images. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Get detailed DataVolume information**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` and `.status.conditions` for provisioning details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o json +``` + +**3. Check PVC (PersistentVolumeClaim) bound status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Check `.status.phase` for each PVC (should be `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. + +#### 2. Storage Class Not Found + +The requested storage class doesn't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List available storage classes**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" +} +``` + +Review the list of available storage classes (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass +``` + +**2. Check DataVolume's requested storage class**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.spec.pvc.storageClassName` in the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' +``` + +**Solution**: +1. Use a valid storage class from the cluster +2. Recreate VM with correct storage class parameter + +#### 3. Insufficient Storage Quota + +Namespace has insufficient storage quota to provision the PVC. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check resource quotas**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**2. Check storage usage**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase +``` + +**Solution**: +1. Request quota increase from cluster admin +2. Delete unused PVCs to free quota +3. Reduce VM storage size + +--- + +### ErrorPvcNotFound + +**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List PVCs in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review the list of available PVCs (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check VM's PVC references**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' +``` + +**Solution**: +- Wait for DataVolume to create the PVC +- Manually create missing PVC +- Fix VM spec to reference correct PVC name + +--- + + +--- + +### Storage Deletion Failures + +**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace + +**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. + +**Possible Causes**: +- PVC still bound to active PersistentVolume with `Retain` policy +- DataVolume still being referenced by another resource +- CDI (Containerized Data Importer) controller issues +- Storage class retention policy preventing deletion +- Finalizers on PVC/DataVolume blocking cleanup +- PVC still mounted by a pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check PVC status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check specific PVC phase**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Released` or `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o jsonpath='{.status.phase}' +``` + +**3. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**4. Check what's using the PVC**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' +``` + +**5. Check PVC finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. + +**6. Check DataVolume finalizers**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.metadata.finalizers}' +``` + +**7. Check PV reclaim policy**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume" +} +``` + +Filter results for PV where `.spec.claimRef.name` matches ``. + +To get specific PV policy: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume", + "name": "" +} +``` + +Check `.spec.persistentVolumeReclaimPolicy`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pv | grep +oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' +``` + +**Common Finalizer Patterns**: +- `kubernetes.io/pvc-protection` - Protects PVC while in use +- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer + +**Solutions** (Use MCP Tools First): + +1. **Delete DataVolume first, then PVC**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete DataVolume first (often blocks PVC deletion): + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Wait a few seconds, then delete PVC: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete datavolume -n + oc delete pvc -n + ``` + +2. **Check for pods still using PVC**: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "" + } + ``` + + Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. + + Then delete the pods using `pods_delete`: + + **Parameters**: + ```json + { + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' + oc delete pod -n + ``` + +3. **Force delete PVC** (if safe to do so): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete pvc -n --grace-period=0 --force + ``` + +4. **Remove finalizers from PVC** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PVC using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update PVC using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Remove finalizers from DataVolume** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current DataVolume using `resources_get` (diagnostic step 6) + 2. Remove items from `.metadata.finalizers` array + 3. Update DataVolume using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Change PV reclaim policy** (if PV has Retain policy): + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PV policy using `resources_get` (diagnostic step 7) + 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` + 3. Update PV using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" + ```bash + oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' + oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' + ``` + +**Storage Quota Check** (Use MCP Tools First): + +After deletion, verify storage quota is freed: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters for quota check**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.used` to verify storage quota is freed. + +**Parameters for PVC verification**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +oc get pvc -n +``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Confirm PVC is deleted: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Confirm DataVolume is deleted: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +# Should return: Error from server (NotFound) + +oc get datavolume -n +# Should return: Error from server (NotFound) +``` + +--- + +### DataVolume Cloning Failures + +**Symptom**: VM clone created successfully but DataVolume clone operation fails + +**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. + +**Possible Causes**: +- CSI driver doesn't support volume cloning +- Source PVC storage class incompatible with cloning +- Cross-namespace cloning not permitted by storage backend +- Insufficient storage quota in target namespace +- Source PVC not in `Bound` state +- Storage class doesn't have volume cloning enabled +- CDI (Containerized Data Importer) controller issues + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume clone status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and `.metadata.name`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Check DataVolume events for errors**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe datavolume -n +``` + +**3. Check DataVolume phase**: + +Use `resources_get` from step 1, extract `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +``` + +**4. Check if storage class supports cloning**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "name": "" +} +``` + +Review the full YAML output for cloning-related configurations. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass -o yaml | grep -A 5 -i clone +``` + +**5. Check CSI driver capabilities**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "CSIDriver" +} +``` + +Review `.items[].metadata.name` for available CSI drivers. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get csidriver +``` + +**6. Check source PVC status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Bound` for cloning to work). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**7. Check target namespace storage quota**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**8. Check CDI controller logs**: + +**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) + +First, list pods in openshift-cnv namespace: + +**Parameters for pods_list_in_namespace**: +```json +{ + "namespace": "openshift-cnv", + "labelSelector": "app.kubernetes.io/component=cdi-deployment" +} +``` + +Then get logs using `pods_log`: + +**Parameters**: +```json +{ + "namespace": "openshift-cnv", + "name": "", + "tail": 100 +} +``` + +**CLI Fallback** (if MCP unavailable or easier via CLI): +```bash +oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') +``` + +**Common Error Messages**: +- `"volume cloning is not supported"` - CSI driver lacks clone capability +- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage +- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace +- `"insufficient quota"` - Target namespace lacks storage quota +- `"source PVC not bound"` - Source PVC must be in Bound state for cloning +- `"StorageClass does not support cloning"` - Storage class configuration issue + +**Solutions** (Use MCP Tools First): + +1. **Check storage class clone support**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" + } + ``` + + For each storage class in `.items[]`, check: + - `.metadata.name` (storage class name) + - `.provisioner` (CSI driver) + + Storage classes using CSI drivers typically support cloning. Look for provisioners like: + - `csi.ovirt.org` (oVirt CSI) + - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) + - `ebs.csi.aws.com` (AWS EBS CSI) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner + ``` + +2. **Verify source PVC is bound**: + + Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pvc -n -o jsonpath='{.status.phase}' + ``` + +3. **Check target namespace quota**: + + Use `resources_list` from diagnostic step 7 to check quota. + + If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + ``` + +4. **Use snapshot-based cloning** (alternative method): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Step 1**: Create VolumeSnapshot of source PVC + + **Parameters**: + ```json + { + "apiVersion": "snapshot.storage.k8s.io/v1", + "kind": "VolumeSnapshot", + "metadata": { + "name": "-snapshot", + "namespace": "" + }, + "spec": { + "source": { + "persistentVolumeClaimName": "" + } + } + } + ``` + + **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) + + **Step 3**: Create new DataVolume from snapshot + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": "-rootdisk", + "namespace": "" + }, + "spec": { + "source": { + "snapshot": { + "name": "-snapshot", + "namespace": "" + } + }, + "storage": { + "resources": { + "requests": { + "storage": "50Gi" + } + }, + "storageClassName": "" + } + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" + ```bash + cat <-snapshot + namespace: + spec: + source: + persistentVolumeClaimName: + EOF + + oc get volumesnapshot -snapshot -n + + cat <-rootdisk + namespace: + spec: + source: + snapshot: + name: -snapshot + namespace: + storage: + resources: + requests: + storage: 50Gi + storageClassName: + EOF + ``` + +5. **Use "new empty storage" option** (vm-clone skill): + - If cloning isn't supported, create VM with empty storage + - Manually copy data if needed + +6. **Cross-namespace cloning workaround**: + - Some storage backends require snapshot for cross-namespace cloning + - Create snapshot in source namespace, restore in target namespace (see solution 4 above) + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Check DataVolume reached Succeeded phase: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should return `Succeeded`). + +Check PVC was created and bound: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "-rootdisk" +} +``` + +Check `.status.phase` (should return `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +# Should return: Succeeded + +oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' +# Should return: Bound +``` + +**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/without_skills/rh-virt__vm-create/environment/mcp-servers/mock-virt-mcp.py b/evaluation/without_skills/rh-virt__vm-create/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..7b17408d --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-create/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1518 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), + ("vm-testing", {"env": "testing"}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + +STORAGE_CLASSES = [ + { + "name": "ocs-storagecluster-ceph-rbd", + "provisioner": "openshift-storage.rbd.csi.ceph.com", + "reclaimPolicy": "Delete", + "volumeBindingMode": "Immediate", + "allowVolumeExpansion": True, + }, + { + "name": "ocs-storagecluster-cephfs", + "provisioner": "openshift-storage.cephfs.csi.ceph.com", + "reclaimPolicy": "Delete", + "volumeBindingMode": "Immediate", + "allowVolumeExpansion": False, + }, +] + +VOLUME_SNAPSHOT_CLASSES = [ + { + "name": "ocs-storagecluster-rbdplugin-snapclass", + "driver": "openshift-storage.rbd.csi.ceph.com", + "deletionPolicy": "Delete", + }, +] + + +def _build_storage_class(sc): + """Build a storage.k8s.io/v1 StorageClass resource.""" + res = { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "metadata": { + "name": sc["name"], + "uid": _uid(sc["name"]), + "creationTimestamp": CREATED, + }, + "provisioner": sc["provisioner"], + "reclaimPolicy": sc["reclaimPolicy"], + "volumeBindingMode": sc["volumeBindingMode"], + } + if sc.get("allowVolumeExpansion"): + res["allowVolumeExpansion"] = True + return res + + +def _build_volume_snapshot_class(vsc): + """Build a snapshot.storage.k8s.io/v1 VolumeSnapshotClass resource.""" + return { + "apiVersion": "snapshot.storage.k8s.io/v1", + "kind": "VolumeSnapshotClass", + "metadata": { + "name": vsc["name"], + "uid": _uid(vsc["name"]), + "creationTimestamp": CREATED, + }, + "driver": vsc["driver"], + "deletionPolicy": vsc["deletionPolicy"], + } + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-virt__vm-create/instruction.md b/evaluation/without_skills/rh-virt__vm-create/instruction.md new file mode 100644 index 00000000..f35ed63f --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-create/instruction.md @@ -0,0 +1,14 @@ +# VM Creation Task + +You are an OpenShift Virtualization administrator. The development team needs a new RHEL 9 VM for testing. Provision `test-vm` in the `vm-testing` namespace with appropriate resources. + +## Requirements +- Examine the cluster to determine available node capacity, storage classes, and existing VM templates +- Define the VM specification: 2 CPUs, 4Gi memory, 30Gi root disk, RHEL 9 operating system +- Choose the storage provisioning strategy (which storage class, access mode, volume mode) based on what the cluster offers +- Document what could go wrong during provisioning (e.g., insufficient capacity, storage class not available, image pull failure) and how to handle each case +- Provide the complete VM resource definition + +Document your provisioning plan and VM specification in `/root/report.md`. + +Use MCP tools to examine the cluster. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-virt__vm-create/solution/solve.sh b/evaluation/without_skills/rh-virt__vm-create/solution/solve.sh new file mode 100644 index 00000000..311af1b5 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-create/solution/solve.sh @@ -0,0 +1,71 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Creation Plan + +## Target: test-vm in vm-testing + +### VirtualMachine Specification + +```yaml +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + name: test-vm + namespace: vm-testing +spec: + runStrategy: Always + template: + spec: + domain: + cpu: + cores: 2 + resources: + requests: + memory: 4Gi + devices: + disks: + - name: rootdisk + disk: + bus: virtio + volumes: + - name: rootdisk + dataVolume: + name: test-vm-rootdisk + dataVolumeTemplates: + - metadata: + name: test-vm-rootdisk + spec: + source: + registry: + url: docker://registry.redhat.io/rhel9/rhel-guest-image:latest + storage: + resources: + requests: + storage: 30Gi +``` + +### Storage Configuration +- Using DataVolume with registry source for RHEL 9 guest image +- DataVolume automatically provisions PVC via CDI +- Default StorageClass used (annotated with storageclass.kubernetes.io/is-default-class) + +### VM Lifecycle +- `runStrategy: Always` ensures VM starts automatically and restarts on failure +- Alternative: `running: true` for simple start, but runStrategy provides more control +- Instance type/size: small (2 vCPU, 4Gi) for testing purposes + +### Default Credentials +- RHEL 9 guest image: requires cloud-init or SSH key for access + +### Prerequisite Checks +- Verify namespace vm-testing exists +- Check default StorageClass is configured (annotation storageclass.kubernetes.io/is-default-class) +- Verify KubeVirt operator is running +- Ensure sufficient node resources (2 CPU, 4Gi memory) + +### Error Handling (from vm-create skill) +- **ErrorUnschedulable**: Consult scheduling-errors.md; add tolerations via oc patch if node taints block scheduling +- **ErrorDataVolumeNotReady**: Storage provisioning; verify StorageClass, check CDI/DataVolume status +- Access VM: `virtctl console test-vm -n vm-testing` or VNC via OpenShift Console + +REPORT_EOF diff --git a/evaluation/without_skills/rh-virt__vm-create/task.toml b/evaluation/without_skills/rh-virt__vm-create/task.toml new file mode 100644 index 00000000..d6ab031e --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-create/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-create" +name = "rh-virt VM Creation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-create", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-virt__vm-create/tests/llm_judge.py b/evaluation/without_skills/rh-virt__vm-create/tests/llm_judge.py new file mode 100644 index 00000000..8fb930ee --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-create/tests/llm_judge.py @@ -0,0 +1,92 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "unschedulable_handling", "file": "/root/report.md", "question": "Does the report address ErrorUnschedulable and taint/toleration handling for VM placement?", "reference": "A skilled report handles scheduling errors with tolerations. An unskilled report doesn't address placement failures."}, + {"id": "datavolume_provisioning", "file": "/root/report.md", "question": "Does the report describe using DataVolume resources (with CDI) for VM disk provisioning, specifying a source (registry, blank, or PVC)?", "reference": "A skilled report uses DataVolume with a source specification for disk provisioning. An unskilled report creates PVCs manually without CDI integration."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-virt__vm-create/tests/test.sh b/evaluation/without_skills/rh-virt__vm-create/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-create/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-virt__vm-create/tests/test_outputs.py b/evaluation/without_skills/rh-virt__vm-create/tests/test_outputs.py new file mode 100644 index 00000000..5cf84d0d --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-create/tests/test_outputs.py @@ -0,0 +1,71 @@ +""" +Tests for rh-virt__vm-create per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_vm(self): + content = read_report().lower() + assert any(t in content for t in ["vm", "virtual machine", "virtualmachine"]), ( + "report should reference the target VM" + ) + + def test_mentions_namespace(self): + content = read_report().lower() + assert "namespace" in content, "report should mention the target namespace" + + +class TestSkillDependent: + def test_data_volume_provisioning(self): + """Skill: DataVolume for disk provisioning with image/blank source.""" + c = read_report().lower() + assert any(t in c for t in ["datavolume", "data volume", "cdi.kubevirt.io", "source.registry", "source.blank"]), ( + "should discuss DataVolume for disk provisioning" + ) + + def test_storage_class_provisioning(self): + """Skill: StorageClass for DataVolume/PVC provisioning.""" + c = read_report().lower() + assert any(t in c for t in ["storageclass", "storage class", "volumeBindingMode", "provisioner"]) and ( + "storage" in c or "pvc" in c or "datavolume" in c + ), ( + "should mention StorageClass for disk provisioning" + ) + + def test_instance_type_or_workload(self): + """Skill: Instance type (u1.medium) or workload (fedora, rhel) resolution.""" + c = read_report().lower() + assert any(t in c for t in ["instancetype", "instance type", "u1.", "u1.medium", "workload", "fedora", "rhel", "ubuntu", "centos"]), ( + "should reference instance types or workload/OS selection" + ) + + def test_unschedulable_toleration(self): + """Skill: ErrorUnschedulable and toleration workaround.""" + c = read_report().lower() + assert any(t in c for t in ["errorunschedulable", "unschedulable", "taint", "toleration", "scheduling"]) and ( + "taint" in c or "toleration" in c or "unschedulable" in c + ), ( + "should address ErrorUnschedulable and taint/toleration handling" + ) + + def test_yaml_or_manifest(self): + """Should include a YAML manifest or structured spec.""" + content = read_report() + assert "apiVersion" in content or "kind:" in content or "spec:" in content or "```yaml" in content or "```yml" in content, ( + "should include a YAML manifest or structured specification" + ) diff --git a/evaluation/without_skills/rh-virt__vm-delete/environment/Dockerfile b/evaluation/without_skills/rh-virt__vm-delete/environment/Dockerfile new file mode 100644 index 00000000..a76f03e8 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-delete/environment/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-virt__vm-delete/environment/mcp-servers/mock-virt-mcp.py b/evaluation/without_skills/rh-virt__vm-delete/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..2aaace7d --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-delete/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1464 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), + ("decommission", {"env": "decommission"}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── decommission (instruction-specific) ────────────────────────────── + _vm("legacy-app", "decommission", "hv-prod-dc1-01", "rhel-8.6", None, + {"app": "legacy-app", "criticality": "low", "legacy": "true"}, + 2, 4, "Running", True, 30), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-virt__vm-delete/instruction.md b/evaluation/without_skills/rh-virt__vm-delete/instruction.md new file mode 100644 index 00000000..5769196b --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-delete/instruction.md @@ -0,0 +1,12 @@ +# VM Deletion Task + +You are an OpenShift Virtualization administrator. Plan the safe deletion of VM `legacy-app` in namespace `decommission`. + +## Requirements +- Perform pre-deletion safety checks +- Define the deletion scope (VM only vs VM + storage) +- Include safeguards against accidental deletion + +Use MCP tools to examine the cluster. Document your methodology, findings, and deletion plan in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-virt__vm-delete/solution/solve.sh b/evaluation/without_skills/rh-virt__vm-delete/solution/solve.sh new file mode 100644 index 00000000..6d87b29d --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-delete/solution/solve.sh @@ -0,0 +1,31 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Deletion Plan + +## Target: legacy-app in decommission + +### Pre-Deletion Safety Checks +1. **Protection label**: Check `metadata.labels.protected` — if `"true"`, deletion is blocked. Remove with `oc label vm legacy-app -n decommission protected-` +2. **Running state**: If VM is running, stop it first via `vm_lifecycle` action=stop +3. **Storage discovery**: List DataVolumes with label `vm.kubevirt.io/name=legacy-app` + +### Deletion Scope Options +- **VM Only** — Keep associated storage (DataVolumes/PVCs) for data recovery +- **VM + Storage** (selected) — Full cleanup of VM and all associated DataVolumes/PVCs + +### Deletion Procedure +1. Verify VM exists and is stopped (use vm_lifecycle action=stop if running) +2. List all associated DataVolumes (apiVersion: cdi.kubevirt.io/v1beta1, labelSelector: vm.kubevirt.io/name=legacy-app) +3. Present deletion scope and storage list +4. **Typed confirmation required**: User must type exact VM name `legacy-app` to proceed +5. Delete VM via resources_delete +6. Delete associated DataVolumes and PVCs via resources_delete +7. Verify deletion completed (resource no longer exists) +8. If VM stuck Terminating: consult lifecycle-errors.md, check finalizers + +### Post-Deletion Verification +- Confirm VM resource is gone +- Confirm DataVolumes and PVCs are cleaned up +- Check for any orphaned resources (finalizers) + +REPORT_EOF diff --git a/evaluation/without_skills/rh-virt__vm-delete/task.toml b/evaluation/without_skills/rh-virt__vm-delete/task.toml new file mode 100644 index 00000000..063c79fd --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-delete/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-delete" +name = "rh-virt VM Deletion Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-delete", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-virt__vm-delete/tests/llm_judge.py b/evaluation/without_skills/rh-virt__vm-delete/tests/llm_judge.py new file mode 100644 index 00000000..e1bed079 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-delete/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "protected_label", "file": "/root/report.md", "question": "Does the report check for protected: true label that blocks deletion?", "reference": "A skilled report checks protection labels. An unskilled report attempts deletion without safety checks."}, + {"id": "storage_scope", "file": "/root/report.md", "question": "Does the report distinguish VM-only vs VM+storage deletion and warn about orphaned PVCs?", "reference": "A skilled report offers storage scope choice. An unskilled report deletes everything without distinction."}, + {"id": "typed_confirmation", "file": "/root/report.md", "question": "Does the report require typed VM name confirmation (exact, case-sensitive) before deletion?", "reference": "A skilled report requires exact name match confirmation. An unskilled report uses yes/no confirmation."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-virt__vm-delete/tests/test.sh b/evaluation/without_skills/rh-virt__vm-delete/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-delete/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-virt__vm-delete/tests/test_outputs.py b/evaluation/without_skills/rh-virt__vm-delete/tests/test_outputs.py new file mode 100644 index 00000000..a1c73806 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-delete/tests/test_outputs.py @@ -0,0 +1,82 @@ +""" +Tests for rh-virt__vm-delete per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_vm(self): + content = read_report().lower() + assert any(t in content for t in ["vm", "virtual machine", "virtualmachine"]), ( + "report should reference the target VM" + ) + + def test_mentions_deletion(self): + content = read_report().lower() + assert "delet" in content, "report should discuss deletion" + + +class TestSkillDependent: + def test_stop_before_delete(self): + """Skill: Must stop VM before deletion; vm_lifecycle stop.""" + c = read_report().lower() + assert any(t in c for t in ["stop before delet", "stop and delete", "vm_lifecycle", "halt", "must stop", "running"]) and ( + "stop" in c or "halt" in c + ), ( + "should require stopping VM before deletion" + ) + + def test_orphan_storage(self): + """Skill: VM-only vs VM+storage; orphan PVCs; delete DataVolume/PVC.""" + c = read_report().lower() + assert any(t in c for t in ["vm only", "vm+storage", "datavolume", "orphan", "preserve storage", "delete storage", "pvc"]) and ( + "storage" in c or "pvc" in c or "datavolume" in c + ), ( + "should address storage scope (VM-only vs VM+storage, orphan PVCs)" + ) + + def test_finalizer_handling(self): + """Skill: Finalizer blocking deletion; stuck Terminating.""" + c = read_report().lower() + assert any(t in c for t in ["finalizer", "terminating", "stuck", "resources_create_or_update", "remove finalizer"]), ( + "should address finalizer handling for stuck deletion" + ) + + def test_typed_confirmation(self): + """Skill: Typed VM name confirmation (exact match) before delete.""" + c = read_report().lower() + assert any(t in c for t in ["type", "typed", "exact name", "confirm", "to confirm"]) and ( + "name" in c or "vm" in c + ), ( + "should require typed VM name confirmation" + ) + + def test_protected_label(self): + """Skill: protected: true label blocks deletion.""" + c = read_report().lower() + assert any(t in c for t in ["protected", "protected label", "metadata.labels", "refuse delet"]), ( + "should address protected label blocking deletion" + ) + + def test_reclaim_policy_retain(self): + """Docs teach PV reclaim policy Retain blocks PVC deletion; must patch PV + to Delete first. Without docs, agents don't handle stuck PVC cleanup.""" + c = read_report().lower() + assert any(t in c for t in [ + "retain", "reclaim", "reclaimpolicy", "reclaim policy", + "patch pv", "delete policy", + ]), "should address PV reclaim policy Retain blocking cleanup" diff --git a/evaluation/without_skills/rh-virt__vm-inventory/environment/Dockerfile b/evaluation/without_skills/rh-virt__vm-inventory/environment/Dockerfile new file mode 100644 index 00000000..a76f03e8 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-inventory/environment/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-virt__vm-inventory/environment/mcp-servers/mock-virt-mcp.py b/evaluation/without_skills/rh-virt__vm-inventory/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..2e083d72 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-inventory/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1458 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-virt__vm-inventory/instruction.md b/evaluation/without_skills/rh-virt__vm-inventory/instruction.md new file mode 100644 index 00000000..28107e57 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-inventory/instruction.md @@ -0,0 +1,14 @@ +# VM Inventory Task + +You are an OpenShift Virtualization administrator. Your team needs a complete picture of every VM in the cluster for capacity planning and compliance reporting. + +## Requirements +- List every VM across all namespaces, grouped by namespace +- For each VM report: name, status (Running/Stopped/Paused), CPU and memory allocation, operating system, and IP address if running +- Identify any VMs with issues: stopped unexpectedly, guest agent not responding, degraded conditions, or running end-of-life operating systems +- Summarize totals: how many VMs per namespace, how many running vs stopped, total resource allocation +- Sort results by namespace, then by VM name + +Write the inventory report in `/root/report.md`. + +Use MCP tools to gather VM data. If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-virt__vm-inventory/solution/solve.sh b/evaluation/without_skills/rh-virt__vm-inventory/solution/solve.sh new file mode 100644 index 00000000..3473c6d5 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-inventory/solution/solve.sh @@ -0,0 +1,32 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Inventory Report + +## Cluster VM Summary + +| Namespace | VM Name | Status | CPU | Memory | Storage | OS | IP | Node | +|-----------|---------|--------|-----|--------|---------|----|----|------| +| prod-vms | production-db | Running | 4 vCPU, 16Gi | 100Gi | RHEL 9.3 | 10.128.2.15 | worker-01 | +| prod-vms | web-frontend | Running | 2 vCPU, 4Gi | 50Gi | Fedora 39 | 10.128.2.16 | worker-02 | +| dev-vms | dev-test | Stopped | 2 vCPU, 8Gi | 50Gi | Ubuntu 22.04 | — | — | + +### Status Summary +- Running: 2 +- Stopped: 1 +- Total: 3 + +### Data Sources +- VM status: `status.printableStatus` from VirtualMachine resource +- Resource details: Extracted from VirtualMachineInstance (VMI) when running via resources_list (apiVersion kubevirt.io/v1, allNamespaces=true for cluster-wide) +- CPU: `.spec.domain.cpu.sockets` (displayed as vCPU) +- Memory: `.spec.domain.memory.guest` +- Storage: `.status.volumeStatus[].persistentVolumeClaimInfo.capacity.storage` +- OS: `.status.guestOSInfo.prettyName` +- IP: `.status.interfaces[0].ipAddress` +- Node: `.status.nodeName` +- Conditions: Ready, AgentConnected, LiveMigratable + +### Sort Order +Sorted by: Namespace → Status (Running → Pending → Stopped → Failed) → VM Name + +REPORT_EOF diff --git a/evaluation/without_skills/rh-virt__vm-inventory/task.toml b/evaluation/without_skills/rh-virt__vm-inventory/task.toml new file mode 100644 index 00000000..6a756f27 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-inventory/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-inventory" +name = "rh-virt VM Inventory Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-inventory", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-virt__vm-inventory/tests/llm_judge.py b/evaluation/without_skills/rh-virt__vm-inventory/tests/llm_judge.py new file mode 100644 index 00000000..aabb1dab --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-inventory/tests/llm_judge.py @@ -0,0 +1,92 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "vmi_vs_vm_distinction", "file": "/root/report.md", "question": "Does the report distinguish between VirtualMachine (spec/desired state) and VirtualMachineInstance (runtime state) as separate resources to query?", "reference": "A skilled report queries both VM and VMI, understanding VM defines the spec while VMI reflects the running state. An unskilled report only queries VirtualMachine without VMI runtime data."}, + {"id": "status_ordering", "file": "/root/report.md", "question": "Does the report organize or sort VMs by operational status (e.g., Running first, then Pending, Stopped, Failed) rather than just listing alphabetically?", "reference": "A skilled report groups or sorts VMs by status priority. An unskilled report lists VMs in arbitrary order without status-based organization."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-virt__vm-inventory/tests/test.sh b/evaluation/without_skills/rh-virt__vm-inventory/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-inventory/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-virt__vm-inventory/tests/test_outputs.py b/evaluation/without_skills/rh-virt__vm-inventory/tests/test_outputs.py new file mode 100644 index 00000000..16ded70a --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-inventory/tests/test_outputs.py @@ -0,0 +1,67 @@ +""" +Tests for rh-virt__vm-inventory per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_has_structured_data(self): + content = read_report() + has_table = "|" in content and content.count("|") >= 4 + has_list = content.count("- ") >= 5 + assert has_table or has_list, "report should present VM inventory in a structured format (table or list)" + + def test_mentions_namespace(self): + content = read_report().lower() + assert "namespace" in content, "report should organize by namespace" + + +class TestSkillDependent: + def test_vmi_runtime_data(self): + """Skill: Query VirtualMachineInstance (VMI) for running VM runtime data.""" + c = read_report().lower() + assert any(t in c for t in ["virtualmachineinstance", "vmi", "virtual machine instance"]), ( + "should reference VMI for runtime data, not just VirtualMachine" + ) + + def test_resource_format(self): + """Skill: Resources as 'X vCPU, YGi' format, not instance type names like u1.medium.""" + c = read_report().lower() + assert any(t in c for t in ["vcpu", "vcpus"]) and any(t in c for t in ["gi", "gib"]), ( + "should use vCPU/Gi resource format, not instance type names" + ) + + def test_status_based_grouping(self): + """Skill: Sort by namespace, then status (Running > Pending > Stopped > Failed), then name.""" + c = read_report().lower() + status_terms = sum(1 for t in ["running", "stopped", "pending", "failed"] if t in c) + has_organization = any(t in c for t in [ + "group", "sort", "order", "organiz", "by namespace", + "by status", "running first", "namespace", + ]) + assert status_terms >= 2 and has_organization, ( + "should organize VMs with status awareness (Running/Stopped/etc) by namespace" + ) + + def test_conditions_awareness(self): + """Skill: KubeVirt-specific conditions — AgentConnected, LiveMigratable.""" + c = read_report().lower() + assert any(t in c for t in [ + "agentconnected", "agent connected", "agent_connected", + "livemigratable", "live migratable", "live_migratable", + "guest agent", + ]), "should mention KubeVirt-specific conditions (AgentConnected, LiveMigratable)" diff --git a/evaluation/without_skills/rh-virt__vm-lifecycle-manager/environment/Dockerfile b/evaluation/without_skills/rh-virt__vm-lifecycle-manager/environment/Dockerfile new file mode 100644 index 00000000..a76f03e8 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-lifecycle-manager/environment/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-virt__vm-lifecycle-manager/environment/mcp-servers/mock-virt-mcp.py b/evaluation/without_skills/rh-virt__vm-lifecycle-manager/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..31b95dd3 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-lifecycle-manager/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1467 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), + ("prod-vms", {"env": "production"}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── prod-vms (instruction-specific) ────────────────────────────────── + _vm("web-frontend", "prod-vms", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "customer-facing": "true", "criticality": "high"}, + 4, 8, "Running", True, 1), + _vm("production-db", "prod-vms", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true"}, + 8, 16, "Running", True, 1), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-virt__vm-lifecycle-manager/instruction.md b/evaluation/without_skills/rh-virt__vm-lifecycle-manager/instruction.md new file mode 100644 index 00000000..622a3d38 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-lifecycle-manager/instruction.md @@ -0,0 +1,12 @@ +# VM Lifecycle Operations Task + +You are an OpenShift Virtualization administrator. Plan lifecycle operations for VMs in the cluster: stop `web-frontend` and restart `production-db`, both in namespace `prod-vms`. + +## Requirements +- Define the procedure for each operation +- Address the correct sequencing for restart (not a single atomic operation) +- Include verification steps + +Use MCP tools to examine the cluster. Document your methodology, procedures, and verification steps in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-virt__vm-lifecycle-manager/solution/solve.sh b/evaluation/without_skills/rh-virt__vm-lifecycle-manager/solution/solve.sh new file mode 100644 index 00000000..851a4668 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-lifecycle-manager/solution/solve.sh @@ -0,0 +1,29 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Lifecycle Operations Plan + +## Operation 1: Stop web-frontend +- Tool: `vm_lifecycle(namespace="prod-vms", name="web-frontend", action="stop")` +- Effect: Sets runStrategy to Halted +- Verify: `status.printableStatus` changes to "Stopped" + +## Operation 2: Restart production-db +Restart requires TWO separate calls to avoid resourceVersion conflicts: +1. `vm_lifecycle(namespace="prod-vms", name="production-db", action="stop")` +2. Wait for `status.printableStatus == "Stopped"` (poll every 5 seconds) +3. `vm_lifecycle(namespace="prod-vms", name="production-db", action="start")` + +### RunStrategy Mapping +| Action | RunStrategy Set | +|--------|----------------| +| start | Always | +| stop | Halted | +| restart | Always (after stop completes) | + +### Caveats +- Restart is NOT a single atomic operation — it's stop + wait + start +- Graceful shutdown: VM guest agent handles ACPI shutdown signal +- If VM doesn't stop within timeout, force stop may be needed +- Always verify stopped status before issuing start to avoid conflicts + +REPORT_EOF diff --git a/evaluation/without_skills/rh-virt__vm-lifecycle-manager/task.toml b/evaluation/without_skills/rh-virt__vm-lifecycle-manager/task.toml new file mode 100644 index 00000000..29808afd --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-lifecycle-manager/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-lifecycle-manager" +name = "rh-virt VM Lifecycle Management Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-lifecycle-manager", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-virt__vm-lifecycle-manager/tests/llm_judge.py b/evaluation/without_skills/rh-virt__vm-lifecycle-manager/tests/llm_judge.py new file mode 100644 index 00000000..1e8ef2e1 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-lifecycle-manager/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "two_step_restart", "file": "/root/report.md", "question": "Does the report implement restart as stop→verify stopped→start rather than a single atomic operation?", "reference": "A skilled report separates stop and start to avoid resourceVersion conflicts. An unskilled report uses a single restart command."}, + {"id": "run_strategy_mapping", "file": "/root/report.md", "question": "Does the report map start to RunStrategy: Always and stop to RunStrategy: Halted?", "reference": "A skilled report uses RunStrategy for lifecycle control. An unskilled report uses power state concepts."}, + {"id": "state_verification", "file": "/root/report.md", "question": "Does the report verify VM reached expected state (Stopped/Running) before proceeding to the next operation?", "reference": "A skilled report verifies printableStatus between operations. An unskilled report assumes instant state changes."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-virt__vm-lifecycle-manager/tests/test.sh b/evaluation/without_skills/rh-virt__vm-lifecycle-manager/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-lifecycle-manager/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-virt__vm-lifecycle-manager/tests/test_outputs.py b/evaluation/without_skills/rh-virt__vm-lifecycle-manager/tests/test_outputs.py new file mode 100644 index 00000000..98907dad --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-lifecycle-manager/tests/test_outputs.py @@ -0,0 +1,75 @@ +""" +Tests for rh-virt__vm-lifecycle-manager per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_operations(self): + c = read_report().lower() + assert ("stop" in c or "halt" in c) and ("restart" in c or "start" in c), ( + "report should discuss stop and restart operations" + ) + + def test_mentions_vms(self): + c = read_report().lower() + assert any(t in c for t in ["vm", "virtual machine", "virtualmachine"]), ( + "report should reference the target VMs" + ) + + +class TestSkillDependent: + def test_two_step_restart(self): + """Skill: Restart = stop then start (not single atomic); resourceVersion conflict.""" + c = read_report().lower() + assert ("stop" in c and "start" in c) and any(t in c for t in ["two", "separate", "sequence", "then", "first", "resourceversion", "conflict"]), ( + "should explain restart as stop-then-start, not single operation" + ) + + def test_run_strategy_control(self): + """Skill: RunStrategy Always/Halted for start/stop; not generic power state.""" + c = read_report().lower() + assert any(t in c for t in ["runstrategy", "run strategy", "always", "halted"]) and ( + "start" in c or "stop" in c + ), ( + "should map start/stop to RunStrategy (Always/Halted)" + ) + + def test_ready_verification(self): + """Skill: Verify status.printableStatus Stopped/Running after each step.""" + c = read_report().lower() + assert any(t in c for t in ["printablestatus", "printable status", "status", "stopped", "running"]) and ( + any(t in c for t in ["verify", "check", "poll", "wait", "before start"]) + ), ( + "should verify VM reached expected state before proceeding" + ) + + def test_vm_lifecycle_tool(self): + """Skill: vm_lifecycle MCP tool for start/stop/restart.""" + c = read_report().lower() + assert any(t in c for t in ["vm_lifecycle", "vm lifecycle", "lifecycle tool", "mcp"]), ( + "should reference vm_lifecycle or MCP lifecycle tool" + ) + + def test_restart_composite(self): + """Skill: Restart implemented as stop → verify stopped → wait → start.""" + c = read_report().lower() + has_stop_start = "stop" in c and "start" in c + has_wait = any(t in c for t in ["wait", "5 second", "poll", "verify stopped"]) + assert has_stop_start and has_wait, ( + "should include wait/verify between stop and start for restart" + ) diff --git a/evaluation/without_skills/rh-virt__vm-rebalance/environment/Dockerfile b/evaluation/without_skills/rh-virt__vm-rebalance/environment/Dockerfile new file mode 100644 index 00000000..a76f03e8 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-rebalance/environment/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-virt__vm-rebalance/environment/mcp-servers/mock-virt-mcp.py b/evaluation/without_skills/rh-virt__vm-rebalance/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..2e083d72 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-rebalance/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1458 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-virt__vm-rebalance/instruction.md b/evaluation/without_skills/rh-virt__vm-rebalance/instruction.md new file mode 100644 index 00000000..b4e5c640 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-rebalance/instruction.md @@ -0,0 +1,13 @@ +# VM Rebalancing Task + +You are an OpenShift Virtualization administrator. Node `hv-prod-dc1-02` is critically overloaded (88% CPU, 82% memory). Plan how to rebalance its workloads by migrating one or more VMs to less utilized nodes. + +## Requirements +- Examine current node utilization and identify which VMs on `hv-prod-dc1-02` are candidates for migration +- Evaluate migration feasibility for each candidate and determine the appropriate migration method +- Select appropriate target nodes based on available capacity and schedulability +- Identify risks and safety considerations that could affect the migration + +Use MCP tools to examine the cluster. Document your methodology, findings, and migration plan in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-virt__vm-rebalance/solution/solve.sh b/evaluation/without_skills/rh-virt__vm-rebalance/solution/solve.sh new file mode 100644 index 00000000..1f48a04e --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-rebalance/solution/solve.sh @@ -0,0 +1,41 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Rebalancing Plan + +## Current State +Node hv-prod-dc1-02 is critically overloaded: 88% CPU (14080m/16000m), 82% memory (53739Mi/65536Mi). +VMs on this node: vm-web-prod-03, vm-api-prod-01, vm-cache-prod-01, vm-etl-prod-01. + +## Migration Candidates +- vm-web-prod-03 (4 CPU, 8Gi): good candidate, RWX storage supports live migration +- vm-cache-prod-01 (2 CPU, 4Gi): good candidate, small footprint +- vm-etl-prod-01 (4 CPU, 8Gi): degraded (high I/O latency), could benefit from migration but risky during active I/O + +## Live Migration Prerequisites +1. **Storage access mode**: Must be ReadWriteMany (RWX) for live migration. ReadWriteOnce (RWO) requires cold migration (VM must be stopped first). +2. **Node schedulability**: Target node must be schedulable (not cordoned or in maintenance). +3. **CPU model compatibility**: Source and target nodes must support the same CPU model. +4. **Available capacity**: Use allocated vCPU/memory from VM spec, not runtime usage metrics. + +## Target Node Selection +- hv-prod-dc1-01: 74% CPU, 68% memory — can accept one small VM +- hv-prod-dc1-03: cordoned for maintenance — NOT schedulable +- hv-prod-dc2-01/02: different datacenter zone, only for cross-zone rebalancing + +Recommendation: Migrate vm-cache-prod-01 (2 CPU, 4Gi) to hv-prod-dc1-01. + +## Anti-Patterns to Avoid +- **No ping-pong**: Don't migrate VMs back and forth between nodes repeatedly +- **Avoid resource overcommit**: Calculate post-migration allocated resources to ensure target stays below 85% +- **Don't migrate during peak hours**: Schedule during maintenance windows +- **Cold migration caution**: Re-read VM before updating nodeAffinity to avoid resourceVersion conflict +- **Overcommit warning**: If any node exceeds 85% after rebalance, escalate + +## Migration Procedure +1. Verify vm-cache-prod-01 storage is RWX (live migration supported) +2. Verify hv-prod-dc1-01 has capacity for 2 CPU + 4Gi after migration +3. Create VirtualMachineInstanceMigration resource +4. Monitor migration progress for convergence +5. Verify VM is healthy on target node post-migration + +REPORT_EOF diff --git a/evaluation/without_skills/rh-virt__vm-rebalance/task.toml b/evaluation/without_skills/rh-virt__vm-rebalance/task.toml new file mode 100644 index 00000000..d79dfbba --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-rebalance/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-rebalance" +name = "rh-virt VM Rebalancing Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-rebalance", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-virt__vm-rebalance/tests/llm_judge.py b/evaluation/without_skills/rh-virt__vm-rebalance/tests/llm_judge.py new file mode 100644 index 00000000..76052f1f --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-rebalance/tests/llm_judge.py @@ -0,0 +1,92 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "cpu_compatibility_check", "file": "/root/report.md", "question": "Does the report check CPU model or feature compatibility between source and target nodes before recommending migration?", "reference": "A skilled report verifies CPU compatibility (model, features) to ensure live migration success. An unskilled report migrates VMs without CPU compatibility checks."}, + {"id": "overcommit_awareness", "file": "/root/report.md", "question": "Does the report assess overcommit risk (whether the target node will exceed capacity after receiving migrated VMs)?", "reference": "A skilled report calculates whether the target node can handle the additional load without overcommitting. An unskilled report moves VMs without capacity verification."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-virt__vm-rebalance/tests/test.sh b/evaluation/without_skills/rh-virt__vm-rebalance/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-rebalance/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-virt__vm-rebalance/tests/test_outputs.py b/evaluation/without_skills/rh-virt__vm-rebalance/tests/test_outputs.py new file mode 100644 index 00000000..ea445584 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-rebalance/tests/test_outputs.py @@ -0,0 +1,57 @@ +""" +Tests for rh-virt__vm-rebalance per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_migration(self): + content = read_report().lower() + assert "migrat" in content, "report should discuss migration" + + def test_mentions_node(self): + content = read_report().lower() + assert any(t in content for t in ["node", "overload", "imbalance", "utilization"]), ( + "report should reference cluster nodes or load imbalance" + ) + + +class TestSkillDependent: + def test_cpu_compatibility(self): + """Skill: CPU model/feature compatibility between source and target nodes.""" + c = read_report().lower() + assert any(t in c for t in ["cpu model", "cpu compatible", "cpu feature", "cpu architecture", "migration compatibility"]) or ( + "cpu" in c and ("compatib" in c or "model" in c) + ), ( + "should address CPU compatibility for migration" + ) + + def test_virtualmachineinstancemigration(self): + """Skill: VirtualMachineInstanceMigration for live migration.""" + c = read_report().lower() + assert any(t in c for t in ["virtualmachineinstancemigration", "vmi migration", "migration cr", "migration resource"]), ( + "should reference VirtualMachineInstanceMigration API" + ) + + def test_overcommit_warning(self): + """Skill: Overcommit detection; warn if node exceeds 100% after rebalance.""" + c = read_report().lower() + assert any(t in c for t in ["overcommit", "over commit", "exceed 100", "capacity"]) and ( + "overcommit" in c or "100" in c or "exceed" in c + ), ( + "should address overcommit risk when rebalancing" + ) diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/Dockerfile b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/Dockerfile new file mode 100644 index 00000000..f77e513d --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/Dockerfile @@ -0,0 +1,63 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY docs /root/docs + +RUN mkdir -p /logs/agent/sessions && \ + ln -s /root/docs /logs/agent/sessions/docs + +COPY docs /root/.claude/docs +COPY docs /root/.codex/docs +COPY docs /root/.opencode/docs +COPY docs /root/.goose/docs +COPY docs /root/.factory/docs +COPY docs /root/.agents/docs +COPY docs /root/.gemini/docs + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/.ai-index/semantic-index.json new file mode 100644 index 00000000..2512c2a3 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/.ai-index/semantic-index.json @@ -0,0 +1,148 @@ +{ + "index_version": "1.0.0", + "last_updated": "2026-02-17", + "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", + + "documents": [ + { + "file": "INDEX.md", + "title": "VM Troubleshooting Guide - Index", + "category": "navigation", + "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], + "use_cases": ["navigation", "tool-reference", "general-guidance"], + "token_estimate": 450, + "skills": ["all"] + }, + { + "file": "scheduling-errors.md", + "title": "VM Scheduling Errors", + "category": "scheduling", + "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], + "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], + "errors_covered": [ + "ErrorUnschedulable - Node Taints", + "ErrorUnschedulable - Insufficient Resources", + "ErrorUnschedulable - Node Selector Mismatch" + ], + "token_estimate": 820, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "storage-errors.md", + "title": "VM Storage Errors", + "category": "storage", + "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], + "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], + "errors_covered": [ + "ErrorDataVolumeNotReady - DataVolume Still Provisioning", + "ErrorDataVolumeNotReady - Storage Class Not Found", + "ErrorDataVolumeNotReady - Insufficient Storage Quota", + "ErrorPvcNotFound", + "Storage Deletion Failures", + "DataVolume Cloning Failures" + ], + "token_estimate": 1350, + "skills": ["vm-creator", "vm-delete", "vm-clone"] + }, + { + "file": "lifecycle-errors.md", + "title": "VM Lifecycle Errors", + "category": "lifecycle", + "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], + "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], + "errors_covered": [ + "VM Stuck in Terminating State", + "VM Won't Start (Non-Scheduling Issues)", + "VM Won't Stop" + ], + "token_estimate": 1150, + "skills": ["vm-delete", "vm-lifecycle-manager"] + }, + { + "file": "runtime-errors.md", + "title": "VM Runtime Errors", + "category": "runtime", + "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], + "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], + "errors_covered": [ + "CrashLoopBackOff" + ], + "token_estimate": 850, + "skills": ["vm-creator", "vm-lifecycle-manager"] + }, + { + "file": "network-errors.md", + "title": "VM Network Errors", + "category": "networking", + "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], + "use_cases": ["vm-creation", "network-troubleshooting"], + "errors_covered": [ + "Network Attachment Failures" + ], + "token_estimate": 580, + "skills": ["vm-creator"] + } + ], + + "skill_to_docs_mapping": { + "vm-creator": { + "primary_docs": ["scheduling-errors.md", "storage-errors.md"], + "secondary_docs": ["runtime-errors.md", "network-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3650, + "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" + }, + "vm-delete": { + "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 2950, + "description": "VM deletion failures - stuck terminating, storage cleanup" + }, + "vm-clone": { + "primary_docs": ["storage-errors.md"], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 1800, + "description": "VM cloning failures - DataVolume cloning, storage provisioning" + }, + "vm-lifecycle-manager": { + "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], + "secondary_docs": ["runtime-errors.md"], + "index": "INDEX.md", + "total_token_estimate": 3270, + "description": "VM start/stop failures - lifecycle management, scheduling" + }, + "vm-inventory": { + "primary_docs": [], + "secondary_docs": [], + "index": "INDEX.md", + "total_token_estimate": 450, + "description": "General guidance for status interpretation" + } + }, + + "error_to_docs_mapping": { + "ErrorUnschedulable": "scheduling-errors.md", + "ErrorDataVolumeNotReady": "storage-errors.md", + "ErrorPvcNotFound": "storage-errors.md", + "Terminating": "lifecycle-errors.md", + "CrashLoopBackOff": "runtime-errors.md", + "NetworkAttachmentFailed": "network-errors.md" + }, + + "cross_references": { + "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], + "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], + "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], + "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], + "network-errors.md": ["scheduling-errors.md"] + }, + + "progressive_disclosure_strategy": { + "description": "Load only the minimum documentation needed for current task", + "pattern": "error_type → primary_doc → cross_references (if needed)", + "token_savings": "29% average reduction vs single-file (based on rh-sre data)", + "navigation_overhead_reduction": "85% vs single-file navigation" + } +} diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/INDEX.md b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/INDEX.md new file mode 100644 index 00000000..a2e76963 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/INDEX.md @@ -0,0 +1,332 @@ +--- +title: VM Troubleshooting Guide - Index +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 + - title: OpenShift Virtualization - Virtual Machine Status + url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html + date_accessed: 2026-02-06 + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] +semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] +use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] +last_updated: 2026-02-17 +--- + +# VM Troubleshooting Guide - Index + +## Overview + +This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: + +- **Creation**: Scheduling failures, storage provisioning issues +- **Lifecycle**: Start/stop failures, stuck states +- **Deletion**: Resources stuck in Terminating, storage cleanup failures +- **Cloning**: DataVolume cloning errors, cross-namespace issues +- **Networking**: Secondary network attachment failures +- **Runtime**: CrashLoopBackOff, guest OS boot failures + +This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. + +--- + +## 🗂️ Troubleshooting Categories + +The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: + +### 1. [Scheduling Errors](scheduling-errors.md) +**When to use**: VM fails to schedule on any node + +**Errors covered**: +- ErrorUnschedulable - Node Taints +- ErrorUnschedulable - Insufficient Resources +- ErrorUnschedulable - Node Selector Mismatch + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 2. [Storage Errors](storage-errors.md) +**When to use**: VM has storage provisioning, deletion, or cloning issues + +**Errors covered**: +- ErrorDataVolumeNotReady (all 3 subsections) +- ErrorPvcNotFound +- Storage Deletion Failures +- DataVolume Cloning Failures + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### 3. [Lifecycle Errors](lifecycle-errors.md) +**When to use**: VM has start/stop/termination issues + +**Errors covered**: +- VM Stuck in Terminating State +- VM Won't Start (Non-Scheduling Issues) +- VM Won't Stop + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### 4. [Runtime Errors](runtime-errors.md) +**When to use**: VM repeatedly crashes or fails at runtime + +**Errors covered**: +- CrashLoopBackOff + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 5. [Network Errors](network-errors.md) +**When to use**: VM has secondary network attachment failures + +**Errors covered**: +- Network Attachment Failures (Multus, SR-IOV) + +**Skills that use this**: vm-create + +--- + +## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last + +### Critical Principle: MCP-First Approach + +**All diagnostic procedures in this guide follow the MCP-first pattern**: + +``` +1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) +2. ⚠️ IF FAILS: Ask user permission to use CLI command +3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval +``` + +### Why MCP Tools First? + +- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources +- **Consistency**: Same interface across all operations +- **Better Error Handling**: MCP tools return structured errors +- **Audit Trail**: MCP tool usage is logged and trackable + +### Command Pattern Examples + +Throughout this guide, you'll see diagnostic steps formatted like this: + +**✅ CORRECT Pattern (MCP First)**: + +```markdown +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Extract**: `.metadata.finalizers` from returned JSON + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` +``` + +### Available MCP Tools + +The `openshift-virtualization` MCP server provides these tools: + +**Resource Operations**: +- `resources_get` - Get specific resource (replaces `oc get `) +- `resources_list` - List resources (replaces `oc get `) +- `resources_delete` - Delete resource (replaces `oc delete`) +- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) + +**Pod Operations**: +- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) +- `pods_get` - Get pod details (replaces `oc get pod`) +- `pods_log` - Get pod logs (replaces `oc logs`) +- `pods_exec` - Execute in pod (replaces `oc exec`) +- `pods_delete` - Delete pod (replaces `oc delete pod`) +- `pods_top` - Pod resource usage (replaces `oc top pods`) + +**Events & Monitoring**: +- `events_list` - List events (replaces `oc get events`) +- `nodes_top` - Node resource usage (replaces `oc top nodes`) +- `nodes_log` - Node logs (replaces `oc adm node-logs`) +- `nodes_stats_summary` - Detailed node stats + +**VM Operations** (KubeVirt toolset): +- `vm_create` - Create VMs +- `vm_lifecycle` - Start/stop/restart VMs + +**Namespaces**: +- `namespaces_list` - List namespaces (replaces `oc get namespaces`) +- `projects_list` - List OpenShift projects (replaces `oc get projects`) + +### When CLI Commands Are Required + +Some operations have **NO MCP equivalent** and require CLI: + +- `virtctl` commands (console, VNC access) +- `oc debug node` (node debugging) +- `oc auth can-i` (permission checks) +- `oc adm taint` (node taint management) +- Complex JSON patch operations + +For these, the guide will note: **"CLI Only - No MCP equivalent"** + +### Quick Reference: CLI → MCP Mapping + +| CLI Command | MCP Tool Equivalent | +|-------------|---------------------| +| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | +| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | +| `oc logs -n ` | `pods_log` with name="", namespace="" | +| `oc get events -n ` | `events_list` with namespace="" | +| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | + +**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. + +### How to Read Diagnostic Sections + +Each error section includes: +1. **Symptom** - What you observe +2. **Description** - What's happening +3. **Possible Causes** - Why it's happening +4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback +5. **Solutions** - **MCP tools first**, then CLI fallback +6. **Verification** - **MCP tools first**, then CLI fallback + +**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. + +--- + +## 🔍 Quick Navigation by Skill + +**vm-create**: +- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics +- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning +- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff +- [Network Errors](network-errors.md) - Network attachment failures + +**vm-delete**: +- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state +- [Storage Errors](storage-errors.md) - Storage deletion failures + +**vm-clone**: +- [Storage Errors](storage-errors.md) - DataVolume cloning failures + +**vm-lifecycle-manager**: +- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop +- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling + +**vm-inventory**: +- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed + +--- + +## 📊 Documentation Coverage & Maintenance + +### Current Coverage + +This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: + +- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) +- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures +- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop +- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) +- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems + +**Total errors documented**: 12 error types across 6 categories + +--- + +### Encountering Undocumented Errors + +#### For AI Agents (Claude Code) + +If you encounter an error **not documented** in the categories above: + +1. **Report to user** with all available details (error message, affected resources, namespace) +2. **Provide best-effort diagnostics** using MCP tools: + - `resources_get` to inspect resource status + - `pods_log` to check virt-launcher or compute container logs + - `events_list` to view Kubernetes events +3. **Suggest documentation update**: + ``` + ⚠️ This error is not yet documented in the troubleshooting guide. + + **Error**: + **Affected resource**: / + + I recommend adding this error to the troubleshooting guide: + - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file + - If it's a new error class → Create new category file in docs/troubleshooting/ + + Would you like me to help document this error for future reference? + ``` + +4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs + +--- + +#### For Human Contributors + +To document a new error: + +1. **Determine the category**: + - Scheduling issue → `scheduling-errors.md` + - Storage problem → `storage-errors.md` + - Lifecycle/start/stop → `lifecycle-errors.md` + - Runtime crash → `runtime-errors.md` + - Network issue → `network-errors.md` + - New error class → Create new category file (e.g., `security-errors.md`) + +2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): + - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback + - **Solutions**: MCP Tool approach → Implementation → CLI Fallback + - **Verification**: MCP Tool checks → Expected results → CLI Fallback + +3. **Update semantic index**: + - Edit `.ai-index/semantic-index.json` + - Add error to `error_to_docs_mapping` + - Update relevant skill's `primary_docs` or `secondary_docs` + - Increment token estimates if needed + +4. **Reference in skill files**: + - Update skill's Reference Documentation section + - Ensure skill's Document Consultation steps point to the right category + +5. **Update this INDEX.md**: + - Add error to the appropriate category section (lines 50-120) + - Update "Current Coverage" list above + +--- + +## 📚 Additional Resources + +- [rh-virt README](../../README.md) - MCP server setup and skill documentation +- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources +- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/lifecycle-errors.md new file mode 100644 index 00000000..6167fcc8 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/lifecycle-errors.md @@ -0,0 +1,869 @@ +--- +title: VM Lifecycle Errors +category: kubevirt +sources: + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 +tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] +semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] +use_cases: [vm-deletion, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Lifecycle Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM lifecycle issues including start/stop failures and stuck termination states. + +**When to use this document**: +- VM stuck in `Terminating` state +- VM won't start (runStrategy is Always but VM never reaches Running) +- VM won't stop (runStrategy is Halted but VM never reaches Stopped) + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### VM Stuck in Terminating State + +**Symptom**: VM shows status `Terminating` but deletion never completes + +**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). + +**Possible Causes**: +- Finalizers blocking deletion +- PVC/DataVolume still attached and preventing cleanup +- VirtualMachineInstance (VMI) not terminating properly +- Custom controllers or operators blocking deletion +- Stuck virt-launcher pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check finalizers on the VM**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` + +**2. Check if VMI still exists**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name containing "virt-launcher-". + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check events for deletion issues**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results where `involvedObject.name` == "" and sort by timestamp. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' +``` + +**5. Check VM deletion timestamp**: + +Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. + +If present, VM is in deletion process. If null, VM is not being deleted. + +**Common Finalizer Patterns**: +- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer +- `foregroundDeletion` - Waits for dependent objects to be deleted +- Custom finalizers from operators + +**Solutions** (Use MCP Tools First): + +1. **Wait for dependent resources** (safest approach): + - Use `resources_list` to check if PVCs, DataVolumes are still deleting + - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) + +2. **Force delete VMI** (if VMI is stuck): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable - requires explicit user permission): + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +3. **Force delete virt-launcher pod** (if pod is stuck): + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + # Find the virt-launcher pod name first + oc get pods -n | grep virt-launcher- + + # Then delete it + oc delete pod virt-launcher--xxx -n --force --grace-period=0 + ``` + +4. **Remove finalizers** (⚠️ dangerous - use only as last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` + 2. Remove items from `.metadata.finalizers` array + 3. Update VM using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch not easily done via MCP): + ```bash + # This operation is complex for MCP - may need CLI + oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Check for protection labels** (vm-delete skill specific): + + Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. + + If value is "true", the vm-delete skill refuses deletion (this is expected behavior). + +**Verification**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error if deletion successful. + +**CLI Fallback**: +```bash +oc get vm -n +# Should return: Error from server (NotFound) +``` + +--- + + +--- + +### VM Won't Start (Non-Scheduling Issues) + +**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state + +**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. + +**Possible Causes**: +- Guest OS kernel panic or boot failure +- Cloud-init configuration errors +- Missing or corrupted disk image +- Insufficient memory for guest OS to boot +- QEMU/KVM virtualization errors +- VirtualMachineInstance (VMI) creation failures +- virt-launcher pod crashes + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VMI (VirtualMachineInstance) status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and overall VMI state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**2. Check VMI conditions for errors**: + +Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. View virt-launcher pod logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +First, get pod name from step 3, then: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +Look for QEMU errors, memory allocation failures, device errors. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**5. Check virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. + +Alternatively, use `pods_get` to get full pod details: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**6. Access VM console to see guest OS boot messages**: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +Look for kernel panic, initramfs errors, filesystem errors. + +**7. Check VM events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep -A 20 "Events:" +``` + +**Common Error Patterns**: + +1. **Guest OS Boot Failure**: + - Console shows kernel panic + - Guest hangs at GRUB or boot loader + - Cloud-init errors during first boot + +2. **Insufficient Memory**: + - Guest OS kills processes due to OOM + - VMI logs show memory allocation errors + +3. **QEMU Crashes**: + - virt-launcher logs show QEMU segfaults + - VMI repeatedly restarts + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for kernel panic, initramfs errors, filesystem errors. + +2. **Check virt-launcher pod logs for QEMU errors**: + + Use `pods_log` from diagnostic step 4 to view logs. + + Look for: + - "qemu-system-x86_64: ..." errors + - Memory allocation failures + - Device errors + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx + ``` + +3. **Increase memory if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check VMI memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +4. **Verify disk image integrity**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check PVC is bound: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + If using container disk, verify image exists and is accessible (check VMI spec). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get pvc -n + ``` + +5. **Check cloud-init configuration** (if applicable): + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. + + Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq + ``` + +6. **Restart VMI** (soft reset): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete VMI (VM controller will recreate it): + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for new VMI to start (use `resources_get` to check status). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n -w + ``` + +7. **Check virtualization extensions** (KVM): + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + + # In debug shell: + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should eventually return `Running`). + +Check VMI is running: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should show `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should eventually return: Running + +oc get vmi -n +# Should show: Running +``` + +--- + +### VM Won't Stop + +**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state + +**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. + +**Possible Causes**: +- Guest OS not responding to ACPI shutdown signal +- virt-launcher pod stuck and not terminating +- VirtualMachineInstance (VMI) deletion blocked by finalizers +- Guest shutdown scripts hanging +- Filesystem sync issues in guest OS + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (might show `Stopping`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**2. Check VMI status and deletion timestamp**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check VMI events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vmi -n | grep -A 10 "Events:" +``` + +**5. Check VMI finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.metadata.finalizers}' +``` + +**6. Check if guest is responsive** (if VMI still exists): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Patterns**: +- VMI shows `deletionTimestamp` but never actually deletes +- virt-launcher pod in `Terminating` state +- VM runStrategy is `Halted` but printableStatus shows `Stopping` + +**Solutions** (Use MCP Tools First): + +1. **Wait for graceful shutdown** (default: 30 seconds): + + Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. + +2. **Force stop by deleting VMI**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + This is the standard way to force-stop a VM. + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n + # Should return: Error from server (NotFound) + ``` + +3. **Force delete VMI with grace period** (if VMI won't delete): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +4. **Force delete virt-launcher pod**: + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find the pod using `pods_list_in_namespace` from diagnostic step 3. + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + ⚠️ **Note**: For force deletion with grace period, use CLI fallback. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') + oc delete pod $POD_NAME -n --force --grace-period=0 + ``` + +5. **Remove VMI finalizers** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VMI using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update VMI using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Patch VM runStrategy directly** (ensure consistency): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` (diagnostic step 1) + 2. Set `.spec.runStrategy` to `"Halted"` + 3. Update VM using `resources_create_or_update` with modified JSON + + **CLI Fallback** (merge patch easier via CLI): + Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Stopped` or `Halted`). + +Verify VMI is gone: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Verify virt-launcher pod is gone: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for pods matching `virt-launcher-`. Should return no results. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Stopped or Halted + +oc get vmi -n +# Should return: Error from server (NotFound) + +oc get pods -n | grep virt-launcher- +# Should return: No resources found +``` + +**Prevention**: +- Ensure guest OS has ACPI support enabled +- Use proper shutdown commands in guest OS +- Avoid forceful stops unless necessary (can corrupt guest filesystem) + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/network-errors.md b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/network-errors.md new file mode 100644 index 00000000..82fa313c --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/network-errors.md @@ -0,0 +1,429 @@ +--- +title: VM Network Errors +category: kubevirt +sources: + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] +semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] +use_cases: [vm-creation, network-troubleshooting] +related_docs: [INDEX.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Network Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. + +**When to use this document**: +- VM created successfully but secondary networks not attached +- NetworkAttachmentDefinition not found errors +- Multus CNI failures +- SR-IOV device attachment issues + +**Skills that use this**: vm-create + +--- + +### Network Attachment Failures + +**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working + +**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). + +**Possible Causes**: +- NetworkAttachmentDefinition doesn't exist in the namespace +- Multus CNI not installed or not configured on cluster +- Namespace mismatch (NAD in different namespace than VM) +- Interface name conflicts in VM spec +- Bridge/network configuration errors in NAD +- SR-IOV device not available (if using SR-IOV) + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List NetworkAttachmentDefinitions in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "" +} +``` + +Review `.items[].metadata.name` for available NADs. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -n +``` + +**2. List NetworkAttachmentDefinitions in all namespaces**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition" +} +``` + +Omit `namespace` parameter to list across all namespaces. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -A +``` + +**3. Check specific NetworkAttachmentDefinition**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "", + "name": "" +} +``` + +Review `.spec.config` for CNI configuration. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definition -n -o yaml +``` + +**4. Check VM network configuration**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.networks` to see network references. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq +``` + +**5. Check VM domain interfaces**: + +Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq +``` + +**6. Check virt-launcher pod network annotations** (shows actual attachments): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**7. Check Multus is installed**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "openshift-multus" +} +``` + +Should show Multus CNI pods running. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n openshift-multus +``` + +**8. Check for errors in virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n +``` + +**Common Error Messages**: +- `"network-attachment-definition not found"` - NAD doesn't exist in namespace +- `"multus CNI not configured"` - Multus not installed or misconfigured +- `"interface name conflict"` - Duplicate interface names in VM spec +- `"failed to add network"` - CNI plugin error (check NAD config) +- `"no available devices"` - SR-IOV device not available (if using SR-IOV) + +**Solutions** (Use MCP Tools First): + +1. **Verify NetworkAttachmentDefinition exists in correct namespace**: + + Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. + + If NAD is in different namespace, copy it to VM namespace: + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get NAD from source namespace using `resources_get` + 2. Modify `.metadata.namespace` to target namespace + 3. Create NAD in target namespace using `resources_create_or_update` + + **CLI Fallback** (stream processing easier via CLI): + Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" + ```bash + oc get network-attachment-definition -n -o yaml | \ + sed "s/namespace: /namespace: /" | \ + oc apply -f - + ``` + +2. **Create missing NetworkAttachmentDefinition**: + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + Example: Linux bridge network + + **Parameters**: + ```json + { + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": { + "name": "vlan100", + "namespace": "" + }, + "spec": { + "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" + ```bash + cat < + spec: + config: '{ + "cniVersion": "0.3.1", + "type": "bridge", + "bridge": "br1", + "vlan": 100, + "ipam": { + "type": "host-local", + "subnet": "192.168.100.0/24" + } + }' + EOF + ``` + +3. **Check Multus CNI installation**: + + Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. + + To check cluster network operator: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterOperator" + } + ``` + + Filter for `network` operator. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n openshift-multus + oc get clusteroperators network + ``` + +4. **Fix interface name conflicts** (if VM has duplicate names): + + Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. + + Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. + + **CLI Fallback** (interactive edit easier via CLI): + Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' + oc edit vm -n + ``` + +5. **Validate NAD configuration syntax**: + + Use `resources_get` from diagnostic step 3, extract `.spec.config`. + + Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get network-attachment-definition -n -o jsonpath='{.spec.config}' + ``` + +6. **Check SR-IOV device availability** (if using SR-IOV networks): + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + List SR-IOV network node policies: + + **Parameters**: + ```json + { + "apiVersion": "sriovnetwork.openshift.io/v1", + "kind": "SriovNetworkNodePolicy", + "namespace": "openshift-sriov-network-operator" + } + ``` + + Check SR-IOV device plugin pods: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "openshift-sriov-network-operator" + } + ``` + + Filter for pods with "device-plugin" in name. + + Check available SR-IOV devices on node: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "Node", + "name": "" + } + ``` + + Review `.status.allocatable` for SR-IOV resources. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get sriovnetworknodepolicy -n openshift-sriov-network-operator + oc get pods -n openshift-sriov-network-operator | grep device-plugin + oc describe node | grep -A 10 "Allocatable:" + ``` + +7. **Recreate VM with corrected network configuration** (if needed): + + If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +After remediation, check virt-launcher pod network status: + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +Should show all attached networks with status. Example output: +```json +[ + { + "name": "openshift-sdn", + "interface": "eth0", + "ips": ["10.128.2.10"], + "default": true + }, + { + "name": "vlan100", + "interface": "net1", + "ips": ["192.168.100.5"] + } +] +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**Check from inside VM** (via console): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# In guest OS: +ip addr show +# Should show all network interfaces (eth0, net1, etc.) +``` + +**Common Network Types**: +- **Linux Bridge**: Layer 2 bridge for VLAN networks +- **SR-IOV**: High-performance direct device assignment +- **macvlan**: MAC-based VLAN for container networks +- **OVN-Kubernetes**: OpenShift native overlay network + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/runtime-errors.md b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/runtime-errors.md new file mode 100644 index 00000000..804d3cbc --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/runtime-errors.md @@ -0,0 +1,616 @@ +--- +title: VM Runtime Errors +category: kubevirt +sources: + - title: Kubernetes Pod Lifecycle + url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ + date_accessed: 2026-02-17 +tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] +semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] +use_cases: [vm-creation, vm-lifecycle, diagnostics] +related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Runtime Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. + +**When to use this document**: +- VM or virt-launcher pod shows `CrashLoopBackOff` status +- virt-launcher pod repeatedly restarting +- Guest OS kernel panics on boot + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### CrashLoopBackOff + +**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting + +**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. + +**Possible Causes**: +- Guest OS kernel panic on boot +- Insufficient resources (memory/CPU) for guest OS +- Corrupted disk image or filesystem +- QEMU/libvirt crashes due to configuration errors +- Missing or incompatible device drivers in guest +- Resource limits too low for virt-launcher pod +- Virtualization features (KVM) not available on node + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check virt-launcher pod restart count**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# Look at RESTARTS column +``` + +**2. View recent crash logs** (previous container instance): + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "previous": true, + "tail": 100 +} +``` + +Look for QEMU errors, kernel panics, or segfaults. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n virt-launcher--xxx --previous +``` + +**3. Check current virt-launcher logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. + +**CLI Fallback** (if MCP unavailable or all containers needed): +```bash +oc logs -n virt-launcher--xxx --all-containers +``` + +**4. Check VMI conditions for crash details**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Extract `.status.conditions` for crash details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**5. Check pod events for crash reasons**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" +``` + +**6. Check pod resource limits**: + +**MCP Tool**: `pods_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +Extract `.spec.containers[0].resources` for resource limits. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq +``` + +**7. Check node kubelet logs for OOM kills**: + +⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +oc adm node-logs -u kubelet | grep -i oom +``` + +**8. Access guest console** (if VM briefly starts): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Crash Patterns**: + +1. **Guest Kernel Panic**: + - Console logs show kernel panic messages + - Guest crashes immediately after boot + - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" + +2. **OOM (Out of Memory)**: + - Pod killed with reason: `OOMKilled` + - Guest runs out of memory during boot or operation + - virt-launcher logs show memory allocation failures + +3. **QEMU Crash**: + - virt-launcher logs show QEMU segmentation fault + - Symptoms: "qemu-system-x86_64: terminated by signal" + - Configuration incompatibility or QEMU bug + +4. **Disk Image Corruption**: + - Guest cannot boot from disk + - Filesystem errors in guest console + - DataVolume import failed + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for kernel panic or boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for: + - Kernel panic messages + - Initramfs errors + - Filesystem mounting failures + - Missing device errors + +2. **Review virt-launcher crash logs**: + + Use `pods_log` with `previous: true` from diagnostic step 2. + + Look for: + - QEMU command line errors + - Device initialization failures + - Memory allocation errors + - Signal termination (SIGSEGV, SIGABRT) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx --previous + ``` + +3. **Check for OOM (Out of Memory) kills**: + + **MCP Tool**: `pods_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + Extract `.status.containerStatuses[0].lastState.terminated.reason`. + + If returns `"OOMKilled"`: + - Option 1: Increase virt-launcher memory limits + - Option 2: Decrease guest memory allocation + - Option 3: Use smaller instance type + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' + ``` + +4. **Increase resources if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check current memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +5. **Verify disk image integrity**: + + **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters for list**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" + } + ``` + + **Parameters for specific DV**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check `.status.phase` (should be `Succeeded`). + + If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get datavolume -n -o jsonpath='{.status.phase}' + ``` + +6. **Check virtualization (KVM) availability**: + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +7. **Simplify VM configuration** (eliminate variables): + + Try creating minimal VM using vm-create skill with: + - Small instance type + - No secondary networks + - Simple container disk (e.g., Fedora) + - No cloud-init + + If minimal VM works, add features back one by one. + +8. **Recreate VM with different workload** (test disk image): + + If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) + +After remediation, check pod restart count stops increasing: + +**Parameters for pods**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). + +Check VM reaches Running state: + +**Parameters for VM**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# RESTARTS should stabilize + +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Running +``` + +Verify guest is responsive: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# Should show login prompt or OS console +``` + +**Advanced Debugging**: + +**MCP Tool**: `pods_exec` (from openshift-virtualization) + +Check libvirt domain XML: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["virsh", "dumpxml", "1"] +} +``` + +Check QEMU process: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["ps", "aux"] +} +``` + +Filter output for "qemu" process. + +**CLI Fallback** (if MCP unavailable): +```bash +oc exec -n virt-launcher--xxx -- virsh dumpxml 1 +oc exec -n virt-launcher--xxx -- ps aux | grep qemu +``` + +**Prevention**: +- Start with minimal VM configuration and add complexity gradually +- Use recommended instance types for your workload +- Test disk images before deploying to production +- Ensure nodes have adequate resources and KVM support +- Monitor resource usage and set appropriate limits + +--- + +## Workaround Patterns for MCP Tool Limitations + +### General Pattern: Diagnose → Propose → Confirm → Execute + +When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): + +**1. Diagnose** the root cause using Kubernetes commands +```bash +oc describe vm -n +oc get events -n --field-selector involvedObject.name= +``` + +**2. Propose** a manual workaround to the user with clear explanation +- Explain why the issue occurred +- Show what will be changed +- List alternative options + +**3. Confirm** with user before executing (Human-in-the-Loop) +- Wait for explicit "yes" or "apply workaround" +- Never auto-execute modifications + +**4. Execute** the workaround using oc/kubectl +```bash +oc patch vm -n --type=merge -p '...' +``` + +**5. Verify** the fix was successful +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**6. Document** the limitation and suggest filing enhancement request +- Note this is temporary until MCP tool is enhanced +- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues + +### Example: Adding Tolerations Workaround + +**Diagnostic Output**: +```markdown +## ⚠️ VM Scheduling Issue Detected + +**Root Cause**: Node taints prevent VM scheduling + +**Details**: +- Found 3 nodes with taint: `virtualization=true:NoSchedule` +- VM spec does not include matching tolerations +- This prevents VM scheduling on virtualization-dedicated nodes +``` + +**Proposed Workaround**: +```bash +oc patch vm web-server -n vms --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' +``` + +**User Confirmation Required**: +``` +How would you like to proceed? +- "apply workaround" - I'll patch the VM with tolerations +- "manual" - I'll provide instructions for you to apply manually +- "cancel" - Delete the VM and abort creation +``` + +**After Execution**: +```markdown +## ✓ Workaround Applied Successfully + +**Action**: Added tolerations for taint `virtualization=true:NoSchedule` +**New Status**: Stopped (VM can now be scheduled) + +**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. +Future VMs in this cluster will need the same fix until the tool is enhanced. +``` + +--- + +## VM Status Reference + +### Status Values + +| Status | Meaning | Action Required | +|--------|---------|-----------------| +| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | +| `Running` | VM is running | Normal | +| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | +| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | +| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | +| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | +| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | +| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | +| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | +| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | + +### Checking VM Status + +```bash +# Get printable status +oc get vm -n -o jsonpath='{.status.printableStatus}' + +# Get detailed status and conditions +oc get vm -n -o jsonpath='{.status}' | jq + +# Watch status changes in real-time +oc get vm -n -w +``` + +--- + +## Best Practices for Agents + +When implementing diagnostic workflows: + +1. **Always verify VM status** after creation (wait 5-10 seconds first) +2. **Consult this document** when encountering error status values +3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) +4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) +5. **Respect human-in-the-loop** for all VM modifications +6. **Document temporary workarounds** and their limitations clearly +7. **Suggest filing issues** for missing MCP tool features + +### Document Consultation Pattern + +```markdown +**Document Consultation** (REQUIRED): +1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes +2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." +``` + +--- + +## Known MCP Tool Limitations + +### vm_create tool + +**Currently Supported**: +- ✓ Namespace, name (required) +- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) +- ✓ Size hints (small, medium, large) +- ✓ Storage size +- ✓ Autostart flag +- ✓ Networks (Multus NetworkAttachmentDefinitions) +- ✓ Performance family (u1, o1, c1, m1) +- ✓ Instance type, preference + +**Not Currently Supported** (requires workarounds): +- ✗ Tolerations (for node taints) +- ✗ Node selectors +- ✗ Affinity/anti-affinity rules +- ✗ Resource requests/limits (beyond instance type) +- ✗ Custom labels/annotations +- ✗ SSH keys injection +- ✗ Cloud-init user data + +**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. + +**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues + +--- + +## Additional Resources + +- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) +- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) +- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) +- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) + +--- + +[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/scheduling-errors.md new file mode 100644 index 00000000..e47c8c4f --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/scheduling-errors.md @@ -0,0 +1,417 @@ +--- +title: VM Scheduling Errors +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 +tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] +semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] +use_cases: [vm-creation, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Scheduling Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**When to use this document**: +- VM shows status `ErrorUnschedulable` after creation or start attempt +- VM events mention scheduling failures, taints, resources, or node selectors + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +## ErrorUnschedulable + +**Symptom**: VM shows status `ErrorUnschedulable` after creation + +**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**Possible Causes**: + +### 1. Node Taints (Most Common) + +Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM events for scheduling failures**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `involvedObject.name` == "" and look for messages like: +- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= +``` + +**2. Check node taints in the cluster**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' +``` + +**Common Taint Patterns**: +- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule +- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes +- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads + +**Solution - Add Tolerations to VM**: + +The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. + +**Workaround (post-creation using MCP Tools)**: + +**Step 1**: Get current VM spec + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Step 2**: Modify the returned JSON to add tolerations + +Add to `.spec.template.spec.tolerations`: +```json +{ + "tolerations": [ + { + "key": "virtualization", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule" + } + ] +} +``` + +**Step 3**: Update VM with modified spec + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +```json +{ + "resource": "" +} +``` + +Pass the complete modified VM resource as YAML or JSON string. + +**Step 4**: Verify tolerations were added + +Use `resources_get` again and check `.spec.template.spec.tolerations` in response. + +**Step 5**: Check if VM status improved + +Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. + +**CLI Fallback** (if MCP patch is too complex): +```bash +# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' + +# Verify tolerations +oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq + +# Check status +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**Example - Multiple Tolerations**: +```bash +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: "dedicated" + operator: "Equal" + value: "virt-workloads" + effect: "NoSchedule" +' +``` + +**Toleration Operators**: +- `Equal` - Key and value must match exactly +- `Exists` - Only key must exist (ignores value) + +**Toleration Effects**: +- `NoSchedule` - Don't schedule new pods (existing pods continue) +- `PreferNoSchedule` - Avoid scheduling if possible +- `NoExecute` - Don't schedule AND evict existing pods + +**Alternative Solutions**: +1. **Remove node taints** (if you have cluster-admin access): + ```bash + oc adm taint nodes virtualization=true:NoSchedule- + ``` + +2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits + +3. **File enhancement request** - Request tolerations support in openshift-mcp-server: + https://github.com/openshift/openshift-mcp-server/issues + +--- + +### 2. Insufficient Resources + +Not enough CPU, memory, or storage available on any node. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM resource requests**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' +``` + +**2. Check node resource availability**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. + +Alternatively, use `nodes_top` MCP tool for current resource usage. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe nodes | grep -A 5 "Allocated resources" +``` + +**3. Look for VM events mentioning "Insufficient"**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep "Insufficient" +``` + +**Example Event**: +``` +0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. +``` + +**Solutions** (Use MCP Tools First): + +1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) +2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill +3. **Delete unused VMs** - Use vm-delete skill to free up resources +4. **Check resource quotas**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters for quota**: + ```json + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" + } + ``` + + **Parameters for limit range**: + ```json + { + "apiVersion": "v1", + "kind": "LimitRange", + "namespace": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + oc describe limitrange -n + ``` + +--- + +### 3. Node Selector Mismatch + +VM requires specific node labels that don't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM node selector requirements**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.nodeSelector` to see required node labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' +``` + +**2. List available node labels**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.metadata.labels` for available labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes --show-labels +``` + +**3. Check if any nodes match the selector**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node", + "labelSelector": "=" +} +``` + +Should return at least one node with matching labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -l = +``` + +**Solutions** (Use MCP Tools First): + +**Option 1: Remove node selector from VM** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get current VM using `resources_get` (diagnostic step 1) +2. Remove `.spec.template.spec.nodeSelector` field +3. Update VM using `resources_create_or_update` with modified JSON + +**CLI Fallback** (JSON patch easier via CLI): +Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" +```bash +oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' +``` + +**Option 2: Add label to nodes** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get node using `resources_get` +2. Add label to `.metadata.labels` +3. Update node using `resources_create_or_update` + +⚠️ **Note**: Node labeling typically requires cluster admin privileges. + +**CLI Fallback** (simpler via CLI): +Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" +```bash +oc label node = +``` + +--- + +[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/storage-errors.md b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/storage-errors.md new file mode 100644 index 00000000..16ba584d --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/storage-errors.md @@ -0,0 +1,1011 @@ +--- +title: VM Storage Errors +category: kubevirt +sources: + - title: KubeVirt CDI - DataVolumes + url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ + date_accessed: 2026-02-17 + - title: Kubernetes Persistent Volumes + url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ + date_accessed: 2026-02-17 +tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] +semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] +use_cases: [vm-creation, vm-deletion, vm-cloning] +related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] +last_updated: 2026-02-17 +--- + +# VM Storage Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. + +**When to use this document**: +- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` +- Storage deletion fails after VM deletion +- DataVolume cloning operations fail +- PVC provisioning issues + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### ErrorDataVolumeNotReady + +**Symptom**: VM shows status `ErrorDataVolumeNotReady` + +**Description**: The DataVolume (persistent storage) backing the VM is not ready. + +**Possible Causes**: + +#### 1. DataVolume Still Provisioning + +Storage provisioning takes time, especially for large disks or when importing images. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Get detailed DataVolume information**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` and `.status.conditions` for provisioning details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o json +``` + +**3. Check PVC (PersistentVolumeClaim) bound status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Check `.status.phase` for each PVC (should be `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. + +#### 2. Storage Class Not Found + +The requested storage class doesn't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List available storage classes**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" +} +``` + +Review the list of available storage classes (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass +``` + +**2. Check DataVolume's requested storage class**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.spec.pvc.storageClassName` in the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' +``` + +**Solution**: +1. Use a valid storage class from the cluster +2. Recreate VM with correct storage class parameter + +#### 3. Insufficient Storage Quota + +Namespace has insufficient storage quota to provision the PVC. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check resource quotas**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**2. Check storage usage**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase +``` + +**Solution**: +1. Request quota increase from cluster admin +2. Delete unused PVCs to free quota +3. Reduce VM storage size + +--- + +### ErrorPvcNotFound + +**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List PVCs in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review the list of available PVCs (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check VM's PVC references**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' +``` + +**Solution**: +- Wait for DataVolume to create the PVC +- Manually create missing PVC +- Fix VM spec to reference correct PVC name + +--- + + +--- + +### Storage Deletion Failures + +**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace + +**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. + +**Possible Causes**: +- PVC still bound to active PersistentVolume with `Retain` policy +- DataVolume still being referenced by another resource +- CDI (Containerized Data Importer) controller issues +- Storage class retention policy preventing deletion +- Finalizers on PVC/DataVolume blocking cleanup +- PVC still mounted by a pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check PVC status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check specific PVC phase**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Released` or `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o jsonpath='{.status.phase}' +``` + +**3. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**4. Check what's using the PVC**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' +``` + +**5. Check PVC finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. + +**6. Check DataVolume finalizers**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.metadata.finalizers}' +``` + +**7. Check PV reclaim policy**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume" +} +``` + +Filter results for PV where `.spec.claimRef.name` matches ``. + +To get specific PV policy: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume", + "name": "" +} +``` + +Check `.spec.persistentVolumeReclaimPolicy`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pv | grep +oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' +``` + +**Common Finalizer Patterns**: +- `kubernetes.io/pvc-protection` - Protects PVC while in use +- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer + +**Solutions** (Use MCP Tools First): + +1. **Delete DataVolume first, then PVC**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete DataVolume first (often blocks PVC deletion): + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Wait a few seconds, then delete PVC: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete datavolume -n + oc delete pvc -n + ``` + +2. **Check for pods still using PVC**: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "" + } + ``` + + Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. + + Then delete the pods using `pods_delete`: + + **Parameters**: + ```json + { + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' + oc delete pod -n + ``` + +3. **Force delete PVC** (if safe to do so): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete pvc -n --grace-period=0 --force + ``` + +4. **Remove finalizers from PVC** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PVC using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update PVC using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Remove finalizers from DataVolume** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current DataVolume using `resources_get` (diagnostic step 6) + 2. Remove items from `.metadata.finalizers` array + 3. Update DataVolume using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Change PV reclaim policy** (if PV has Retain policy): + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PV policy using `resources_get` (diagnostic step 7) + 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` + 3. Update PV using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" + ```bash + oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' + oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' + ``` + +**Storage Quota Check** (Use MCP Tools First): + +After deletion, verify storage quota is freed: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters for quota check**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.used` to verify storage quota is freed. + +**Parameters for PVC verification**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +oc get pvc -n +``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Confirm PVC is deleted: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Confirm DataVolume is deleted: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +# Should return: Error from server (NotFound) + +oc get datavolume -n +# Should return: Error from server (NotFound) +``` + +--- + +### DataVolume Cloning Failures + +**Symptom**: VM clone created successfully but DataVolume clone operation fails + +**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. + +**Possible Causes**: +- CSI driver doesn't support volume cloning +- Source PVC storage class incompatible with cloning +- Cross-namespace cloning not permitted by storage backend +- Insufficient storage quota in target namespace +- Source PVC not in `Bound` state +- Storage class doesn't have volume cloning enabled +- CDI (Containerized Data Importer) controller issues + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume clone status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and `.metadata.name`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Check DataVolume events for errors**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe datavolume -n +``` + +**3. Check DataVolume phase**: + +Use `resources_get` from step 1, extract `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +``` + +**4. Check if storage class supports cloning**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "name": "" +} +``` + +Review the full YAML output for cloning-related configurations. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass -o yaml | grep -A 5 -i clone +``` + +**5. Check CSI driver capabilities**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "CSIDriver" +} +``` + +Review `.items[].metadata.name` for available CSI drivers. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get csidriver +``` + +**6. Check source PVC status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Bound` for cloning to work). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**7. Check target namespace storage quota**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**8. Check CDI controller logs**: + +**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) + +First, list pods in openshift-cnv namespace: + +**Parameters for pods_list_in_namespace**: +```json +{ + "namespace": "openshift-cnv", + "labelSelector": "app.kubernetes.io/component=cdi-deployment" +} +``` + +Then get logs using `pods_log`: + +**Parameters**: +```json +{ + "namespace": "openshift-cnv", + "name": "", + "tail": 100 +} +``` + +**CLI Fallback** (if MCP unavailable or easier via CLI): +```bash +oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') +``` + +**Common Error Messages**: +- `"volume cloning is not supported"` - CSI driver lacks clone capability +- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage +- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace +- `"insufficient quota"` - Target namespace lacks storage quota +- `"source PVC not bound"` - Source PVC must be in Bound state for cloning +- `"StorageClass does not support cloning"` - Storage class configuration issue + +**Solutions** (Use MCP Tools First): + +1. **Check storage class clone support**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" + } + ``` + + For each storage class in `.items[]`, check: + - `.metadata.name` (storage class name) + - `.provisioner` (CSI driver) + + Storage classes using CSI drivers typically support cloning. Look for provisioners like: + - `csi.ovirt.org` (oVirt CSI) + - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) + - `ebs.csi.aws.com` (AWS EBS CSI) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner + ``` + +2. **Verify source PVC is bound**: + + Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pvc -n -o jsonpath='{.status.phase}' + ``` + +3. **Check target namespace quota**: + + Use `resources_list` from diagnostic step 7 to check quota. + + If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + ``` + +4. **Use snapshot-based cloning** (alternative method): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Step 1**: Create VolumeSnapshot of source PVC + + **Parameters**: + ```json + { + "apiVersion": "snapshot.storage.k8s.io/v1", + "kind": "VolumeSnapshot", + "metadata": { + "name": "-snapshot", + "namespace": "" + }, + "spec": { + "source": { + "persistentVolumeClaimName": "" + } + } + } + ``` + + **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) + + **Step 3**: Create new DataVolume from snapshot + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": "-rootdisk", + "namespace": "" + }, + "spec": { + "source": { + "snapshot": { + "name": "-snapshot", + "namespace": "" + } + }, + "storage": { + "resources": { + "requests": { + "storage": "50Gi" + } + }, + "storageClassName": "" + } + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" + ```bash + cat <-snapshot + namespace: + spec: + source: + persistentVolumeClaimName: + EOF + + oc get volumesnapshot -snapshot -n + + cat <-rootdisk + namespace: + spec: + source: + snapshot: + name: -snapshot + namespace: + storage: + resources: + requests: + storage: 50Gi + storageClassName: + EOF + ``` + +5. **Use "new empty storage" option** (vm-clone skill): + - If cloning isn't supported, create VM with empty storage + - Manually copy data if needed + +6. **Cross-namespace cloning workaround**: + - Some storage backends require snapshot for cross-namespace cloning + - Create snapshot in source namespace, restore in target namespace (see solution 4 above) + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Check DataVolume reached Succeeded phase: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should return `Succeeded`). + +Check PVC was created and bound: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "-rootdisk" +} +``` + +Check `.status.phase` (should return `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +# Should return: Succeeded + +oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' +# Should return: Bound +``` + +**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/mcp-servers/mock-virt-mcp.py b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..912fb2d6 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1539 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), + ("prod-vms", {"env": "production"}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── prod-vms (instruction-specific) ────────────────────────────────── + _vm("production-db", "prod-vms", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true"}, + 8, 16, "Running", True, 1), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + +STORAGE_CLASSES = [ + { + "name": "ocs-storagecluster-ceph-rbd", + "provisioner": "openshift-storage.rbd.csi.ceph.com", + "reclaimPolicy": "Delete", + "volumeBindingMode": "Immediate", + "allowVolumeExpansion": True, + }, + { + "name": "ocs-storagecluster-cephfs", + "provisioner": "openshift-storage.cephfs.csi.ceph.com", + "reclaimPolicy": "Delete", + "volumeBindingMode": "Immediate", + "allowVolumeExpansion": False, + }, +] + +VOLUME_SNAPSHOT_CLASSES = [ + { + "name": "ocs-storagecluster-rbdplugin-snapclass", + "driver": "openshift-storage.rbd.csi.ceph.com", + "deletionPolicy": "Delete", + }, +] + + +def _build_storage_class(sc): + """Build a storage.k8s.io/v1 StorageClass resource.""" + res = { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "metadata": { + "name": sc["name"], + "uid": _uid(sc["name"]), + "creationTimestamp": CREATED, + }, + "provisioner": sc["provisioner"], + "reclaimPolicy": sc["reclaimPolicy"], + "volumeBindingMode": sc["volumeBindingMode"], + } + if sc.get("allowVolumeExpansion"): + res["allowVolumeExpansion"] = True + return res + + +def _build_volume_snapshot_class(vsc): + """Build a snapshot.storage.k8s.io/v1 VolumeSnapshotClass resource.""" + return { + "apiVersion": "snapshot.storage.k8s.io/v1", + "kind": "VolumeSnapshotClass", + "metadata": { + "name": vsc["name"], + "uid": _uid(vsc["name"]), + "creationTimestamp": CREATED, + }, + "driver": vsc["driver"], + "deletionPolicy": vsc["deletionPolicy"], + } + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "storage.k8s.io/v1" and kind == "StorageClass": + resources = [_build_storage_class(sc) for sc in STORAGE_CLASSES] + headers = ["NAME", "PROVISIONER", "RECLAIMPOLICY", "VOLUMEBINDINGMODE", "ALLOWVOLUMEEXPANSION", "AGE"] + def row(r): + return [r["metadata"]["name"], r["provisioner"], + r["reclaimPolicy"], r["volumeBindingMode"], + str(r.get("allowVolumeExpansion", False)), "90d"] + return resources, headers, row, False + + if api_version == "snapshot.storage.k8s.io/v1" and kind == "VolumeSnapshotClass": + resources = [_build_volume_snapshot_class(vsc) for vsc in VOLUME_SNAPSHOT_CLASSES] + headers = ["NAME", "DRIVER", "DELETIONPOLICY", "AGE"] + def row(r): + return [r["metadata"]["name"], r["driver"], r["deletionPolicy"], "90d"] + return resources, headers, row, False + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/instruction.md b/evaluation/without_skills/rh-virt__vm-snapshot-create/instruction.md new file mode 100644 index 00000000..34f38f23 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-create/instruction.md @@ -0,0 +1,12 @@ +# VM Snapshot Creation Task + +You are an OpenShift Virtualization administrator. Create a snapshot of VM `production-db` in namespace `prod-vms`. + +## Requirements +- Verify snapshot prerequisites (storage support, guest agent) +- Define the snapshot specification +- Address snapshot consistency levels and monitoring + +Use MCP tools to examine the cluster. Work autonomously — do not wait for user confirmation at any step. Document your methodology, findings, and snapshot plan in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/solution/solve.sh b/evaluation/without_skills/rh-virt__vm-snapshot-create/solution/solve.sh new file mode 100644 index 00000000..22659dde --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-create/solution/solve.sh @@ -0,0 +1,39 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Snapshot Plan + +## Target: production-db in prod-vms + +### Storage Snapshot Support Checks +1. Check VM `status.volumeSnapshotStatuses` for snapshot support +2. Verify no hot-plugged volumes (block snapshots - must stop VM and persist or remove) +3. Check StorageClass has a VolumeSnapshotClass +4. Verify CSI driver supports snapshots +5. Check for guest agent (determines consistency level) +6. Create via resources_create_or_update; poll status.phase (InProgress/Succeeded/Failed) and status.readyToUse + +### Snapshot Type +- **With guest agent**: Application-consistent (freeze/thaw of filesystem) + - `status.indications` will show `GuestAgent` +- **Without guest agent**: Crash-consistent (point-in-time disk state) + - `status.indications` will show `Online` only + +### VirtualMachineSnapshot YAML +```yaml +apiVersion: snapshot.kubevirt.io/v1beta1 +kind: VirtualMachineSnapshot +metadata: + name: production-db-backup-20240301 + namespace: prod-vms +spec: + source: + apiGroup: kubevirt.io + kind: VirtualMachine + name: production-db +``` + +### Monitoring +- Poll `status.phase`: InProgress → Succeeded or Failed +- Check `status.readyToUse: true` before relying on snapshot + +REPORT_EOF diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/task.toml b/evaluation/without_skills/rh-virt__vm-snapshot-create/task.toml new file mode 100644 index 00000000..c563a3ed --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-create/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-snapshot-create" +name = "rh-virt VM Snapshot Creation Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-snapshot-create", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/tests/llm_judge.py b/evaluation/without_skills/rh-virt__vm-snapshot-create/tests/llm_judge.py new file mode 100644 index 00000000..cf067a9c --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-create/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "volume_snapshot_class", "file": "/root/report.md", "question": "Does the report check for VolumeSnapshotClass as a prerequisite for CSI snapshot support?", "reference": "A skilled report verifies VolumeSnapshotClass exists. An unskilled report attempts snapshots without checking prerequisites."}, + {"id": "hot_plugged_blocker", "file": "/root/report.md", "question": "Does the report note that hot-plugged volumes block snapshot creation entirely?", "reference": "A skilled report checks for hot-plugged volumes. An unskilled report doesn't know about this blocker."}, + {"id": "consistency_levels", "file": "/root/report.md", "question": "Does the report distinguish application-consistent (GuestAgent) from crash-consistent (Online only) snapshots?", "reference": "A skilled report checks status.indications for GuestAgent presence. An unskilled report doesn't distinguish consistency levels."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/tests/test.sh b/evaluation/without_skills/rh-virt__vm-snapshot-create/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-create/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/tests/test_outputs.py b/evaluation/without_skills/rh-virt__vm-snapshot-create/tests/test_outputs.py new file mode 100644 index 00000000..c4189fb6 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-create/tests/test_outputs.py @@ -0,0 +1,77 @@ +""" +Tests for rh-virt__vm-snapshot-create per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_snapshot(self): + content = read_report().lower() + assert "snapshot" in content, "report should mention snapshots" + + def test_mentions_vm(self): + content = read_report().lower() + assert any(t in content for t in ["vm", "virtual machine", "virtualmachine"]), ( + "report should reference the target VM" + ) + + +class TestSkillDependent: + def test_volume_snapshot_class(self): + """Skill: VolumeSnapshotClass prerequisite for CSI snapshot support.""" + c = read_report().lower() + assert any(t in c for t in ["volumesnapshotclass", "volume snapshot class", "snapshot class", "csi driver"]), ( + "should mention VolumeSnapshotClass for snapshot prerequisites" + ) + + def test_quiesce_consistency(self): + """Skill: Quiesce/freeze for application-consistent snapshots; guest agent.""" + c = read_report().lower() + assert any(t in c for t in ["quiesce", "freeze", "thaw", "guest agent", "application-consistent", "qemu-guest-agent"]), ( + "should discuss quiesce/freeze for consistency" + ) + + def test_snapshot_cr_structure(self): + """Skill: VirtualMachineSnapshot CR with spec.source.""" + c = read_report().lower() + assert "virtualmachinesnapshot" in c and any(t in c for t in ["spec", "source", "snapshot.kubevirt", "apiversion"]), ( + "should define VirtualMachineSnapshot resource structure" + ) + + def test_hot_plugged_blocker(self): + """Skill: Hot-plugged volumes block snapshot creation.""" + c = read_report().lower() + assert any(t in c for t in ["hot-plug", "hotplug", "hot plug", "block snapshot", "cannot snapshot"]), ( + "should address hot-plugged volumes blocking snapshots" + ) + + def test_status_indications(self): + """Skill: status.indications (GuestAgent, Online) for consistency level.""" + c = read_report().lower() + assert any(t in c for t in ["indications", "guestagent", "online", "status.phase", "inprogress", "succeeded"]), ( + "should reference snapshot status/indications" + ) + + def test_guest_agent_connected_check(self): + """Docs teach checking AgentConnected condition to determine if + application-consistent (vs crash-consistent) snapshots are possible. + Without docs, agents don't check guest agent status before snapshot.""" + c = read_report().lower() + assert any(t in c for t in [ + "agentconnected", "agent connected", "guest agent", + "application-consistent", "crash-consistent", + ]), "should check AgentConnected for snapshot consistency level" diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-delete/environment/Dockerfile b/evaluation/without_skills/rh-virt__vm-snapshot-delete/environment/Dockerfile new file mode 100644 index 00000000..a76f03e8 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-delete/environment/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-delete/environment/mcp-servers/mock-virt-mcp.py b/evaluation/without_skills/rh-virt__vm-snapshot-delete/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..2e083d72 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-delete/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1458 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-delete/instruction.md b/evaluation/without_skills/rh-virt__vm-snapshot-delete/instruction.md new file mode 100644 index 00000000..3058c144 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-delete/instruction.md @@ -0,0 +1,12 @@ +# VM Snapshot Deletion Task + +You are an OpenShift Virtualization administrator. Delete snapshot `production-db-backup-20240215` for VM `production-db` in namespace `prod-vms`. + +## Requirements +- Verify the snapshot is safe to delete (no active restores, not the last snapshot) +- Include user confirmation safeguards +- Verify deletion completed + +Use MCP tools to examine the cluster. Document your methodology, findings, and deletion plan in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-delete/solution/solve.sh b/evaluation/without_skills/rh-virt__vm-snapshot-delete/solution/solve.sh new file mode 100644 index 00000000..11098bb3 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-delete/solution/solve.sh @@ -0,0 +1,26 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Snapshot Deletion Plan + +## Target: production-db-backup-20240215 + +### Safety Checks +1. **Restore conflict check**: Verify no active VirtualMachineRestore references this snapshot + - If snapshot is in use by a restore operation, deletion will fail +2. **Last snapshot warning**: List all snapshots for production-db + - Other snapshots exist (production-db-backup-20240301) — NOT the last snapshot + - If this were the only remaining snapshot, show explicit warning + +### Deletion Procedure +1. Verify snapshot exists (apiVersion: snapshot.kubevirt.io/v1beta1, kind: VirtualMachineSnapshot) +2. Check for active VirtualMachineRestore resources (snapshot in use blocks deletion) +3. List other snapshots for production-db via labelSelector vm.kubevirt.io/name +4. Request user confirmation (proceed yes/no) +5. Delete snapshot via resources_delete +6. Verify deletion completed +7. Impact: Storage freed, recovery point removed + +### Note +This is NOT the last snapshot — production-db-backup-20240301 remains available for restore. + +REPORT_EOF diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-delete/task.toml b/evaluation/without_skills/rh-virt__vm-snapshot-delete/task.toml new file mode 100644 index 00000000..7d13e981 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-delete/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-snapshot-delete" +name = "rh-virt VM Snapshot Deletion Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-snapshot-delete", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-delete/tests/llm_judge.py b/evaluation/without_skills/rh-virt__vm-snapshot-delete/tests/llm_judge.py new file mode 100644 index 00000000..92546360 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-delete/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "restore_conflict", "file": "/root/report.md", "question": "Does the report check for active VirtualMachineRestore before deleting a snapshot?", "reference": "A skilled report checks for active restores. An unskilled report deletes without checking conflicts."}, + {"id": "last_snapshot_warning", "file": "/root/report.md", "question": "Does the report warn when deleting the only remaining snapshot for a VM?", "reference": "A skilled report warns about loss of last recovery point. An unskilled report deletes without warning."}, + {"id": "label_selector_filter", "file": "/root/report.md", "question": "Does the report use spec.source.name or vm.kubevirt.io/name label to list other snapshots for the same VM?", "reference": "A skilled report uses proper filtering to find related snapshots. An unskilled report lists all snapshots without VM filtering."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-delete/tests/test.sh b/evaluation/without_skills/rh-virt__vm-snapshot-delete/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-delete/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-delete/tests/test_outputs.py b/evaluation/without_skills/rh-virt__vm-snapshot-delete/tests/test_outputs.py new file mode 100644 index 00000000..f7220d55 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-delete/tests/test_outputs.py @@ -0,0 +1,71 @@ +""" +Tests for rh-virt__vm-snapshot-delete per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_snapshot(self): + content = read_report().lower() + assert "snapshot" in content, "report should mention snapshots" + + def test_mentions_deletion(self): + content = read_report().lower() + assert "delet" in content, "report should discuss deletion" + + +class TestSkillDependent: + def test_restore_conflict_check(self): + """Skill: Active VirtualMachineRestore blocks snapshot deletion.""" + c = read_report().lower() + assert any(t in c for t in ["virtualmachinerestore", "restore", "in use", "active restore", "block delet"]) and ( + "restore" in c or "conflict" in c + ), ( + "should check for active restore blocking deletion" + ) + + def test_last_snapshot_warning(self): + """Skill: Warn when deleting the only snapshot for a VM.""" + c = read_report().lower() + assert any(t in c for t in ["last snapshot", "only snapshot", "no recovery", "only remaining", "no other snapshot"]) or ( + "last" in c and "snapshot" in c and ("warn" in c or "only" in c) + ), ( + "should warn when deleting the last snapshot for a VM" + ) + + def test_storage_reclaim(self): + """Skill: Storage freed by deletion; recovery point lost.""" + c = read_report().lower() + assert any(t in c for t in ["storage freed", "storage reclaim", "freed", "recovery point"]), ( + "should mention storage reclamation or recovery point loss" + ) + + def test_virtualmachinesnapshot_delete(self): + """Skill: Delete VirtualMachineSnapshot resource.""" + c = read_report().lower() + assert any(t in c for t in ["virtualmachinesnapshot", "resources_delete", "delete snapshot"]) and ( + "snapshot" in c + ), ( + "should reference VirtualMachineSnapshot deletion" + ) + + def test_list_other_snapshots(self): + """Skill: List other snapshots for same VM before delete.""" + c = read_report().lower() + assert any(t in c for t in ["spec.source.name", "label selector", "vm.kubevirt.io/name", "other snapshot", "list snapshot", "same vm"]), ( + "should list other snapshots for the VM" + ) diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-list/environment/Dockerfile b/evaluation/without_skills/rh-virt__vm-snapshot-list/environment/Dockerfile new file mode 100644 index 00000000..a76f03e8 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-list/environment/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-list/environment/mcp-servers/mock-virt-mcp.py b/evaluation/without_skills/rh-virt__vm-snapshot-list/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..1d1132df --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-list/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1500 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), + ("prod-vms", {"env": "production"}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── prod-vms (instruction-specific) ────────────────────────────────── + _vm("production-db", "prod-vms", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true"}, + 8, 16, "Running", True, 1), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, + # ── prod-vms / production-db (instruction-specific) ─────────────────── + { + "name": "production-db-backup-20260210", + "namespace": "prod-vms", + "vm_name": "production-db", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-10T08:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-proddb-root-20260210"}, + ], + }, + { + "name": "production-db-snap-20260218", + "namespace": "prod-vms", + "vm_name": "production-db", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-18T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-proddb-root-20260218"}, + ], + }, + { + "name": "production-db-snap-failed", + "namespace": "prod-vms", + "vm_name": "production-db", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-22T11:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-list/instruction.md b/evaluation/without_skills/rh-virt__vm-snapshot-list/instruction.md new file mode 100644 index 00000000..2c6ed187 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-list/instruction.md @@ -0,0 +1,12 @@ +# VM Snapshot Listing Task + +You are an OpenShift Virtualization administrator. List and inspect all snapshots for VM `production-db` in namespace `prod-vms`. + +## Requirements +- List all snapshots with their status and readiness +- Show creation timestamps +- Identify any failed or incomplete snapshots + +Use MCP tools to query snapshot data. Document your methodology and write the snapshot inventory in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-list/solution/solve.sh b/evaluation/without_skills/rh-virt__vm-snapshot-list/solution/solve.sh new file mode 100644 index 00000000..2e33f350 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-list/solution/solve.sh @@ -0,0 +1,30 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Snapshot Inventory + +## Snapshots for production-db in prod-vms + +### Query Method +- API: `resources_list(apiVersion="snapshot.kubevirt.io/v1beta1", kind="VirtualMachineSnapshot", namespace="prod-vms")` +- Filter: `labelSelector: vm.kubevirt.io/name=production-db` +- Fallback: If label missing, filter by `spec.source.name == "production-db"` + +### Snapshot List +| Name | Status | Ready | Created | +|------|--------|-------|---------| +| production-db-backup-20240301 | Succeeded | true | 2024-03-01T10:00:00Z | +| production-db-backup-20240215 | Succeeded | true | 2024-02-15T08:30:00Z | + +### Status Fields +- `status.phase`: InProgress, Succeeded, Failed +- `status.readyToUse`: true/false — snapshot can be used for restore +- `spec.source.name`: Source VM name +- `metadata.creationTimestamp`: Creation time + +### Actions +- Restore: "Restore VM production-db from snapshot " +- Delete: "Delete snapshot " + +### No failed or incomplete snapshots found. + +REPORT_EOF diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-list/task.toml b/evaluation/without_skills/rh-virt__vm-snapshot-list/task.toml new file mode 100644 index 00000000..3e9cc1cd --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-list/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-snapshot-list" +name = "rh-virt VM Snapshot Listing Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-snapshot-list", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-list/tests/llm_judge.py b/evaluation/without_skills/rh-virt__vm-snapshot-list/tests/llm_judge.py new file mode 100644 index 00000000..aa42d89d --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-list/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "ready_to_use_status", "file": "/root/report.md", "question": "Does the report show readyToUse status indicating which snapshots are safe to restore?", "reference": "A skilled report includes readyToUse for each snapshot. An unskilled report only shows names and dates."}, + {"id": "phase_and_creation", "file": "/root/report.md", "question": "Does the report show status.phase (Succeeded/Failed/InProgress) and creation timestamp for each snapshot?", "reference": "A skilled report includes phase and timestamp. An unskilled report shows minimal snapshot metadata."}, + {"id": "label_selector_filtering", "file": "/root/report.md", "question": "Does the report mention using the vm.kubevirt.io/name label or label selector to filter or identify snapshots belonging to a specific VM?", "reference": "A skilled report references the vm.kubevirt.io/name label for filtering snapshots by source VM, or shows label selector parameters in API calls. An unskilled report lists snapshots without mentioning the KubeVirt label-based filtering mechanism."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-list/tests/test.sh b/evaluation/without_skills/rh-virt__vm-snapshot-list/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-list/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-list/tests/test_outputs.py b/evaluation/without_skills/rh-virt__vm-snapshot-list/tests/test_outputs.py new file mode 100644 index 00000000..06ac48d3 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-list/tests/test_outputs.py @@ -0,0 +1,62 @@ +""" +Tests for rh-virt__vm-snapshot-list per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_snapshots(self): + content = read_report().lower() + assert "snapshot" in content, "report should mention snapshots" + + def test_has_structured_output(self): + content = read_report() + assert "|" in content or "- " in content, "report should have structured output (table or list)" + + +class TestSkillDependent: + def test_ready_to_use_status(self): + """Skill: readyToUse status for restore readiness.""" + c = read_report().lower() + assert any(t in c for t in ["readytouse", "ready to use", "ready for restore"]), ( + "should reference readyToUse status for snapshot readiness" + ) + + def test_creation_timestamp(self): + """Skill: metadata.creationTimestamp or creation time.""" + c = read_report().lower() + assert any(t in c for t in ["creationtimestamp", "creation timestamp", "created", "when"]), ( + "should show creation timestamp for each snapshot" + ) + + def test_phase_status(self): + """Skill: status.phase (Succeeded, Failed, InProgress).""" + c = read_report().lower() + assert any(t in c for t in ["succeeded", "failed", "inprogress", "status.phase", "phase"]) and ( + "succeeded" in c or "failed" in c or "phase" in c + ), ( + "should show phase (Succeeded/Failed/InProgress)" + ) + + def test_label_selector_for_vm_filtering(self): + """Skill teaches using vm.kubevirt.io/name label selector to + filter snapshots by source VM. Without skill, agents list all + snapshots without label-based filtering.""" + c = read_report() + assert "vm.kubevirt.io" in c or "labelSelector" in c or "label selector" in c.lower(), ( + "should reference vm.kubevirt.io/name label for snapshot filtering" + ) diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-restore/environment/Dockerfile b/evaluation/without_skills/rh-virt__vm-snapshot-restore/environment/Dockerfile new file mode 100644 index 00000000..a76f03e8 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-restore/environment/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +RUN pip3 install --break-system-packages \ + pyyaml==6.0.1 \ + fastmcp + +ENV KUBECONFIG=/root/.kube/config +RUN mkdir -p /root/.kube && echo '\ +apiVersion: v1\n\ +kind: Config\n\ +current-context: ocp-prod\n\ +clusters:\n\ +- name: ocp-prod\n\ + cluster:\n\ + server: https://api.ocp-prod.example.com:6443\n\ +contexts:\n\ +- name: ocp-prod\n\ + context:\n\ + cluster: ocp-prod\n\ + user: admin\n\ + namespace: default\n\ +users:\n\ +- name: admin\n\ + user:\n\ + token: mock-token-for-testing\n' > /root/.kube/config + +COPY mcp-servers /root/.mcp-servers + +RUN echo '{ \ + "mcpServers": { \ + "openshift-virtualization": { \ + "command": "python3", \ + "args": ["/root/.mcp-servers/mock-virt-mcp.py"] \ + } \ + } \ +}' > /root/.mcp.json + +WORKDIR /root diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-restore/environment/mcp-servers/mock-virt-mcp.py b/evaluation/without_skills/rh-virt__vm-snapshot-restore/environment/mcp-servers/mock-virt-mcp.py new file mode 100644 index 00000000..2e083d72 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-restore/environment/mcp-servers/mock-virt-mcp.py @@ -0,0 +1,1458 @@ +#!/usr/bin/env python3 +""" +Mock OpenShift MCP Server for OpenShift Virtualization. + +Faithfully implements the tool interface of: + https://github.com/openshift/openshift-mcp-server +Enabled toolsets: config, core, kubevirt + +Simulated OpenShift cluster: + Cluster: ocp-virt-prod (OpenShift 4.15, K8s 1.28) + Namespaces: virt-prod-dc1, virt-prod-dc2, virt-staging, virt-dev, + openshift-cnv, openshift-compliance, openshift-monitoring, default + Nodes: 8 workers (hypervisor-class) + VMs: 32 KubeVirt VirtualMachines + Security: 5 VulnerabilityReports in openshift-compliance +""" + +import hashlib +import json +from typing import Optional + +import yaml +from fastmcp import FastMCP + +mcp = FastMCP("openshift-virtualization") + +CLUSTER = "ocp-virt-prod" +API_URL = "https://api.ocp-virt-prod.example.com:6443" +K8S_VER = "v1.28.12+f26e58e" +OCP_VER = "4.15.8" +NOW = "2026-03-02T12:00:00Z" +CREATED = "2025-11-15T10:00:00Z" + +# ═══════════════════════════════════════════════════════════════════════════ +# COMPACT DATA +# ═══════════════════════════════════════════════════════════════════════════ + +NAMESPACES = [ + ("virt-prod-dc1", {"env": "production", "dc": "dc1"}), + ("virt-prod-dc2", {"env": "production", "dc": "dc2"}), + ("virt-staging", {"env": "staging"}), + ("virt-dev", {"env": "development"}), + ("openshift-cnv", {"operator": "kubevirt-hyperconverged"}), + ("openshift-compliance", {"operator": "compliance"}), + ("openshift-monitoring", {}), + ("default", {}), +] + + +def _n(name, zone, status, unschedulable, cpu_cap, cpu_use, mem_cap, mem_use, pods, + taints=None, maint=None, itype="m5.4xlarge"): + return dict(name=name, zone=zone, status=status, unschedulable=unschedulable, + cpu_cap=cpu_cap, cpu_use=cpu_use, mem_cap=mem_cap, mem_use=mem_use, + pods=pods, taints=taints or [], maint=maint, itype=itype) + + +NODES = [ + _n("hv-prod-dc1-01", "dc1", "Ready", False, 16000, 11840, 65536, 44564, 12), + _n("hv-prod-dc1-02", "dc1", "Ready", False, 16000, 14080, 65536, 53739, 14), + _n("hv-prod-dc1-03", "dc1", "Ready,SchedulingDisabled", True, 16000, 1920, 65536, 9830, 6, + taints=[{"key": "node.kubernetes.io/unschedulable", "effect": "NoSchedule"}], + maint="Scheduled firmware update — ETA 6 hours"), + _n("hv-prod-dc2-01", "dc2", "Ready", False, 16000, 11360, 65536, 41287, 12), + _n("hv-prod-dc2-02", "dc2", "Ready", False, 16000, 12640, 65536, 49807, 15), + _n("hv-staging-01", "staging", "Ready", False, 8000, 4160, 32768, 15728, 10, itype="m5.2xlarge"), + _n("hv-staging-02", "staging", "Ready", False, 8000, 3040, 32768, 11468, 8, itype="m5.2xlarge"), + _n("hv-dev-01", "dev", "Ready", False, 8000, 4880, 32768, 18022, 14, itype="m5.2xlarge"), +] + + +def _vm(name, ns, node, os, env, labels, cpu, mem, status, ready, last_seen, + conds=None, pinned=False): + return dict(name=name, ns=ns, node=node, os=os, env=env, labels=labels, + cpu=cpu, mem=mem, status=status, ready=ready, + last_seen=last_seen, conds=conds or [], pinned=pinned) + + +VMS = [ + # ── virt-prod-dc1 / hv-prod-dc1-01 (4) ────────────────────────────── + _vm("vm-web-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true", "compliance/soc2": "true", + "criticality": "high", "customer-facing": "true"}, 4, 8, "Running", True, 1), + _vm("vm-web-prod-02", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "web", "compliance/pci-dss": "true"}, 4, 8, "Running", True, 1), + _vm("vm-lb-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-8.8", "production", + {"app": "lb", "criticality": "high", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-monitor-prod-01", "virt-prod-dc1", "hv-prod-dc1-01", "rhel-9.3", "production", + {"app": "monitoring"}, 2, 4, "Running", True, 1), + + # ── virt-prod-dc1 / hv-prod-dc1-02 (4 — CRITICAL utilization) ─────── + _vm("vm-web-prod-03", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "web", "customer-facing": "true"}, 4, 8, "Running", True, 2), + _vm("vm-api-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true", "criticality": "high"}, 4, 8, "Running", True, 1), + _vm("vm-cache-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "cache", "ha": "true"}, 2, 4, "Running", True, 1), + _vm("vm-etl-prod-01", "virt-prod-dc1", "hv-prod-dc1-02", "rhel-8.9", "production", + {"app": "etl", "compliance/hipaa": "true"}, + 4, 8, "Running", True, 1, + conds=[("Degraded", "True", "High I/O latency: avg write latency 45ms (threshold 20ms)")]), + + # ── virt-prod-dc1 / hv-prod-dc1-03 (2 — MAINTENANCE node) ─────────── + _vm("vm-backup-prod-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-8.8", "production", + {"app": "backup", "criticality": "low"}, 2, 4, "Stopped", False, 3, pinned=True), + _vm("vm-legacy-auth-01", "virt-prod-dc1", "hv-prod-dc1-03", "rhel-7.9", None, + {"app": "auth", "criticality": "high", "legacy": "true"}, + 2, 4, "Running", True, 3, + conds=[("Degraded", "True", "EOL operating system: RHEL 7.9 reached end of life")]), + + # ── virt-prod-dc2 / hv-prod-dc2-01 (4) ────────────────────────────── + _vm("vm-api-prod-02", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "api", "compliance/soc2": "true"}, 4, 8, "Running", True, 2), + _vm("vm-db-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/pci-dss": "true", + "compliance/soc2": "true"}, 8, 16, "Running", True, 1), + _vm("vm-queue-prod-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-9.2", "production", + {"app": "queue", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + _vm("vm-legacy-pay-01", "virt-prod-dc2", "hv-prod-dc2-01", "rhel-8.7", None, + {"app": "payment-gateway", "criticality": "high", "legacy": "true"}, + 4, 8, "Running", True, 2), + + # ── virt-prod-dc2 / hv-prod-dc2-02 (5 — WARNING utilization) ──────── + _vm("vm-db-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.3", "production", + {"app": "db", "criticality": "high", "compliance/soc2": "true"}, + 8, 16, "Running", True, 1), + _vm("vm-cache-prod-02", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "cache"}, 2, 4, "Running", False, 12, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 12 days")]), + _vm("vm-batch-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.9", "production", + {"app": "batch"}, 4, 8, "Stopped", False, 4), + _vm("vm-legacy-reports-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-8.6", None, + {"app": "financial-reporting", "legacy": "true"}, + 2, 4, "Running", True, 6), + _vm("vm-log-prod-01", "virt-prod-dc2", "hv-prod-dc2-02", "rhel-9.2", "production", + {"app": "logging", "compliance/soc2": "true"}, 2, 4, "Running", True, 1), + + # ── virt-staging / hv-staging-01 (4) ───────────────────────────────── + _vm("vm-web-stg-01", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 1), + _vm("vm-web-stg-02", "virt-staging", "hv-staging-01", "rhel-9.2", "staging", + {"app": "web"}, 2, 4, "Running", True, 2), + _vm("vm-api-stg-01", "virt-staging", "hv-staging-01", "rhel-8.9", "staging", + {"app": "api"}, 2, 4, "Running", True, 2), + _vm("vm-perf-stg-01", "virt-staging", "hv-staging-01", "rhel-9.3", "staging", + {"app": "perf-test"}, 4, 8, "Running", True, 1), + + # ── virt-staging / hv-staging-02 (3) ───────────────────────────────── + _vm("vm-db-stg-01", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Running", True, 1), + _vm("vm-db-stg-02", "virt-staging", "hv-staging-02", "rhel-9.2", "staging", + {"app": "db"}, 4, 8, "Paused", False, 3), + _vm("vm-qa-stg-01", "virt-staging", "hv-staging-02", "rhel-8.9", "staging", + {"app": "qa"}, 2, 4, "Running", True, 1), + + # ── virt-dev / hv-dev-01 (6) ───────────────────────────────────────── + _vm("vm-dev-01", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-02", "virt-dev", "hv-dev-01", "rhel-8.8", "development", + {"app": "dev"}, 2, 4, "Running", True, 2), + _vm("vm-dev-03", "virt-dev", "hv-dev-01", "rhel-8.9", "development", + {"app": "dev"}, 2, 4, "Stopped", False, 14, + conds=[("AgentConnected", "False", "Guest agent not responding")]), + _vm("vm-sandbox-01", "virt-dev", "hv-dev-01", "rhel-9.2", "development", + {"app": "sandbox"}, 2, 4, "Running", True, 1), + _vm("vm-test-01", "virt-dev", "hv-dev-01", "rhel-9.3", "development", + {"app": "test"}, 2, 4, "Running", True, 1), + _vm("vm-archive-01", "virt-dev", "hv-dev-01", "rhel-8.6", "development", + {"app": "archive", "legacy": "true"}, + 2, 4, "Running", False, 45, + conds=[("AgentConnected", "False", + "Guest agent has not responded for 45 days")]), +] + + +def _adv(adv_id, name, synopsis, severity, cvss, compliance, deadline, + description, affected, remediation_available=True): + return dict(id=adv_id, name=name, synopsis=synopsis, severity=severity, + cvss=cvss, compliance=compliance, deadline=deadline, + description=description, affected=affected, + remediation_available=remediation_available) + + +ADVISORIES = [ + _adv("RHSA-2026:1234", "rhsa-2026-1234", + "Critical: kernel security update", "Critical", 9.8, + ["pci-dss", "soc2"], 30, + "Remote code execution in kernel network stack allows unauthenticated " + "attackers to execute arbitrary code via crafted packets.", + [("vm-web-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-02", "virt-prod-dc1", "Vulnerable"), + ("vm-db-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-web-stg-01", "virt-staging", "Remediated"), + ("vm-web-stg-02", "virt-staging", "Remediated")]), + _adv("RHSA-2026:2345", "rhsa-2026-2345", + "Important: openssl security update", "Important", 7.8, + ["soc2"], 60, + "Buffer overflow in OpenSSL TLS handshake processing allows " + "authenticated attackers to escalate privileges.", + [("vm-api-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-api-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-db-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-queue-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-log-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-api-stg-01", "virt-staging", "Remediated"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:3456", "rhsa-2026-3456", + "Moderate: glibc security update", "Moderate", 5.4, + ["hipaa"], 90, + "Information disclosure in glibc DNS resolver allows adjacent " + "network attackers to read portions of process memory.", + [("vm-etl-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-cache-prod-02", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-01", "virt-dev", "Vulnerable"), + ("vm-dev-02", "virt-dev", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-dev-03", "virt-dev", "Remediated"), + ("vm-archive-01", "virt-dev", "Remediated")]), + _adv("RHSA-2026:4567", "rhsa-2026-4567", + "Important: httpd security update", "Important", 7.2, + ["pci-dss"], 90, + "Request smuggling in Apache httpd allows attackers to bypass " + "access controls on payment-processing endpoints.", + [("vm-legacy-pay-01", "virt-prod-dc2", "Vulnerable"), + ("vm-lb-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-auth-01", "virt-prod-dc1", "Vulnerable"), + ("vm-web-prod-03", "virt-prod-dc1", "Vulnerable"), + ("vm-legacy-reports-01", "virt-prod-dc2", "Remediated")]), + _adv("RHSA-2026:5678", "rhsa-2026-5678", + "Low: systemd information disclosure", "Low", 3.1, + [], None, + "Information disclosure in systemd-journald allows local users to " + "read journal entries from other user sessions under specific " + "SELinux configurations.", + [("vm-monitor-prod-01", "virt-prod-dc1", "Vulnerable"), + ("vm-batch-prod-01", "virt-prod-dc2", "Vulnerable"), + ("vm-db-stg-02", "virt-staging", "Vulnerable"), + ("vm-archive-01", "virt-dev", "Vulnerable")], + remediation_available=False), +] + +# Build per-VM advisory lookup +_VM_ADV = {} +for _a in ADVISORIES: + for _vn, _vns, _vs in _a["affected"]: + _VM_ADV.setdefault(_vn, []).append( + {"id": _a["id"], "severity": _a["severity"], "status": _vs, + "remediationAvailable": _a["remediation_available"]}) + +EVENTS = [ + ("virt-prod-dc1", "Warning", "NodeSchedulingDisabled", + "Node/hv-prod-dc1-03", + "Node cordoned for maintenance: Scheduled firmware update — ETA 6 hours"), + ("virt-prod-dc2", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-cache-prod-02", + "Guest agent has not responded for 12 days — last contact 2026-02-18"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-archive-01", + "Guest agent has not responded for 45 days — last contact 2026-01-16"), + ("virt-dev", "Warning", "GuestAgentNotResponding", + "VirtualMachine/vm-dev-03", + "Guest agent not responding — VM stopped for 14 days"), + ("virt-prod-dc1", "Warning", "HighIOLatency", + "VirtualMachineInstance/vm-etl-prod-01", + "Average write latency 45ms exceeds threshold 20ms"), + ("virt-prod-dc1", "Warning", "EOLOperatingSystem", + "VirtualMachine/vm-legacy-auth-01", + "RHEL 7.9 has reached end of life — no further security updates"), + ("virt-prod-dc2", "Normal", "GracefulShutdown", + "VirtualMachine/vm-batch-prod-01", + "VM stopped by scheduler after batch job completion"), + ("virt-staging", "Normal", "UserPaused", + "VirtualMachineInstance/vm-db-stg-02", + "VM paused by user request"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-1234", + "Vulnerability scan completed: 6 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-2345", + "Vulnerability scan completed: 7 affected VMs, 5 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-3456", + "Vulnerability scan completed: 8 affected VMs, 6 vulnerable"), + ("openshift-compliance", "Normal", "ScanCompleted", + "VulnerabilityReport/rhsa-2026-4567", + "Vulnerability scan completed: 5 affected VMs, 4 vulnerable"), + ("openshift-compliance", "Warning", "NoRemediationAvailable", + "VulnerabilityReport/rhsa-2026-5678", + "Advisory RHSA-2026:5678 has no vendor remediation — " + "compensating controls required for 4 vulnerable VMs"), +] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE BUILDERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _os_parts(os_str): + """Parse 'rhel-9.3' into (id, version, pretty).""" + parts = os_str.split("-", 1) + oid = parts[0] + ver = parts[1] if len(parts) > 1 else "" + major = ver.split(".")[0] if ver else "" + pretty = f"Red Hat Enterprise Linux {major} ({ver})" if oid == "rhel" else os_str + return oid, ver, pretty + + +def _uid(name): + return hashlib.md5(name.encode()).hexdigest()[:8] + "-0000-0000-0000-" + \ + hashlib.md5(name.encode()).hexdigest()[:12] + + +def _pod_hash(name): + return hashlib.md5(name.encode()).hexdigest()[:5] + + +def _firmware_uuid(name): + h = hashlib.sha256(name.encode()).hexdigest() + return f"{h[:8]}-{h[8:12]}-4{h[13:16]}-{h[16:20]}-{h[20:32]}" + + +def _firmware_serial(name): + h = hashlib.sha256((name + "-serial").encode()).hexdigest()[:12] + return f"sn-{h}" + + +def _build_vm(vm): + """Build a kubevirt.io/v1 VirtualMachine resource dict.""" + labels = {"kubevirt.io/domain": vm["name"], "vm.kubevirt.io/name": vm["name"]} + if vm["env"]: + labels["env"] = vm["env"] + labels.update(vm["labels"]) + + annotations = {"vm.kubevirt.io/os": vm["os"]} + adv_map = _VM_ADV.get(vm["name"]) + if adv_map: + annotations["security.openshift.io/vulnerabilities"] = json.dumps( + {a["id"]: a["status"] for a in adv_map}) + + is_running = vm["status"] in ("Running", "Paused") + conditions = [ + {"type": "Ready", "status": str(vm["ready"]), + "lastTransitionTime": CREATED}, + ] + agent_connected = True + for ct, cs, cm in vm["conds"]: + if ct == "AgentConnected": + agent_connected = False + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + else: + conditions.append({"type": ct, "status": cs, "message": cm, + "lastTransitionTime": CREATED}) + if agent_connected and is_running: + conditions.append({"type": "AgentConnected", "status": "True", + "lastTransitionTime": CREATED}) + + res = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "running": is_running, + "template": { + "metadata": {"labels": { + "kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"], + }}, + "spec": { + "domain": { + "cpu": {"cores": vm["cpu"], "sockets": 1, "threads": 1}, + "memory": {"guest": f"{vm['mem']}Gi"}, + "resources": { + "requests": {"cpu": str(vm["cpu"]), + "memory": f"{vm['mem']}Gi"}, + }, + "firmware": { + "uuid": _firmware_uuid(vm["name"]), + "serial": _firmware_serial(vm["name"]), + }, + }, + "volumes": [ + {"name": "rootdisk", + "persistentVolumeClaim": { + "claimName": f"{vm['name']}-rootdisk"}}, + ], + }, + }, + }, + "status": { + "printableStatus": vm["status"], + "ready": vm["ready"], + "created": True, + "conditions": conditions, + }, + } + if vm.get("pinned"): + res["spec"]["template"]["spec"]["nodeSelector"] = { + "kubernetes.io/hostname": vm["node"] + } + return res + + +def _build_vmi(vm): + """Build a kubevirt.io/v1 VirtualMachineInstance (only for running/paused VMs).""" + if vm["status"] not in ("Running", "Paused"): + return None + oid, ver, pretty = _os_parts(vm["os"]) + phase = "Running" if vm["status"] == "Running" else "Paused" + ip_hash = int(hashlib.md5(vm["name"].encode()).hexdigest()[:4], 16) + ip = f"10.244.{(ip_hash >> 8) & 0xFF}.{ip_hash & 0xFF}" + + conditions = [{"type": "Ready", "status": str(vm["ready"])}] + for ct, cs, cm in vm["conds"]: + conditions.append({"type": ct, "status": cs, "message": cm}) + + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": { + "name": vm["name"], + "namespace": vm["ns"], + "uid": _uid(vm["name"] + "-vmi"), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": vm["name"], "uid": _uid(vm["name"]), + }], + "creationTimestamp": CREATED, + }, + "status": { + "phase": phase, + "nodeName": vm["node"], + "guestOSInfo": {"id": oid, "version": ver, "prettyName": pretty}, + "interfaces": [{"ipAddress": ip, "name": "default"}], + "conditions": conditions, + "migrationMethod": "LiveMigration", + "activePods": {_uid(vm["name"] + "-pod"): vm["node"]}, + }, + } + + +def _build_node(n): + """Build a v1/Node resource dict.""" + labels = { + "kubernetes.io/hostname": n["name"], + "node-role.kubernetes.io/worker": "", + "topology.kubernetes.io/zone": n["zone"], + "node.kubernetes.io/instance-type": n["itype"], + } + if not n["unschedulable"]: + labels["kubevirt.io/schedulable"] = "true" + annotations = {} + if n["maint"]: + annotations["machine.openshift.io/maintenance"] = n["maint"] + + conditions = [{"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}] + if n["unschedulable"]: + conditions.append({"type": "MemoryPressure", "status": "False"}) + conditions.append({"type": "DiskPressure", "status": "False"}) + + cpu_str = str(n["cpu_cap"] // 1000) + mem_ki = n["mem_cap"] * 1024 + + res = { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": n["name"], + "uid": _uid(n["name"]), + "labels": labels, + "annotations": annotations, + "creationTimestamp": CREATED, + }, + "spec": { + "unschedulable": n["unschedulable"], + }, + "status": { + "conditions": conditions, + "capacity": { + "cpu": cpu_str, "memory": f"{mem_ki}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "allocatable": { + "cpu": f"{n['cpu_cap'] - 200}m", + "memory": f"{mem_ki - 1024}Ki", "pods": "250", + "devices.kubevirt.io/kvm": "1", + "devices.kubevirt.io/tun": "1", + "devices.kubevirt.io/vhost-net": "1", + }, + "nodeInfo": { + "kubeletVersion": K8S_VER, + "osImage": "Red Hat Enterprise Linux CoreOS 415.92.202402130034-0", + "containerRuntimeVersion": "cri-o://1.28.4", + "kernelVersion": "5.14.0-284.52.1.el9_2.x86_64", + "architecture": "amd64", + "operatingSystem": "linux", + }, + }, + } + if n["taints"]: + res["spec"]["taints"] = n["taints"] + return res + + +def _build_vuln_report(adv): + """Build a security.openshift.io/v1 VulnerabilityReport resource.""" + vuln_count = sum(1 for _, _, s in adv["affected"] if s == "Vulnerable") + rem_count = sum(1 for _, _, s in adv["affected"] if s == "Remediated") + return { + "apiVersion": "security.openshift.io/v1", + "kind": "VulnerabilityReport", + "metadata": { + "name": adv["name"], + "namespace": "openshift-compliance", + "uid": _uid(adv["name"]), + "labels": { + "advisory-id": adv["id"], + "severity": adv["severity"].lower(), + }, + "creationTimestamp": CREATED, + }, + "spec": { + "advisoryId": adv["id"], + "synopsis": adv["synopsis"], + "severity": adv["severity"], + "cvssScore": adv["cvss"], + "complianceImpact": adv["compliance"], + "remediationDeadlineDays": adv["deadline"], + "remediationAvailable": adv["remediation_available"], + "description": adv["description"], + "affectedWorkloads": [ + {"name": vn, "namespace": vns, "kind": "VirtualMachine", + "status": vs, "remediationAvailable": adv["remediation_available"]} + for vn, vns, vs in adv["affected"] + ], + }, + "status": { + "phase": "Completed", + "totalAffected": len(adv["affected"]), + "totalVulnerable": vuln_count, + "totalRemediated": rem_count, + "lastScanTime": NOW, + }, + } + + +def _build_ns(name, labels): + return { + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "uid": _uid(name), "labels": labels, + "creationTimestamp": CREATED}, + "status": {"phase": "Active"}, + } + + +_STORAGE_SIZES = { + "db": "100Gi", "web": "50Gi", "api": "50Gi", "cache": "30Gi", + "queue": "30Gi", "monitoring": "30Gi", "logging": "30Gi", +} + + +_RWO_VMS = {"vm-backup-prod-01", "vm-batch-prod-01", "vm-archive-01"} + +def _build_pvc(vm): + """Build a v1/PersistentVolumeClaim for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-pvc"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + "status": { + "phase": "Bound", + "capacity": {"storage": size}, + "accessModes": [access], + }, + } + + +def _build_datavolume(vm): + """Build a cdi.kubevirt.io/v1beta1 DataVolume for a VM's rootdisk.""" + app = vm["labels"].get("app", "") + size = _STORAGE_SIZES.get(app, "30Gi") + access = "ReadWriteOnce" if vm["name"] in _RWO_VMS else "ReadWriteMany" + return { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": f"{vm['name']}-rootdisk", + "namespace": vm["ns"], + "uid": _uid(f"{vm['name']}-dv"), + "labels": { + "vm.kubevirt.io/name": vm["name"], + "app.kubernetes.io/managed-by": "cdi-controller", + }, + "creationTimestamp": CREATED, + }, + "spec": { + "source": {"pvc": {"namespace": vm["ns"], + "name": f"{vm['name']}-rootdisk-source"}}, + "pvc": { + "accessModes": [access], + "resources": {"requests": {"storage": size}}, + "storageClassName": "ocs-storagecluster-ceph-rbd", + "volumeMode": "Block", + }, + }, + "status": { + "phase": "Succeeded", + "progress": "100.0%", + "conditions": [ + {"type": "Ready", "status": "True", + "lastTransitionTime": CREATED}, + {"type": "Bound", "status": "True", + "lastTransitionTime": CREATED}, + ], + }, + } + + +SNAPSHOTS = [ + { + "name": "vm-db-prod-01-backup-20260201", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-01T08:00:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260201"}, + ], + }, + { + "name": "vm-db-prod-01-backup-20260215", + "namespace": "virt-prod-dc2", + "vm_name": "vm-db-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-15T10:30:00Z", + "indications": ["Online", "GuestAgent"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-db01-root-20260215"}, + ], + }, + { + "name": "vm-web-prod-01-snap-20260220", + "namespace": "virt-prod-dc1", + "vm_name": "vm-web-prod-01", + "phase": "Succeeded", + "ready_to_use": True, + "creation": "2026-02-20T14:00:00Z", + "indications": ["Online"], + "volume_statuses": [ + {"name": "rootdisk", "volumeSnapshotName": "vsnap-web01-root-20260220"}, + ], + }, + { + "name": "vm-etl-prod-01-snap-failed", + "namespace": "virt-prod-dc1", + "vm_name": "vm-etl-prod-01", + "phase": "Failed", + "ready_to_use": False, + "creation": "2026-02-25T09:00:00Z", + "indications": [], + "volume_statuses": [], + "error": "VolumeSnapshot creation timed out for rootdisk", + }, +] + +RESTORES = [ + { + "name": "restore-vm-web-prod-01-20260220", + "namespace": "virt-prod-dc1", + "target_vm": "vm-web-prod-01", + "snapshot_name": "vm-web-prod-01-snap-20260220", + "complete": True, + "creation": "2026-02-22T16:00:00Z", + }, +] + +MIGRATIONS = [ + { + "name": "migration-vm-web-prod-03", + "namespace": "virt-prod-dc1", + "vmi_name": "vm-web-prod-03", + "phase": "Succeeded", + "source_node": "hv-prod-dc1-02", + "target_node": "hv-prod-dc1-01", + "creation": "2026-02-28T11:00:00Z", + }, +] + + +def _build_snapshot(snap): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineSnapshot resource.""" + res = { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineSnapshot", + "metadata": { + "name": snap["name"], + "namespace": snap["namespace"], + "uid": _uid(snap["name"]), + "labels": {"vm.kubevirt.io/name": snap["vm_name"]}, + "creationTimestamp": snap["creation"], + }, + "spec": { + "source": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": snap["vm_name"], + }, + }, + "status": { + "phase": snap["phase"], + "readyToUse": snap["ready_to_use"], + "creationTime": snap["creation"], + "indications": snap["indications"], + "volumeSnapshotStatus": snap["volume_statuses"], + }, + } + if snap.get("error"): + res["status"]["error"] = {"message": snap["error"]} + return res + + +def _build_restore(restore): + """Build a snapshot.kubevirt.io/v1beta1 VirtualMachineRestore resource.""" + return { + "apiVersion": "snapshot.kubevirt.io/v1beta1", + "kind": "VirtualMachineRestore", + "metadata": { + "name": restore["name"], + "namespace": restore["namespace"], + "uid": _uid(restore["name"]), + "creationTimestamp": restore["creation"], + }, + "spec": { + "target": { + "apiGroup": "kubevirt.io", + "kind": "VirtualMachine", + "name": restore["target_vm"], + }, + "virtualMachineSnapshotName": restore["snapshot_name"], + }, + "status": { + "complete": restore["complete"], + "restoreTime": restore["creation"], + }, + } + + +def _build_migration(mig): + """Build a kubevirt.io/v1 VirtualMachineInstanceMigration resource.""" + return { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstanceMigration", + "metadata": { + "name": mig["name"], + "namespace": mig["namespace"], + "uid": _uid(mig["name"]), + "creationTimestamp": mig["creation"], + }, + "spec": { + "vmiName": mig["vmi_name"], + }, + "status": { + "phase": mig["phase"], + "migrationState": { + "sourceNode": mig["source_node"], + "targetNode": mig["target_node"], + "completed": mig["phase"] == "Succeeded", + "startTimestamp": mig["creation"], + }, + }, + } + + +def _build_pod(vm): + """Build a virt-launcher Pod for a running/paused VM.""" + if vm["status"] not in ("Running", "Paused"): + return None + pod_name = f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}" + return { + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, "namespace": vm["ns"], + "uid": _uid(pod_name), + "labels": {"kubevirt.io/domain": vm["name"], + "vm.kubevirt.io/name": vm["name"]}, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "name": vm["name"], + }], + "creationTimestamp": CREATED, + }, + "spec": {"nodeName": vm["node"]}, + "status": { + "phase": "Running", + "containerStatuses": [{ + "name": "compute", "ready": True, + "state": {"running": {"startedAt": CREATED}}, + }], + }, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# FORMATTING HELPERS +# ═══════════════════════════════════════════════════════════════════════════ + +def _table(headers, rows): + """Format as a kubectl-style table with dynamic column widths.""" + widths = [len(h) for h in headers] + str_rows = [[str(c) for c in r] for r in rows] + for r in str_rows: + for i, c in enumerate(r): + if i < len(widths): + widths[i] = max(widths[i], len(c)) + lines = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers))] + for r in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(r))) + return "\n".join(lines) + + +def _to_yaml(resource): + return yaml.dump(resource, default_flow_style=False, sort_keys=False) + + +def _match_labels(labels, selector_str): + if not selector_str: + return True + for sel in selector_str.split(","): + sel = sel.strip() + if "!=" in sel: + k, v = sel.split("!=", 1) + if labels.get(k.strip()) == v.strip(): + return False + elif "=" in sel: + k, v = sel.split("=", 1) + if labels.get(k.strip()) != v.strip(): + return False + elif sel.startswith("!"): + if sel[1:] in labels: + return False + elif sel not in labels: + return False + return True + + +def _filter_by_ns(resources, namespace): + if namespace is None: + return resources + return [r for r in resources if r.get("metadata", {}).get("namespace") == namespace] + + +# ═══════════════════════════════════════════════════════════════════════════ +# RESOURCE DISPATCH +# ═══════════════════════════════════════════════════════════════════════════ + +def _all_resources(api_version, kind): + """Return (resources_list, table_headers, row_extractor, is_namespaced).""" + if api_version == "kubevirt.io/v1" and kind == "VirtualMachine": + resources = [_build_vm(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["printableStatus"], + str(s["ready"]), "30d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstance": + resources = [_build_vmi(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "PHASE", "IP", "NODENAME", "READY", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + ip = s.get("interfaces", [{}])[0].get("ipAddress", "") + return [m["namespace"], m["name"], s["phase"], ip, + s.get("nodeName", ""), str(s.get("conditions", [{}])[0].get("status", "")), "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Node": + resources = [_build_node(n) for n in NODES] + headers = ["NAME", "STATUS", "ROLES", "AGE", "VERSION"] + def row(r): + m = r["metadata"] + s = r.get("spec", {}) + status = "Ready,SchedulingDisabled" if s.get("unschedulable") else "Ready" + return [m["name"], status, "worker", "60d", K8S_VER] + return resources, headers, row, False + + if api_version == "v1" and kind == "Namespace": + resources = [_build_ns(n, lb) for n, lb in NAMESPACES] + headers = ["NAME", "STATUS", "AGE"] + def row(r): + return [r["metadata"]["name"], r["status"]["phase"], "60d"] + return resources, headers, row, False + + if api_version == "security.openshift.io/v1" and kind == "VulnerabilityReport": + resources = [_build_vuln_report(a) for a in ADVISORIES] + headers = ["NAMESPACE", "NAME", "SEVERITY", "CVSS", "AFFECTED", "VULNERABLE", "AGE"] + def row(r): + s = r["status"] + sp = r["spec"] + return [r["metadata"]["namespace"], r["metadata"]["name"], + sp["severity"], str(sp["cvssScore"]), + str(s["totalAffected"]), str(s["totalVulnerable"]), "5d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "Pod": + resources = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + def row(r): + m = r["metadata"] + return [m["namespace"], m["name"], "1/1", "Running", "0", "30d"] + return resources, headers, row, True + + if api_version == "v1" and kind == "PersistentVolumeClaim": + resources = [_build_pvc(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "STATUS", "VOLUME", "CAPACITY", "ACCESS MODES", "STORAGECLASS", "AGE"] + def row(r): + m = r["metadata"] + cap = r["status"].get("capacity", {}).get("storage", "") + sc = r["spec"].get("storageClassName", "") + am = ",".join(a.replace("ReadWriteMany", "RWX").replace("ReadWriteOnce", "RWO") + for a in r["spec"].get("accessModes", [])) + return [m["namespace"], m["name"], "Bound", _uid(m["name"]), cap, am, sc, "30d"] + return resources, headers, row, True + + if api_version == "cdi.kubevirt.io/v1beta1" and kind == "DataVolume": + resources = [_build_datavolume(vm) for vm in VMS] + headers = ["NAMESPACE", "NAME", "PHASE", "PROGRESS", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], s.get("progress", ""), "30d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineSnapshot": + resources = [_build_snapshot(s) for s in SNAPSHOTS] + headers = ["NAMESPACE", "NAME", "PHASE", "READY", "VM", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + vm_name = r["spec"]["source"]["name"] + return [m["namespace"], m["name"], s["phase"], + str(s["readyToUse"]), vm_name, "5d"] + return resources, headers, row, True + + if api_version == "snapshot.kubevirt.io/v1beta1" and kind == "VirtualMachineRestore": + resources = [_build_restore(r) for r in RESTORES] + headers = ["NAMESPACE", "NAME", "TARGET", "SNAPSHOT", "COMPLETE", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], + r["spec"]["target"]["name"], + r["spec"]["virtualMachineSnapshotName"], + str(s["complete"]), "3d"] + return resources, headers, row, True + + if api_version == "kubevirt.io/v1" and kind == "VirtualMachineInstanceMigration": + resources = [_build_migration(m) for m in MIGRATIONS] + headers = ["NAMESPACE", "NAME", "PHASE", "VMI", "AGE"] + def row(r): + m = r["metadata"] + s = r["status"] + return [m["namespace"], m["name"], s["phase"], + r["spec"]["vmiName"], "2d"] + return resources, headers, row, True + + return [], [], None, True + + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIG TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def configuration_view(minified: bool = True) -> str: + """Get the current Kubernetes configuration content as a kubeconfig YAML.""" + cfg = { + "apiVersion": "v1", "kind": "Config", + "current-context": CLUSTER, + "clusters": [{"name": CLUSTER, "cluster": {"server": API_URL}}], + "contexts": [{"name": CLUSTER, "context": { + "cluster": CLUSTER, "user": "admin", "namespace": "default"}}], + "users": [{"name": "admin", "user": { + "token": "[REDACTED]"}}], + } + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +@mcp.tool() +def configuration_contexts_list() -> str: + """List all available context names and associated server urls from the kubeconfig file.""" + return _table( + ["CURRENT", "NAME", "CLUSTER", "AUTHINFO", "NAMESPACE"], + [["*", CLUSTER, CLUSTER, "admin", "default"]]) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: RESOURCES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def resources_list( + apiVersion: str, + kind: str, + namespace: Optional[str] = None, + labelSelector: Optional[str] = None, + fieldSelector: Optional[str] = None, +) -> str: + """List Kubernetes resources by apiVersion and kind, optionally filtered by namespace and label selector.""" + resources, headers, row_fn, is_namespaced = _all_resources(apiVersion, kind) + if not resources and row_fn is None: + return f"error: the server doesn't have a resource type \"{kind}\"" + + if is_namespaced and namespace: + resources = _filter_by_ns(resources, namespace) + if labelSelector: + resources = [r for r in resources + if _match_labels(r.get("metadata", {}).get("labels", {}), + labelSelector)] + if fieldSelector: + for sel in fieldSelector.split(","): + if "=" in sel: + k, v = sel.split("=", 1) + k, v = k.strip(), v.strip() + if k == "status.printableStatus": + resources = [r for r in resources + if r.get("status", {}).get("printableStatus") == v] + elif k == "metadata.name": + resources = [r for r in resources + if r.get("metadata", {}).get("name") == v] + elif k == "spec.nodeName": + resources = [r for r in resources + if r.get("spec", {}).get("nodeName") == v or + r.get("status", {}).get("nodeName") == v or + r.get("spec", {}).get("template", {}).get("spec", {}) + .get("nodeSelector", {}).get("kubernetes.io/hostname") == v] + + if not resources: + ns_msg = f" in namespace \"{namespace}\"" if namespace else "" + return f"No resources found{ns_msg}." + + show_ns = is_namespaced and namespace is None + h = headers if show_ns else [h for h in headers if h != "NAMESPACE"] + rows = [] + for r in resources: + full_row = row_fn(r) + if show_ns: + rows.append(full_row) + else: + ns_idx = headers.index("NAMESPACE") if "NAMESPACE" in headers else -1 + rows.append([c for i, c in enumerate(full_row) if i != ns_idx]) + return _table(h, rows) + + +@mcp.tool() +def resources_get( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, +) -> str: + """Get a Kubernetes resource by apiVersion, kind, and name, returned as YAML.""" + resources, _, _, is_namespaced = _all_resources(apiVersion, kind) + for r in resources: + m = r.get("metadata", {}) + if m.get("name") != name: + continue + if is_namespaced and namespace and m.get("namespace") != namespace: + continue + return _to_yaml(r) + kind_lower = kind.lower() + "s" + return f'Error from server (NotFound): {kind_lower}.{apiVersion.split("/")[0]} "{name}" not found' + + +@mcp.tool() +def resources_create_or_update(resource: str) -> str: + """Create or update a Kubernetes resource (YAML or JSON).""" + try: + data = yaml.safe_load(resource) + name = data.get("metadata", {}).get("name", "unknown") + kind = data.get("kind", "unknown") + return f'{kind} "{name}" configured' + except Exception as e: + return f"Error: invalid resource definition: {e}" + + +@mcp.tool() +def resources_delete( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + gracePeriodSeconds: Optional[int] = None, +) -> str: + """Delete a Kubernetes resource.""" + return f'{kind} "{name}" deleted' + + +@mcp.tool() +def resources_scale( + apiVersion: str, + kind: str, + name: str, + namespace: Optional[str] = None, + scale: Optional[int] = None, +) -> str: + """Get or update the scale of a Kubernetes resource.""" + return f'Error: {kind} does not support scaling' + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: NAMESPACES, EVENTS, NODES +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def namespaces_list() -> str: + """List all Kubernetes namespaces in the current cluster.""" + headers = ["NAME", "STATUS", "AGE"] + rows = [[n, "Active", "60d"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def projects_list() -> str: + """List all OpenShift projects in the current cluster.""" + headers = ["NAME", "DISPLAY NAME", "STATUS"] + rows = [[n, "", "Active"] for n, _ in NAMESPACES] + return _table(headers, rows) + + +@mcp.tool() +def events_list(namespace: Optional[str] = None) -> str: + """List Kubernetes events (warnings, errors, state changes).""" + filtered = EVENTS + if namespace: + filtered = [e for e in filtered if e[0] == namespace] + if not filtered: + return "No events found." + headers = ["NAMESPACE", "LAST SEEN", "TYPE", "REASON", "OBJECT", "MESSAGE"] + rows = [] + for i, (ns, etype, reason, obj, msg) in enumerate(filtered): + last_seen = f"{(i + 1) * 5}m" + rows.append([ns, last_seen, etype, reason, obj, msg]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_top( + name: Optional[str] = None, + label_selector: Optional[str] = None, +) -> str: + """List node resource consumption (CPU and memory) from the Metrics Server.""" + nodes = NODES + if name: + nodes = [n for n in nodes if n["name"] == name] + if label_selector: + all_nodes = [_build_node(n) for n in nodes] + matched = [n for n, r in zip(nodes, all_nodes) + if _match_labels(r["metadata"]["labels"], label_selector)] + nodes = matched + if not nodes: + return "No metrics available for the requested node(s)." + + headers = ["NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"] + rows = [] + for n in nodes: + cpu_pct = round(n["cpu_use"] / n["cpu_cap"] * 100) + mem_pct = round(n["mem_use"] / n["mem_cap"] * 100) + rows.append([n["name"], f"{n['cpu_use']}m", f"{cpu_pct}%", + f"{n['mem_use']}Mi", f"{mem_pct}%"]) + return _table(headers, rows) + + +@mcp.tool() +def nodes_stats_summary(name: str) -> str: + """Get detailed resource usage statistics from a node via the kubelet Summary API.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + + cpu_nano = node["cpu_use"] * 1_000_000 + mem_bytes = node["mem_use"] * 1024 * 1024 + mem_avail = (node["mem_cap"] - node["mem_use"]) * 1024 * 1024 + + vm_pods = [vm for vm in VMS + if vm["node"] == name and vm["status"] in ("Running", "Paused")] + pod_stats = [] + for vm in vm_pods: + pod_stats.append({ + "podRef": {"name": f"virt-launcher-{vm['name']}-{_pod_hash(vm['name'])}", + "namespace": vm["ns"]}, + "cpu": {"usageNanoCores": vm["cpu"] * 250_000_000}, + "memory": {"usageBytes": vm["mem"] * 512 * 1024 * 1024, + "workingSetBytes": vm["mem"] * 400 * 1024 * 1024}, + }) + + summary = { + "node": { + "nodeName": name, + "cpu": {"usageNanoCores": cpu_nano, + "usageCoreNanoSeconds": cpu_nano * 3600}, + "memory": {"availableBytes": mem_avail, + "usageBytes": mem_bytes, + "workingSetBytes": int(mem_bytes * 0.95)}, + "fs": {"availableBytes": 200_000_000_000, + "capacityBytes": 500_000_000_000, + "usedBytes": 300_000_000_000}, + "network": { + "interfaces": [{ + "name": "eth0", + "rxBytes": 1_500_000_000_000, + "txBytes": 800_000_000_000, + }], + }, + }, + "pods": pod_stats, + } + return json.dumps(summary, indent=2) + + +@mcp.tool() +def nodes_log(name: str, query: str, tailLines: int = 100) -> str: + """Get logs from a Kubernetes node.""" + node = next((n for n in NODES if n["name"] == name), None) + if not node: + return f'Error: node "{name}" not found' + return (f"-- Logs begin for {name} ({query}) --\n" + f"Mar 02 12:00:00 {name} kubelet[1234]: I0302 12:00:00.000000 " + f"node_status.go:123] Node {name} status: Ready\n" + f"-- End of logs --") + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE TOOLSET: PODS +# ═══════════════════════════════════════════════════════════════════════════ + +def _pod_list_filtered(namespace=None, fieldSelector=None, labelSelector=None): + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + if namespace: + pods = _filter_by_ns(pods, namespace) + if labelSelector: + pods = [p for p in pods + if _match_labels(p["metadata"]["labels"], labelSelector)] + return pods + + +@mcp.tool() +def pods_list( + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the cluster from all namespaces.""" + pods = _pod_list_filtered(None, fieldSelector, labelSelector) + if not pods: + return "No pods found." + headers = ["NAMESPACE", "NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["namespace"], p["metadata"]["name"], + "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_list_in_namespace( + namespace: str, + fieldSelector: Optional[str] = None, + labelSelector: Optional[str] = None, +) -> str: + """List all pods in the specified namespace.""" + pods = _pod_list_filtered(namespace, fieldSelector, labelSelector) + if not pods: + return f'No pods found in namespace "{namespace}".' + headers = ["NAME", "READY", "STATUS", "RESTARTS", "AGE"] + rows = [[p["metadata"]["name"], "1/1", "Running", "0", "30d"] for p in pods] + return _table(headers, rows) + + +@mcp.tool() +def pods_get(name: str, namespace: Optional[str] = None) -> str: + """Get a Pod by name, returned as YAML.""" + pods = [_build_pod(vm) for vm in VMS if vm["status"] in ("Running", "Paused")] + for p in pods: + if p["metadata"]["name"] == name: + if namespace and p["metadata"]["namespace"] != namespace: + continue + return _to_yaml(p) + return f'Error from server (NotFound): pods "{name}" not found' + + +@mcp.tool() +def pods_delete(name: str, namespace: Optional[str] = None) -> str: + """Delete a Pod by name.""" + return f'pod "{name}" deleted' + + +@mcp.tool() +def pods_log( + name: str, + namespace: Optional[str] = None, + container: Optional[str] = None, + tail: int = 100, + previous: bool = False, +) -> str: + """Get the logs of a Pod.""" + vm_name = name.replace("virt-launcher-", "").rsplit("-", 1)[0] + vm = next((v for v in VMS if v["name"] == vm_name), None) + if not vm: + return f'Error from server (NotFound): pods "{name}" not found' + return ( + f'{{"component":"virt-launcher","level":"info","msg":"Configured with ' + f'VM {vm["name"]}","timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-launcher","level":"info","msg":"Domain started",' + f'"timestamp":"{CREATED}"}}\n' + f'{{"component":"virt-handler","level":"info","msg":"VM is running on ' + f'node {vm["node"]}","timestamp":"{CREATED}"}}' + ) + + +@mcp.tool() +def pods_exec( + name: str, + command: list, + namespace: Optional[str] = None, + container: Optional[str] = None, +) -> str: + """Execute a command in a Pod.""" + cmd = " ".join(command) + return f"command '{cmd}' executed successfully" + + +@mcp.tool() +def pods_run( + image: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + port: Optional[int] = None, +) -> str: + """Run a Pod with the provided container image.""" + pod_name = name or "run-" + _pod_hash(image) + return f'pod/{pod_name} created' + + +@mcp.tool() +def pods_top( + name: Optional[str] = None, + namespace: Optional[str] = None, + all_namespaces: bool = False, + label_selector: Optional[str] = None, +) -> str: + """List pod resource consumption from the Metrics Server.""" + pods_data = [(vm, _build_pod(vm)) for vm in VMS + if vm["status"] in ("Running", "Paused")] + if namespace and not all_namespaces: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["namespace"] == namespace] + if name: + pods_data = [(vm, p) for vm, p in pods_data + if p["metadata"]["name"] == name] + + if not pods_data: + return "No metrics available." + + show_ns = all_namespaces or (namespace is None and name is None) + headers = (["NAMESPACE"] if show_ns else []) + ["NAME", "CPU(cores)", "MEMORY(bytes)"] + rows = [] + for vm, p in pods_data: + cpu_m = f"{vm['cpu'] * 250}m" + mem_mi = f"{vm['mem'] * 512}Mi" + row = ([p["metadata"]["namespace"]] if show_ns else []) + \ + [p["metadata"]["name"], cpu_m, mem_mi] + rows.append(row) + return _table(headers, rows) + + +# ═══════════════════════════════════════════════════════════════════════════ +# KUBEVIRT TOOLSET +# ═══════════════════════════════════════════════════════════════════════════ + +@mcp.tool() +def vm_lifecycle(name: str, namespace: str, action: str) -> str: + """Manage VirtualMachine lifecycle: start, stop, or restart a VM.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + if action not in ("start", "stop", "restart"): + return f'Error: invalid action "{action}". Must be start, stop, or restart' + return f'VirtualMachine "{name}" was scheduled to {action}' + + +@mcp.tool() +def vm_create( + name: str, + namespace: str, + workload: str = "fedora", + autostart: bool = False, + instancetype: Optional[str] = None, + preference: Optional[str] = None, + size: Optional[str] = None, + storage: Optional[str] = None, + performance: Optional[str] = None, + networks: Optional[list] = None, +) -> str: + """Create a VirtualMachine in the cluster.""" + return f'VirtualMachine "{name}" created in namespace "{namespace}"' + + +@mcp.tool() +def vm_clone(name: str, namespace: str, targetName: str) -> str: + """Clone a KubeVirt VirtualMachine.""" + vm = next((v for v in VMS if v["name"] == name and v["ns"] == namespace), None) + if not vm: + return (f'Error from server (NotFound): virtualmachines.kubevirt.io ' + f'"{name}" not found in namespace "{namespace}"') + return f'VirtualMachineClone "{name}-to-{targetName}" created' + + +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + mcp.run() diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-restore/instruction.md b/evaluation/without_skills/rh-virt__vm-snapshot-restore/instruction.md new file mode 100644 index 00000000..d28e79fd --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-restore/instruction.md @@ -0,0 +1,12 @@ +# VM Snapshot Restore Task + +You are an OpenShift Virtualization administrator. Restore VM `production-db` from snapshot `production-db-backup-20240301` in namespace `prod-vms`. + +## Requirements +- Verify snapshot is ready and valid +- Address VM state requirements for restore +- Include safeguards (this is a destructive operation) + +Use MCP tools to examine the cluster. Document your methodology, findings, and restore plan in `/root/report.md`. + +If reference documentation or skills are available in this environment, consult them before beginning work. diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-restore/solution/solve.sh b/evaluation/without_skills/rh-virt__vm-snapshot-restore/solution/solve.sh new file mode 100644 index 00000000..7bb6e343 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-restore/solution/solve.sh @@ -0,0 +1,39 @@ +#!/bin/bash +cat > /root/report.md << 'REPORT_EOF' +# VM Snapshot Restore Plan + +## Restore production-db from production-db-backup-20240301 + +### Prerequisites +1. Verify snapshot exists and `status.phase == "Succeeded"` and `status.readyToUse == true` +2. **VM must be stopped** before restore — use `vm_lifecycle` action=stop +3. Verify no active VirtualMachineRestore in progress + +### VirtualMachineRestore YAML +```yaml +apiVersion: snapshot.kubevirt.io/v1beta1 +kind: VirtualMachineRestore +metadata: + name: restore-production-db-20240301 + namespace: prod-vms +spec: + target: + apiGroup: kubevirt.io + kind: VirtualMachine + name: production-db + virtualMachineSnapshotName: production-db-backup-20240301 +``` + +### Procedure +1. Stop VM production-db +2. Verify snapshot is ready (readyToUse: true) +3. **Typed confirmation**: Type snapshot name for safety +4. Create VirtualMachineRestore resource +5. Monitor restore progress (poll status.phase) +6. Start VM after restore completes + +### Warning +- Restore **overwrites** current VM state with snapshot state +- All changes since snapshot will be lost + +REPORT_EOF diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-restore/task.toml b/evaluation/without_skills/rh-virt__vm-snapshot-restore/task.toml new file mode 100644 index 00000000..bf15ebed --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-restore/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +id = "rh-virt__vm-snapshot-restore" +name = "rh-virt VM Snapshot Restore Skill Evaluation" +difficulty = "medium" +category = "per-skill-eval" +tags = ["rh-virt", "vm-snapshot-restore", "per-skill-eval"] + +[verifier] +timeout_sec = 900.0 + +[verifier.env] +ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL}" +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +LLM_JUDGE_MODEL = "claude-sonnet-4-5" + +[agent] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +gpus = 0 +memory = "5.5G" +storage = "10G" diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-restore/tests/llm_judge.py b/evaluation/without_skills/rh-virt__vm-snapshot-restore/tests/llm_judge.py new file mode 100644 index 00000000..0a348593 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-restore/tests/llm_judge.py @@ -0,0 +1,93 @@ +import json +import os +import sys +import time +from pathlib import Path + +try: + from anthropic import Anthropic +except ImportError: + print("ERROR: anthropic package not installed") + sys.exit(1) + +CRITERIA = [ + {"id": "vm_stopped_prerequisite", "file": "/root/report.md", "question": "Does the report require the VM to be stopped before restore and explain this is enforced by the platform?", "reference": "A skilled report enforces stop-before-restore. An unskilled report doesn't mention this prerequisite."}, + {"id": "restore_cr_definition", "file": "/root/report.md", "question": "Does the report define a VirtualMachineRestore CR with virtualMachineSnapshotName reference?", "reference": "A skilled report creates proper VirtualMachineRestore resource. An unskilled report doesn't know the restore API."}, + {"id": "destructive_warning_and_verification", "file": "/root/report.md", "question": "Does the report warn about data loss (changes since snapshot) and verify restore completion via status.complete?", "reference": "A skilled report warns about destructive nature and verifies completion. An unskilled report restores without warnings."} +] + +SYSTEM_PROMPT = ( + "You are an evaluator for a cloud operations benchmark. You will be given a " + "file produced by an AI agent, a yes/no question about its contents, and a " + "REFERENCE ANSWER that describes what a correct, skilled response looks like.\n\n" + "Rules:\n" + "- Answer ONLY with a JSON object: {\"pass\": true} or {\"pass\": false}\n" + "- Base your answer strictly on what is written in the file content\n" + "- Do not infer or assume knowledge the agent did not demonstrate\n" + "- Use the REFERENCE ANSWER to calibrate what counts as a pass\n" + "- Accept different phrasings that convey the SAME concept\n" + "- Do NOT use your own general knowledge to fill gaps" +) + + +def judge_criterion(client, model, criterion): + filepath = criterion["file"] + if not Path(filepath).exists(): + return {"id": criterion["id"], "pass": False, "reason": "file not found"} + content = Path(filepath).read_text() + if len(content) > 50000: + content = content[:50000] + "\n... (truncated)" + reference = criterion.get("reference", "") + ref_block = f"\n\n## Reference Answer\n{reference}" if reference else "" + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.messages.create( + model=model, max_tokens=64, system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": ( + f"## File: {filepath}\n\n```\n{content}\n```\n\n" + f"## Question\n{criterion['question']}{ref_block}" + )}], + ) + text = response.content[0].text.strip() + if "{" in text: + text = text[text.index("{"):text.rindex("}") + 1] + result = json.loads(text) + return {"id": criterion["id"], "pass": bool(result.get("pass", False))} + except Exception as e: + if attempt < max_retries - 1: + time.sleep(5 * (attempt + 1)) + else: + return {"id": criterion["id"], "pass": False, "reason": str(e)} + + +def main(): + api_key = os.getenv("ANTHROPIC_API_KEY") + base_url = os.getenv("ANTHROPIC_BASE_URL") + model = os.getenv("LLM_JUDGE_MODEL", "claude-haiku-4-5") + if not api_key: + print("ERROR: ANTHROPIC_API_KEY not set, skipping LLM judge") + json.dump({"criteria": [], "passed": 0, "total": 0, "score": 0.0}, + open("/logs/verifier/llm_judge.json", "w"), indent=2) + return + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = Anthropic(**client_kwargs) + results = [] + print(f"=== LLM Judge: evaluating {len(CRITERIA)} criteria with {model} ===") + for criterion in CRITERIA: + print(f" Evaluating: {criterion['id']} ...", end=" ", flush=True) + result = judge_criterion(client, model, criterion) + results.append(result) + print("PASS" if result["pass"] else "FAIL") + passed = sum(1 for r in results if r["pass"]) + total = len(results) + score = round(passed / total, 4) if total > 0 else 0.0 + print(f"=== LLM Judge: {passed}/{total} criteria passed (score={score}) ===") + Path("/logs/verifier/llm_judge.json").write_text(json.dumps( + {"criteria": results, "passed": passed, "total": total, "score": score}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-restore/tests/test.sh b/evaluation/without_skills/rh-virt__vm-snapshot-restore/tests/test.sh new file mode 100644 index 00000000..fb1242b7 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-restore/tests/test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +pip3 install --break-system-packages \ + pytest==8.4.1 \ + pytest-json-ctrf==0.3.5 \ + anthropic>=0.75.0 + +TEST_FILE=$(find / -name "test_outputs.py" 2>/dev/null | head -1) +JUDGE_FILE=$(find / -name "llm_judge.py" 2>/dev/null | head -1) + +if [ -z "$TEST_FILE" ]; then + echo "ERROR: Could not find test_outputs.py" + echo "0" > /logs/verifier/reward.txt + exit 1 +fi + +echo "=== Files created by agent in /root ===" +ls -la /root/*.md 2>/dev/null || echo "No markdown files found" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 1: Deterministic Tests (pytest)" +echo "════════════════════════════════════════════" + +pytest "$TEST_FILE" \ + --ctrf=/logs/verifier/ctrf.json \ + -v 2>&1 + +pytest_exit=$? + +pytest_passed=0 +pytest_total=0 +if [ -f /logs/verifier/ctrf.json ]; then + pytest_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['passed'])" 2>/dev/null) + pytest_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/ctrf.json')); print(d['results']['summary']['tests'])" 2>/dev/null) +fi +echo "=== Pytest: ${pytest_passed}/${pytest_total} passed ===" + +echo "" +echo "════════════════════════════════════════════" +echo " Phase 2: LLM Judge (skill evaluation)" +echo "════════════════════════════════════════════" + +llm_passed=0 +llm_total=0 + +if [ -n "$JUDGE_FILE" ] && [ -n "$ANTHROPIC_API_KEY" ]; then + timeout 180 python3 "$JUDGE_FILE" + + if [ -f /logs/verifier/llm_judge.json ]; then + llm_passed=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['passed'])" 2>/dev/null) + llm_total=$(python3 -c "import json; d=json.load(open('/logs/verifier/llm_judge.json')); print(d['total'])" 2>/dev/null) + fi + echo "=== LLM Judge: ${llm_passed}/${llm_total} passed ===" +else + if [ -z "$JUDGE_FILE" ]; then + echo "WARNING: llm_judge.py not found, skipping LLM evaluation" + fi + if [ -z "$ANTHROPIC_API_KEY" ]; then + echo "WARNING: ANTHROPIC_API_KEY not set, skipping LLM evaluation" + fi +fi + +echo "" +echo "════════════════════════════════════════════" +echo " Combined Score" +echo "════════════════════════════════════════════" + +reward=$(python3 -c " +pytest_p = int('${pytest_passed}' or 0) +pytest_t = int('${pytest_total}' or 0) +llm_p = int('${llm_passed}' or 0) +llm_t = int('${llm_total}' or 0) +total_p = pytest_p + llm_p +total_t = pytest_t + llm_t +reward = round(total_p / total_t, 4) if total_t > 0 else 0.0 +print(reward) +" 2>/dev/null) + +echo "$reward" > /logs/verifier/reward.txt +echo "=== Final Reward: $reward (pytest=${pytest_passed}/${pytest_total} + llm=${llm_passed}/${llm_total}) ===" + +cp /root/*.md /logs/verifier/ 2>/dev/null || true + +exit 0 diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-restore/tests/test_outputs.py b/evaluation/without_skills/rh-virt__vm-snapshot-restore/tests/test_outputs.py new file mode 100644 index 00000000..e02b5cf9 --- /dev/null +++ b/evaluation/without_skills/rh-virt__vm-snapshot-restore/tests/test_outputs.py @@ -0,0 +1,71 @@ +""" +Tests for rh-virt__vm-snapshot-restore per-skill evaluation. +Baseline tests: report structure. +Skill-dependent tests: conceptual checks (no exact tool/field name matching). +""" +import os +import pytest + +REPORT = "/root/report.md" + + +def read_report(): + if not os.path.exists(REPORT): + pytest.fail(f"Required file not found: {REPORT}") + with open(REPORT) as f: + return f.read() + +class TestBaseline: + def test_report_exists(self): + assert os.path.exists(REPORT), "report.md must exist" + + def test_mentions_restore(self): + content = read_report().lower() + assert "restor" in content, "report should discuss restore operation" + + def test_mentions_snapshot(self): + content = read_report().lower() + assert "snapshot" in content or "backup" in content, "report should mention the snapshot" + + +class TestSkillDependent: + def test_vm_stopped_prerequisite(self): + """Skill: VM must be stopped before restore; stop-and-restore option.""" + c = read_report().lower() + assert any(t in c for t in ["stop before restor", "must be stopped", "stop-and-restore", "vm must be stopped", "halt"]) and ( + "stop" in c and "restor" in c + ), ( + "should require VM stopped before restore" + ) + + def test_destructive_warning(self): + """Skill: Data loss warning; changes since snapshot will be lost.""" + c = read_report().lower() + assert any(t in c for t in ["data loss", "changes since", "will be lost", "overwrite", "destructive", "replace current", "cannot recover"]), ( + "should warn about data loss from restore" + ) + + def test_restore_cr(self): + """Skill: VirtualMachineRestore CR with target and snapshot reference.""" + c = read_report().lower() + assert "virtualmachinerestore" in c and any(t in c for t in ["target", "virtualmachinesnapshotname", "spec"]), ( + "should define VirtualMachineRestore resource" + ) + + def test_post_restore_verification(self): + """Skill: Verify restore complete; status.complete; start VM after.""" + c = read_report().lower() + assert any(t in c for t in ["status.complete", "restore complete", "post-restore", "after restore", "start vm", "start the vm"]) and ( + "restor" in c or "complete" in c or "start" in c + ), ( + "should include post-restore verification or start step" + ) + + def test_typed_confirmation(self): + """Skill: Typed snapshot name confirmation before restore.""" + c = read_report().lower() + assert any(t in c for t in ["type", "typed", "exact name", "to confirm", "snapshot name"]) and ( + "confirm" in c or "type" in c + ), ( + "should require typed snapshot name confirmation" + ) From d31d0c4e35d25f40c48e9a6b691af90124248d67 Mon Sep 17 00:00:00 2001 From: gziv Date: Tue, 24 Mar 2026 09:27:01 +0200 Subject: [PATCH 2/2] Cleaned up environment folders by removing docs/, skills/, and scripts/ directories from both with_skills/ and without_skills/ evaluation environments. These folders are not needed in the evaluation setup. --- .../environment/docs/multi-cluster-auth.md | 248 --- .../scripts/cluster-report/aggregate.py | 601 ------- .../scripts/cluster-report/assemble.py | 110 -- .../cluster-report/build-kubeconfig.py | 446 ----- .../cluster-report/cluster-reporter-rbac.yaml | 72 - .../scripts/cluster-report/test_aggregate.py | 863 ---------- .../scripts/cluster-report/test_assemble.py | 490 ------ .../skills/cluster-report/SKILL.md | 387 ----- .../environment/docs/common-issues.md | 84 - .../environment/docs/examples/model-deploy.md | 160 -- .../environment/docs/examples/nim-setup.md | 115 -- .../environment/docs/live-doc-lookup.md | 106 -- .../docs/references/known-model-profiles.md | 83 - .../docs/references/supported-runtimes.md | 104 -- .../environment/docs/skill-conventions.md | 85 - .../skills/ai-observability/SKILL.md | 420 ----- .../skills/references/common-issues.md | 84 - .../skills/references/live-doc-lookup.md | 106 -- .../skills/references/skill-conventions.md | 85 - .../environment/docs/common-issues.md | 84 - .../environment/docs/examples/model-deploy.md | 160 -- .../environment/docs/examples/nim-setup.md | 115 -- .../environment/docs/live-doc-lookup.md | 106 -- .../docs/references/known-model-profiles.md | 83 - .../docs/references/supported-runtimes.md | 104 -- .../environment/docs/skill-conventions.md | 85 - .../skills/debug-inference/SKILL.md | 348 ---- .../skills/references/common-issues.md | 84 - .../skills/references/live-doc-lookup.md | 106 -- .../skills/references/skill-conventions.md | 85 - .../environment/docs/common-issues.md | 84 - .../environment/docs/examples/model-deploy.md | 160 -- .../environment/docs/examples/nim-setup.md | 115 -- .../environment/docs/live-doc-lookup.md | 106 -- .../docs/references/known-model-profiles.md | 83 - .../docs/references/supported-runtimes.md | 104 -- .../environment/docs/skill-conventions.md | 85 - .../skills/ds-project-setup/SKILL.md | 336 ---- .../skills/references/common-issues.md | 84 - .../skills/references/live-doc-lookup.md | 106 -- .../skills/references/skill-conventions.md | 85 - .../environment/docs/common-issues.md | 84 - .../environment/docs/examples/model-deploy.md | 160 -- .../environment/docs/examples/nim-setup.md | 115 -- .../environment/docs/live-doc-lookup.md | 106 -- .../docs/references/known-model-profiles.md | 83 - .../docs/references/supported-runtimes.md | 104 -- .../environment/docs/skill-conventions.md | 85 - .../environment/skills/model-deploy/SKILL.md | 382 ----- .../model-deploy-preflight-checklist.md | 64 - .../skills/references/common-issues.md | 84 - .../skills/references/live-doc-lookup.md | 106 -- .../skills/references/skill-conventions.md | 85 - .../environment/docs/common-issues.md | 84 - .../environment/docs/examples/model-deploy.md | 160 -- .../environment/docs/examples/nim-setup.md | 115 -- .../environment/docs/live-doc-lookup.md | 106 -- .../docs/references/known-model-profiles.md | 83 - .../docs/references/supported-runtimes.md | 104 -- .../environment/docs/skill-conventions.md | 85 - .../environment/skills/nim-setup/SKILL.md | 370 ---- .../skills/references/common-issues.md | 84 - .../skills/references/live-doc-lookup.md | 106 -- .../skills/references/skill-conventions.md | 85 - .../environment/docs/common-issues.md | 84 - .../environment/docs/examples/model-deploy.md | 160 -- .../environment/docs/examples/nim-setup.md | 115 -- .../environment/docs/live-doc-lookup.md | 106 -- .../docs/references/known-model-profiles.md | 83 - .../docs/references/supported-runtimes.md | 104 -- .../environment/docs/skill-conventions.md | 85 - .../skills/references/common-issues.md | 84 - .../skills/references/live-doc-lookup.md | 106 -- .../skills/references/skill-conventions.md | 85 - .../skills/serving-runtime-config/SKILL.md | 278 --- .../environment/docs/common-issues.md | 84 - .../environment/docs/examples/model-deploy.md | 160 -- .../environment/docs/examples/nim-setup.md | 115 -- .../environment/docs/live-doc-lookup.md | 106 -- .../docs/references/known-model-profiles.md | 83 - .../docs/references/supported-runtimes.md | 104 -- .../environment/docs/skill-conventions.md | 85 - .../skills/references/common-issues.md | 84 - .../skills/references/live-doc-lookup.md | 106 -- .../skills/references/skill-conventions.md | 85 - .../skills/workbench-manage/SKILL.md | 396 ----- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../skills/containerize-deploy/SKILL.md | 477 ------ .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/skills/debug-build/SKILL.md | 315 ---- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../skills/debug-container/SKILL.md | 344 ---- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/skills/debug-network/SKILL.md | 331 ---- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../skills/debug-pipeline/SKILL.md | 306 ---- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/skills/debug-pod/SKILL.md | 260 --- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/skills/debug-rhel/SKILL.md | 455 ----- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/skills/deploy/SKILL.md | 277 --- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../skills/detect-project/SKILL.md | 277 --- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/skills/helm-deploy/SKILL.md | 356 ---- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../skills/recommend-image/SKILL.md | 282 ---- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/skills/rhel-deploy/SKILL.md | 482 ------ .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/skills/s2i-build/SKILL.md | 391 ----- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../skills/validate-environment/SKILL.md | 232 --- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../environment/skills/cve-impact/SKILL.md | 409 ----- .../cve-impact/flows/01-account-cves.md | 92 - .../cve-impact/flows/02-system-all-cves.md | 89 - .../flows/03-system-remediatable-cves.md | 96 -- .../references/01-cve-response-parser.py | 225 --- .../references/02-cve-parsing-guide.md | 147 -- .../references/03-output-templates.md | 39 - .../cve-impact/references/04-examples.md | 37 - .../references/05-error-handling.md | 24 - .../skills/mcp-lightspeed-validator/SKILL.md | 61 - .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../skills/cve-validation/SKILL.md | 340 ---- .../references/01-remediation-indicators.md | 66 - .../references/03-output-template.md | 36 - .../cve-validation/references/04-examples.md | 35 - .../references/05-error-handling.md | 37 - .../skills/mcp-lightspeed-validator/SKILL.md | 61 - .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../skills/execution-summary/SKILL.md | 357 ---- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../skills/fleet-inventory/SKILL.md | 254 --- .../references/01-parameter-reference.md | 49 - .../references/03-output-templates.md | 80 - .../fleet-inventory/references/04-examples.md | 32 - .../references/05-error-handling.md | 45 - .../skills/mcp-lightspeed-validator/SKILL.md | 61 - .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../skills/job-template-creator/SKILL.md | 321 ---- .../references/01-git-setup.md | 25 - .../references/02-web-ui-form.md | 24 - .../references/03-output-template.md | 20 - .../references/04-examples.md | 19 - .../skills/mcp-aap-validator/SKILL.md | 66 - .../skills/playbook-executor/SKILL.md | 499 ------ .../01-execution-report-templates.md | 168 -- .../references/02-error-handling-guide.md | 108 -- .../references/03-workflow-examples.md | 119 -- .../04-dry-run-display-templates.md | 93 - .../references/05-git-flow-prompts.md | 97 -- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../skills/job-template-creator/SKILL.md | 321 ---- .../references/01-git-setup.md | 25 - .../references/02-web-ui-form.md | 24 - .../references/03-output-template.md | 20 - .../references/04-examples.md | 19 - .../SKILL.md | 414 ----- .../skills/mcp-aap-validator/SKILL.md | 66 - .../skills/playbook-executor/SKILL.md | 499 ------ .../01-execution-report-templates.md | 168 -- .../references/02-error-handling-guide.md | 108 -- .../references/03-workflow-examples.md | 119 -- .../04-dry-run-display-templates.md | 93 - .../references/05-git-flow-prompts.md | 97 -- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../skills/mcp-aap-validator/SKILL.md | 66 - .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../skills/mcp-lightspeed-validator/SKILL.md | 61 - .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../skills/mcp-aap-validator/SKILL.md | 66 - .../skills/playbook-executor/SKILL.md | 499 ------ .../01-execution-report-templates.md | 168 -- .../references/02-error-handling-guide.md | 108 -- .../references/03-workflow-examples.md | 119 -- .../04-dry-run-display-templates.md | 93 - .../references/05-git-flow-prompts.md | 97 -- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../skills/playbook-generator/SKILL.md | 377 ----- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../skills/remediation-verifier/SKILL.md | 399 ----- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../skills/cve-validation/SKILL.md | 340 ---- .../references/01-remediation-indicators.md | 66 - .../references/03-output-template.md | 36 - .../cve-validation/references/04-examples.md | 35 - .../references/05-error-handling.md | 37 - .../environment/skills/remediation/SKILL.md | 279 --- .../01-remediation-plan-template.md | 85 - .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../skills/system-context/SKILL.md | 488 ------ .../.ai-index/semantic-index.json | 148 -- .../environment/docs/troubleshooting/INDEX.md | 332 ---- .../docs/troubleshooting/lifecycle-errors.md | 869 ---------- .../docs/troubleshooting/network-errors.md | 429 ----- .../docs/troubleshooting/runtime-errors.md | 616 ------- .../docs/troubleshooting/scheduling-errors.md | 417 ----- .../docs/troubleshooting/storage-errors.md | 1011 ----------- .../environment/skills/vm-clone/SKILL.md | 456 ----- .../.ai-index/semantic-index.json | 148 -- .../environment/docs/troubleshooting/INDEX.md | 332 ---- .../docs/troubleshooting/lifecycle-errors.md | 869 ---------- .../docs/troubleshooting/network-errors.md | 429 ----- .../docs/troubleshooting/runtime-errors.md | 616 ------- .../docs/troubleshooting/scheduling-errors.md | 417 ----- .../docs/troubleshooting/storage-errors.md | 1011 ----------- .../environment/skills/vm-create/SKILL.md | 403 ----- .../.ai-index/semantic-index.json | 148 -- .../environment/docs/troubleshooting/INDEX.md | 332 ---- .../docs/troubleshooting/lifecycle-errors.md | 869 ---------- .../docs/troubleshooting/network-errors.md | 429 ----- .../docs/troubleshooting/runtime-errors.md | 616 ------- .../docs/troubleshooting/scheduling-errors.md | 417 ----- .../docs/troubleshooting/storage-errors.md | 1011 ----------- .../environment/skills/vm-delete/SKILL.md | 403 ----- .../.ai-index/semantic-index.json | 148 -- .../environment/docs/troubleshooting/INDEX.md | 332 ---- .../docs/troubleshooting/lifecycle-errors.md | 869 ---------- .../docs/troubleshooting/network-errors.md | 429 ----- .../docs/troubleshooting/runtime-errors.md | 616 ------- .../docs/troubleshooting/scheduling-errors.md | 417 ----- .../docs/troubleshooting/storage-errors.md | 1011 ----------- .../environment/skills/vm-inventory/SKILL.md | 390 ----- .../.ai-index/semantic-index.json | 148 -- .../environment/docs/troubleshooting/INDEX.md | 332 ---- .../docs/troubleshooting/lifecycle-errors.md | 869 ---------- .../docs/troubleshooting/network-errors.md | 429 ----- .../docs/troubleshooting/runtime-errors.md | 616 ------- .../docs/troubleshooting/scheduling-errors.md | 417 ----- .../docs/troubleshooting/storage-errors.md | 1011 ----------- .../skills/vm-lifecycle-manager/SKILL.md | 308 ---- .../.ai-index/semantic-index.json | 148 -- .../environment/docs/troubleshooting/INDEX.md | 332 ---- .../docs/troubleshooting/lifecycle-errors.md | 869 ---------- .../docs/troubleshooting/network-errors.md | 429 ----- .../docs/troubleshooting/runtime-errors.md | 616 ------- .../docs/troubleshooting/scheduling-errors.md | 417 ----- .../docs/troubleshooting/storage-errors.md | 1011 ----------- .../vm-rebalance/REBALANCE_AUTOMATIC.md | 760 --------- .../skills/vm-rebalance/REBALANCE_MANUAL.md | 848 ---------- .../environment/skills/vm-rebalance/SKILL.md | 391 ----- .../vm-rebalance/references/anti-patterns.md | 869 ---------- .../live-migration-best-practices.md | 794 --------- .../references/performance-tuning.md | 719 -------- .../references/production-considerations.md | 868 ---------- .../.ai-index/semantic-index.json | 148 -- .../environment/docs/troubleshooting/INDEX.md | 332 ---- .../docs/troubleshooting/lifecycle-errors.md | 869 ---------- .../docs/troubleshooting/network-errors.md | 429 ----- .../docs/troubleshooting/runtime-errors.md | 616 ------- .../docs/troubleshooting/scheduling-errors.md | 417 ----- .../docs/troubleshooting/storage-errors.md | 1011 ----------- .../skills/vm-snapshot-create/SKILL.md | 423 ----- .../.ai-index/semantic-index.json | 148 -- .../environment/docs/troubleshooting/INDEX.md | 332 ---- .../docs/troubleshooting/lifecycle-errors.md | 869 ---------- .../docs/troubleshooting/network-errors.md | 429 ----- .../docs/troubleshooting/runtime-errors.md | 616 ------- .../docs/troubleshooting/scheduling-errors.md | 417 ----- .../docs/troubleshooting/storage-errors.md | 1011 ----------- .../skills/vm-snapshot-delete/SKILL.md | 447 ----- .../.ai-index/semantic-index.json | 148 -- .../environment/docs/troubleshooting/INDEX.md | 332 ---- .../docs/troubleshooting/lifecycle-errors.md | 869 ---------- .../docs/troubleshooting/network-errors.md | 429 ----- .../docs/troubleshooting/runtime-errors.md | 616 ------- .../docs/troubleshooting/scheduling-errors.md | 417 ----- .../docs/troubleshooting/storage-errors.md | 1011 ----------- .../skills/vm-snapshot-list/SKILL.md | 402 ----- .../.ai-index/semantic-index.json | 148 -- .../environment/docs/troubleshooting/INDEX.md | 332 ---- .../docs/troubleshooting/lifecycle-errors.md | 869 ---------- .../docs/troubleshooting/network-errors.md | 429 ----- .../docs/troubleshooting/runtime-errors.md | 616 ------- .../docs/troubleshooting/scheduling-errors.md | 417 ----- .../docs/troubleshooting/storage-errors.md | 1011 ----------- .../skills/vm-snapshot-restore/SKILL.md | 495 ------ .../environment/docs/multi-cluster-auth.md | 248 --- .../environment/docs/common-issues.md | 84 - .../environment/docs/examples/model-deploy.md | 160 -- .../environment/docs/examples/nim-setup.md | 115 -- .../environment/docs/live-doc-lookup.md | 106 -- .../docs/references/known-model-profiles.md | 83 - .../docs/references/supported-runtimes.md | 104 -- .../environment/docs/skill-conventions.md | 85 - .../environment/docs/common-issues.md | 84 - .../environment/docs/examples/model-deploy.md | 160 -- .../environment/docs/examples/nim-setup.md | 115 -- .../environment/docs/live-doc-lookup.md | 106 -- .../docs/references/known-model-profiles.md | 83 - .../docs/references/supported-runtimes.md | 104 -- .../environment/docs/skill-conventions.md | 85 - .../environment/docs/common-issues.md | 84 - .../environment/docs/examples/model-deploy.md | 160 -- .../environment/docs/examples/nim-setup.md | 115 -- .../environment/docs/live-doc-lookup.md | 106 -- .../docs/references/known-model-profiles.md | 83 - .../docs/references/supported-runtimes.md | 104 -- .../environment/docs/skill-conventions.md | 85 - .../environment/docs/common-issues.md | 84 - .../environment/docs/examples/model-deploy.md | 160 -- .../environment/docs/examples/nim-setup.md | 115 -- .../environment/docs/live-doc-lookup.md | 106 -- .../docs/references/known-model-profiles.md | 83 - .../docs/references/supported-runtimes.md | 104 -- .../environment/docs/skill-conventions.md | 85 - .../environment/docs/common-issues.md | 84 - .../environment/docs/examples/model-deploy.md | 160 -- .../environment/docs/examples/nim-setup.md | 115 -- .../environment/docs/live-doc-lookup.md | 106 -- .../docs/references/known-model-profiles.md | 83 - .../docs/references/supported-runtimes.md | 104 -- .../environment/docs/skill-conventions.md | 85 - .../environment/docs/common-issues.md | 84 - .../environment/docs/examples/model-deploy.md | 160 -- .../environment/docs/examples/nim-setup.md | 115 -- .../environment/docs/live-doc-lookup.md | 106 -- .../docs/references/known-model-profiles.md | 83 - .../docs/references/supported-runtimes.md | 104 -- .../environment/docs/skill-conventions.md | 85 - .../environment/docs/common-issues.md | 84 - .../environment/docs/examples/model-deploy.md | 160 -- .../environment/docs/examples/nim-setup.md | 115 -- .../environment/docs/live-doc-lookup.md | 106 -- .../docs/references/known-model-profiles.md | 83 - .../docs/references/supported-runtimes.md | 104 -- .../environment/docs/skill-conventions.md | 85 - .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../environment/docs/builder-images.md | 308 ---- .../environment/docs/debugging-patterns.md | 478 ------ .../environment/docs/dynamic-validation.md | 259 --- .../environment/docs/human-in-the-loop.md | 98 -- .../docs/image-selection-criteria.md | 221 --- .../environment/docs/prerequisites.md | 212 --- .../docs/python-s2i-entrypoints.md | 70 - .../environment/docs/rhel-deployment.md | 580 ------- .../docs/selinux-troubleshooting.md | 387 ----- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../docs/.ai-index/cross-reference-graph.json | 147 -- .../docs/.ai-index/semantic-index.json | 297 ---- .../docs/.ai-index/task-to-docs-mapping.json | 230 --- .../environment/docs/INDEX.md | 389 ----- .../environment/docs/SOURCES.md | 107 -- .../environment/docs/ansible/README.md | 50 - .../docs/ansible/aap-job-execution.md | 532 ------ .../docs/ansible/cve-remediation-templates.md | 1500 ----------------- .../docs/ansible/playbook-integration-aap.md | 667 -------- .../environment/docs/insights/README.md | 38 - .../docs/insights/vulnerability-logic.md | 568 ------- .../environment/docs/references/README.md | 39 - .../docs/references/cvss-scoring.md | 636 ------- .../references/lightspeed-mcp-parameters.md | 89 - .../lightspeed-mcp-tool-failures.md | 69 - .../docs/references/skill-invocation.md | 35 - .../environment/docs/rhel/README.md | 40 - .../docs/rhel/package-management.md | 738 -------- .../testing/aap-integration-test-guide.md | 649 ------- .../.ai-index/semantic-index.json | 148 -- .../environment/docs/troubleshooting/INDEX.md | 332 ---- .../docs/troubleshooting/lifecycle-errors.md | 869 ---------- .../docs/troubleshooting/network-errors.md | 429 ----- .../docs/troubleshooting/runtime-errors.md | 616 ------- .../docs/troubleshooting/scheduling-errors.md | 417 ----- .../docs/troubleshooting/storage-errors.md | 1011 ----------- .../.ai-index/semantic-index.json | 148 -- .../environment/docs/troubleshooting/INDEX.md | 332 ---- .../docs/troubleshooting/lifecycle-errors.md | 869 ---------- .../docs/troubleshooting/network-errors.md | 429 ----- .../docs/troubleshooting/runtime-errors.md | 616 ------- .../docs/troubleshooting/scheduling-errors.md | 417 ----- .../docs/troubleshooting/storage-errors.md | 1011 ----------- 1063 files changed, 338034 deletions(-) delete mode 100644 evaluation/with_skills/ocp-admin__cluster-report/environment/docs/multi-cluster-auth.md delete mode 100644 evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/aggregate.py delete mode 100644 evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/assemble.py delete mode 100644 evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/build-kubeconfig.py delete mode 100644 evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/cluster-reporter-rbac.yaml delete mode 100644 evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/test_aggregate.py delete mode 100644 evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/test_assemble.py delete mode 100644 evaluation/with_skills/ocp-admin__cluster-report/environment/skills/cluster-report/SKILL.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/common-issues.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/examples/model-deploy.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/examples/nim-setup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/live-doc-lookup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/references/known-model-profiles.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/references/supported-runtimes.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/skill-conventions.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/ai-observability/SKILL.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/common-issues.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/live-doc-lookup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/skill-conventions.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/common-issues.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/examples/model-deploy.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/examples/nim-setup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/live-doc-lookup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/references/known-model-profiles.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/references/supported-runtimes.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/skill-conventions.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/debug-inference/SKILL.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/common-issues.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/live-doc-lookup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/skill-conventions.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/common-issues.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/model-deploy.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/nim-setup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/live-doc-lookup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/known-model-profiles.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/supported-runtimes.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/skill-conventions.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/ds-project-setup/SKILL.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/common-issues.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/live-doc-lookup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/skill-conventions.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/common-issues.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/examples/model-deploy.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/examples/nim-setup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/live-doc-lookup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/references/known-model-profiles.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/references/supported-runtimes.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/skill-conventions.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/model-deploy/SKILL.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/model-deploy/references/model-deploy-preflight-checklist.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/common-issues.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/live-doc-lookup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/skill-conventions.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/common-issues.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/examples/model-deploy.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/examples/nim-setup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/live-doc-lookup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/references/known-model-profiles.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/references/supported-runtimes.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/skill-conventions.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/nim-setup/SKILL.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/common-issues.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/live-doc-lookup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/skill-conventions.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/common-issues.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/model-deploy.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/nim-setup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/live-doc-lookup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/known-model-profiles.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/supported-runtimes.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/skill-conventions.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/common-issues.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/live-doc-lookup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/skill-conventions.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/serving-runtime-config/SKILL.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/common-issues.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/model-deploy.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/nim-setup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/live-doc-lookup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/references/known-model-profiles.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/references/supported-runtimes.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/skill-conventions.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/common-issues.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/live-doc-lookup.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/skill-conventions.md delete mode 100644 evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/workbench-manage/SKILL.md delete mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/builder-images.md delete mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/debugging-patterns.md delete mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/dynamic-validation.md delete mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/prerequisites.md delete mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/rhel-deployment.md delete mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/with_skills/rh-developer__containerize-deploy/environment/skills/containerize-deploy/SKILL.md delete mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/docs/builder-images.md delete mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/docs/debugging-patterns.md delete mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/docs/dynamic-validation.md delete mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/docs/prerequisites.md delete mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/docs/rhel-deployment.md delete mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/with_skills/rh-developer__debug-build/environment/skills/debug-build/SKILL.md delete mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/docs/builder-images.md delete mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/docs/debugging-patterns.md delete mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/docs/dynamic-validation.md delete mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/docs/prerequisites.md delete mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/docs/rhel-deployment.md delete mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/with_skills/rh-developer__debug-container/environment/skills/debug-container/SKILL.md delete mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/docs/builder-images.md delete mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/docs/debugging-patterns.md delete mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/docs/dynamic-validation.md delete mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/docs/prerequisites.md delete mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/docs/rhel-deployment.md delete mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/with_skills/rh-developer__debug-network/environment/skills/debug-network/SKILL.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/builder-images.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/debugging-patterns.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/dynamic-validation.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/prerequisites.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/rhel-deployment.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pipeline/environment/skills/debug-pipeline/SKILL.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/docs/builder-images.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/docs/debugging-patterns.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/docs/dynamic-validation.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/docs/prerequisites.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/docs/rhel-deployment.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/with_skills/rh-developer__debug-pod/environment/skills/debug-pod/SKILL.md delete mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/docs/builder-images.md delete mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/docs/debugging-patterns.md delete mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/docs/dynamic-validation.md delete mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/docs/prerequisites.md delete mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/docs/rhel-deployment.md delete mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/with_skills/rh-developer__debug-rhel/environment/skills/debug-rhel/SKILL.md delete mode 100644 evaluation/with_skills/rh-developer__deploy/environment/docs/builder-images.md delete mode 100644 evaluation/with_skills/rh-developer__deploy/environment/docs/debugging-patterns.md delete mode 100644 evaluation/with_skills/rh-developer__deploy/environment/docs/dynamic-validation.md delete mode 100644 evaluation/with_skills/rh-developer__deploy/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/with_skills/rh-developer__deploy/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/with_skills/rh-developer__deploy/environment/docs/prerequisites.md delete mode 100644 evaluation/with_skills/rh-developer__deploy/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/with_skills/rh-developer__deploy/environment/docs/rhel-deployment.md delete mode 100644 evaluation/with_skills/rh-developer__deploy/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/with_skills/rh-developer__deploy/environment/skills/deploy/SKILL.md delete mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/docs/builder-images.md delete mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/docs/debugging-patterns.md delete mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/docs/dynamic-validation.md delete mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/docs/prerequisites.md delete mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/docs/rhel-deployment.md delete mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/with_skills/rh-developer__detect-project/environment/skills/detect-project/SKILL.md delete mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/docs/builder-images.md delete mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/docs/debugging-patterns.md delete mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/docs/dynamic-validation.md delete mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/docs/prerequisites.md delete mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/docs/rhel-deployment.md delete mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/with_skills/rh-developer__helm-deploy/environment/skills/helm-deploy/SKILL.md delete mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/docs/builder-images.md delete mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/docs/debugging-patterns.md delete mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/docs/dynamic-validation.md delete mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/docs/prerequisites.md delete mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/docs/rhel-deployment.md delete mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/with_skills/rh-developer__recommend-image/environment/skills/recommend-image/SKILL.md delete mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/builder-images.md delete mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/debugging-patterns.md delete mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/dynamic-validation.md delete mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/prerequisites.md delete mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/rhel-deployment.md delete mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/with_skills/rh-developer__rhel-deploy/environment/skills/rhel-deploy/SKILL.md delete mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/docs/builder-images.md delete mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/docs/debugging-patterns.md delete mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/docs/dynamic-validation.md delete mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/docs/prerequisites.md delete mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/docs/rhel-deployment.md delete mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/with_skills/rh-developer__s2i-build/environment/skills/s2i-build/SKILL.md delete mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/docs/builder-images.md delete mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/docs/debugging-patterns.md delete mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/docs/dynamic-validation.md delete mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/docs/prerequisites.md delete mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/docs/rhel-deployment.md delete mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/with_skills/rh-developer__validate-environment/environment/skills/validate-environment/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/INDEX.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/SOURCES.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/README.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/insights/README.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/README.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/rhel/README.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/rhel/package-management.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/01-account-cves.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/02-system-all-cves.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/03-system-remediatable-cves.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/01-cve-response-parser.py delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/02-cve-parsing-guide.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/03-output-templates.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/04-examples.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/05-error-handling.md delete mode 100644 evaluation/with_skills/rh-sre__cve-impact/environment/skills/mcp-lightspeed-validator/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/INDEX.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/SOURCES.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/README.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/insights/README.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/README.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/rhel/README.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/rhel/package-management.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/01-remediation-indicators.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/03-output-template.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/04-examples.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/05-error-handling.md delete mode 100644 evaluation/with_skills/rh-sre__cve-validation/environment/skills/mcp-lightspeed-validator/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/INDEX.md delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/SOURCES.md delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/README.md delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/insights/README.md delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/README.md delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/rhel/README.md delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/rhel/package-management.md delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/with_skills/rh-sre__execution-summary/environment/skills/execution-summary/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/INDEX.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/SOURCES.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/README.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/insights/README.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/README.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/rhel/README.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/rhel/package-management.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/01-parameter-reference.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/03-output-templates.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/04-examples.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/05-error-handling.md delete mode 100644 evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/mcp-lightspeed-validator/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/INDEX.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/SOURCES.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/README.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/insights/README.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/README.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/rhel/README.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/rhel/package-management.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/01-git-setup.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/02-web-ui-form.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/03-output-template.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/04-examples.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/mcp-aap-validator/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/01-execution-report-templates.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/02-error-handling-guide.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/03-workflow-examples.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/04-dry-run-display-templates.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/05-git-flow-prompts.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/INDEX.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/SOURCES.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/README.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/README.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/README.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/README.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/package-management.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/01-git-setup.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/02-web-ui-form.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/03-output-template.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/04-examples.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-remediation-validator/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/mcp-aap-validator/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/01-execution-report-templates.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/02-error-handling-guide.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/03-workflow-examples.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/04-dry-run-display-templates.md delete mode 100644 evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/05-git-flow-prompts.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/INDEX.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/SOURCES.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/README.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/insights/README.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/README.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/README.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/package-management.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-aap-validator/environment/skills/mcp-aap-validator/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/INDEX.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/SOURCES.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/README.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/README.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/README.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/README.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/package-management.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/skills/mcp-lightspeed-validator/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/INDEX.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/SOURCES.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/README.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/insights/README.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/README.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/rhel/README.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/rhel/package-management.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/skills/mcp-aap-validator/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/01-execution-report-templates.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/02-error-handling-guide.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/03-workflow-examples.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/04-dry-run-display-templates.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/05-git-flow-prompts.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/INDEX.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/SOURCES.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/README.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/insights/README.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/README.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/rhel/README.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/rhel/package-management.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/with_skills/rh-sre__playbook-generator/environment/skills/playbook-generator/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/INDEX.md delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/SOURCES.md delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/README.md delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/insights/README.md delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/README.md delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/rhel/README.md delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/rhel/package-management.md delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/with_skills/rh-sre__remediation-verifier/environment/skills/remediation-verifier/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/INDEX.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/SOURCES.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/README.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/insights/README.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/references/README.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/rhel/README.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/rhel/package-management.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/01-remediation-indicators.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/03-output-template.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/04-examples.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/05-error-handling.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/skills/remediation/SKILL.md delete mode 100644 evaluation/with_skills/rh-sre__remediation/environment/skills/remediation/references/01-remediation-plan-template.md delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/INDEX.md delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/SOURCES.md delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/README.md delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/insights/README.md delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/references/README.md delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/rhel/README.md delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/rhel/package-management.md delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/with_skills/rh-sre__system-context/environment/skills/system-context/SKILL.md delete mode 100644 evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/INDEX.md delete mode 100644 evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/lifecycle-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/network-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/runtime-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/scheduling-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/storage-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-clone/environment/skills/vm-clone/SKILL.md delete mode 100644 evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/INDEX.md delete mode 100644 evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/lifecycle-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/network-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/runtime-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/scheduling-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/storage-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-create/environment/skills/vm-create/SKILL.md delete mode 100644 evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/INDEX.md delete mode 100644 evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/lifecycle-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/network-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/runtime-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/scheduling-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/storage-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-delete/environment/skills/vm-delete/SKILL.md delete mode 100644 evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/INDEX.md delete mode 100644 evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/lifecycle-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/network-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/runtime-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/scheduling-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/storage-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-inventory/environment/skills/vm-inventory/SKILL.md delete mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/INDEX.md delete mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/lifecycle-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/network-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/runtime-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/scheduling-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/storage-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/skills/vm-lifecycle-manager/SKILL.md delete mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/INDEX.md delete mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/lifecycle-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/network-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/runtime-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/scheduling-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/storage-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/REBALANCE_AUTOMATIC.md delete mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/REBALANCE_MANUAL.md delete mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/SKILL.md delete mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/anti-patterns.md delete mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/live-migration-best-practices.md delete mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/performance-tuning.md delete mode 100644 evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/production-considerations.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/INDEX.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/lifecycle-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/network-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/runtime-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/scheduling-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/storage-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-create/environment/skills/vm-snapshot-create/SKILL.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/INDEX.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/lifecycle-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/network-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/runtime-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/scheduling-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/storage-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/skills/vm-snapshot-delete/SKILL.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/INDEX.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/lifecycle-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/network-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/runtime-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/scheduling-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/storage-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-list/environment/skills/vm-snapshot-list/SKILL.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/.ai-index/semantic-index.json delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/INDEX.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/lifecycle-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/network-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/runtime-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/scheduling-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/storage-errors.md delete mode 100644 evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/skills/vm-snapshot-restore/SKILL.md delete mode 100644 evaluation/without_skills/ocp-admin__cluster-report/environment/docs/multi-cluster-auth.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/common-issues.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/examples/model-deploy.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/examples/nim-setup.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/live-doc-lookup.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/references/known-model-profiles.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/references/supported-runtimes.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/skill-conventions.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/common-issues.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/examples/model-deploy.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/examples/nim-setup.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/live-doc-lookup.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/references/known-model-profiles.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/references/supported-runtimes.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/skill-conventions.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/common-issues.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/model-deploy.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/nim-setup.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/live-doc-lookup.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/known-model-profiles.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/supported-runtimes.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/skill-conventions.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/common-issues.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/examples/model-deploy.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/examples/nim-setup.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/live-doc-lookup.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/references/known-model-profiles.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/references/supported-runtimes.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/skill-conventions.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/common-issues.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/examples/model-deploy.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/examples/nim-setup.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/live-doc-lookup.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/references/known-model-profiles.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/references/supported-runtimes.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/skill-conventions.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/common-issues.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/model-deploy.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/nim-setup.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/live-doc-lookup.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/known-model-profiles.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/supported-runtimes.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/skill-conventions.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/common-issues.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/model-deploy.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/nim-setup.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/live-doc-lookup.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/references/known-model-profiles.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/references/supported-runtimes.md delete mode 100644 evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/skill-conventions.md delete mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/builder-images.md delete mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/debugging-patterns.md delete mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/dynamic-validation.md delete mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/prerequisites.md delete mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/rhel-deployment.md delete mode 100644 evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/docs/builder-images.md delete mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/docs/debugging-patterns.md delete mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/docs/dynamic-validation.md delete mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/docs/prerequisites.md delete mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/docs/rhel-deployment.md delete mode 100644 evaluation/without_skills/rh-developer__debug-build/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/docs/builder-images.md delete mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/docs/debugging-patterns.md delete mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/docs/dynamic-validation.md delete mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/docs/prerequisites.md delete mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/docs/rhel-deployment.md delete mode 100644 evaluation/without_skills/rh-developer__debug-container/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/docs/builder-images.md delete mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/docs/debugging-patterns.md delete mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/docs/dynamic-validation.md delete mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/docs/prerequisites.md delete mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/docs/rhel-deployment.md delete mode 100644 evaluation/without_skills/rh-developer__debug-network/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/builder-images.md delete mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/debugging-patterns.md delete mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/dynamic-validation.md delete mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/prerequisites.md delete mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/rhel-deployment.md delete mode 100644 evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/docs/builder-images.md delete mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/docs/debugging-patterns.md delete mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/docs/dynamic-validation.md delete mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/docs/prerequisites.md delete mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/docs/rhel-deployment.md delete mode 100644 evaluation/without_skills/rh-developer__debug-pod/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/docs/builder-images.md delete mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/docs/debugging-patterns.md delete mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/docs/dynamic-validation.md delete mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/docs/prerequisites.md delete mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/docs/rhel-deployment.md delete mode 100644 evaluation/without_skills/rh-developer__debug-rhel/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/without_skills/rh-developer__deploy/environment/docs/builder-images.md delete mode 100644 evaluation/without_skills/rh-developer__deploy/environment/docs/debugging-patterns.md delete mode 100644 evaluation/without_skills/rh-developer__deploy/environment/docs/dynamic-validation.md delete mode 100644 evaluation/without_skills/rh-developer__deploy/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/without_skills/rh-developer__deploy/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/without_skills/rh-developer__deploy/environment/docs/prerequisites.md delete mode 100644 evaluation/without_skills/rh-developer__deploy/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/without_skills/rh-developer__deploy/environment/docs/rhel-deployment.md delete mode 100644 evaluation/without_skills/rh-developer__deploy/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/docs/builder-images.md delete mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/docs/debugging-patterns.md delete mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/docs/dynamic-validation.md delete mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/docs/prerequisites.md delete mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/docs/rhel-deployment.md delete mode 100644 evaluation/without_skills/rh-developer__detect-project/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/docs/builder-images.md delete mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/docs/debugging-patterns.md delete mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/docs/dynamic-validation.md delete mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/docs/prerequisites.md delete mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/docs/rhel-deployment.md delete mode 100644 evaluation/without_skills/rh-developer__helm-deploy/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/docs/builder-images.md delete mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/docs/debugging-patterns.md delete mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/docs/dynamic-validation.md delete mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/docs/prerequisites.md delete mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/docs/rhel-deployment.md delete mode 100644 evaluation/without_skills/rh-developer__recommend-image/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/builder-images.md delete mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/debugging-patterns.md delete mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/dynamic-validation.md delete mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/prerequisites.md delete mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/rhel-deployment.md delete mode 100644 evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/docs/builder-images.md delete mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/docs/debugging-patterns.md delete mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/docs/dynamic-validation.md delete mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/docs/prerequisites.md delete mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/docs/rhel-deployment.md delete mode 100644 evaluation/without_skills/rh-developer__s2i-build/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/docs/builder-images.md delete mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/docs/debugging-patterns.md delete mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/docs/dynamic-validation.md delete mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/docs/human-in-the-loop.md delete mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/docs/image-selection-criteria.md delete mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/docs/prerequisites.md delete mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/docs/python-s2i-entrypoints.md delete mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/docs/rhel-deployment.md delete mode 100644 evaluation/without_skills/rh-developer__validate-environment/environment/docs/selinux-troubleshooting.md delete mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/INDEX.md delete mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/SOURCES.md delete mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/README.md delete mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/insights/README.md delete mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/README.md delete mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/rhel/README.md delete mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/rhel/package-management.md delete mode 100644 evaluation/without_skills/rh-sre__cve-impact/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/INDEX.md delete mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/SOURCES.md delete mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/README.md delete mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/insights/README.md delete mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/README.md delete mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/rhel/README.md delete mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/rhel/package-management.md delete mode 100644 evaluation/without_skills/rh-sre__cve-validation/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/INDEX.md delete mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/SOURCES.md delete mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/README.md delete mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/insights/README.md delete mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/README.md delete mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/rhel/README.md delete mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/rhel/package-management.md delete mode 100644 evaluation/without_skills/rh-sre__execution-summary/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/INDEX.md delete mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/SOURCES.md delete mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/README.md delete mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/insights/README.md delete mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/README.md delete mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/rhel/README.md delete mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/rhel/package-management.md delete mode 100644 evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/INDEX.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/SOURCES.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/README.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/insights/README.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/README.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/rhel/README.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/rhel/package-management.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-creator/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/INDEX.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/SOURCES.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/README.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/README.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/README.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/README.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/package-management.md delete mode 100644 evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/INDEX.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/SOURCES.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/README.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/insights/README.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/README.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/README.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/package-management.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/INDEX.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/SOURCES.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/README.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/README.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/README.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/README.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/package-management.md delete mode 100644 evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/INDEX.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/SOURCES.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/README.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/insights/README.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/README.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/rhel/README.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/rhel/package-management.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-executor/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/INDEX.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/SOURCES.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/README.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/insights/README.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/README.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/rhel/README.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/rhel/package-management.md delete mode 100644 evaluation/without_skills/rh-sre__playbook-generator/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/INDEX.md delete mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/SOURCES.md delete mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/README.md delete mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/insights/README.md delete mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/README.md delete mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/rhel/README.md delete mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/rhel/package-management.md delete mode 100644 evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/INDEX.md delete mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/SOURCES.md delete mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/README.md delete mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/insights/README.md delete mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/references/README.md delete mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/rhel/README.md delete mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/rhel/package-management.md delete mode 100644 evaluation/without_skills/rh-sre__remediation/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/cross-reference-graph.json delete mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/semantic-index.json delete mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/task-to-docs-mapping.json delete mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/INDEX.md delete mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/SOURCES.md delete mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/README.md delete mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/aap-job-execution.md delete mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/cve-remediation-templates.md delete mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/playbook-integration-aap.md delete mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/insights/README.md delete mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/insights/vulnerability-logic.md delete mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/references/README.md delete mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/references/cvss-scoring.md delete mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-parameters.md delete mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-tool-failures.md delete mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/references/skill-invocation.md delete mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/rhel/README.md delete mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/rhel/package-management.md delete mode 100644 evaluation/without_skills/rh-sre__system-context/environment/docs/testing/aap-integration-test-guide.md delete mode 100644 evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/.ai-index/semantic-index.json delete mode 100644 evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/INDEX.md delete mode 100644 evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/lifecycle-errors.md delete mode 100644 evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/network-errors.md delete mode 100644 evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/runtime-errors.md delete mode 100644 evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/scheduling-errors.md delete mode 100644 evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/storage-errors.md delete mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/.ai-index/semantic-index.json delete mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/INDEX.md delete mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/lifecycle-errors.md delete mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/network-errors.md delete mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/runtime-errors.md delete mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/scheduling-errors.md delete mode 100644 evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/storage-errors.md diff --git a/evaluation/with_skills/ocp-admin__cluster-report/environment/docs/multi-cluster-auth.md b/evaluation/with_skills/ocp-admin__cluster-report/environment/docs/multi-cluster-auth.md deleted file mode 100644 index e187471b..00000000 --- a/evaluation/with_skills/ocp-admin__cluster-report/environment/docs/multi-cluster-auth.md +++ /dev/null @@ -1,248 +0,0 @@ -# Multi-Cluster Authentication with Service Account Tokens - -Set up non-interactive, long-lived authentication for running `cluster-report` across many OpenShift clusters without repeated `oc login` sessions. - -## Overview - -The `cluster-report` skill requires valid kubeconfig contexts for every cluster it reports on. Interactive `oc login --web` opens a browser for each cluster and produces tokens that expire in ~24 hours which make it difficult to do at scale. - -**Solution**: Create a read-only ServiceAccount on each cluster with a non-expiring token. A builder script assembles these tokens into a single merged kubeconfig that the skill uses unchanged. - -## Prerequisites - -- `oc` or `kubectl` CLI -- `python3` (stdlib only, no extra packages) -- `cluster-admin` access on each target cluster (one-time setup only) - -## Quick Start (Automated) - -If you're currently logged into all the clusters you would like to get a report for via `oc login`: - -```bash -# Step 1: Setup — applies RBAC to each cluster, extracts SA tokens -python3 ocp-admin/scripts/cluster-report/build-kubeconfig.py setup --all-contexts - -# Step 2: Build — assembles a merged kubeconfig from the inventory -python3 ocp-admin/scripts/cluster-report/build-kubeconfig.py \ - build --clusters ~/.ocp-clusters/clusters.json --verify - -# Step 3: Use — export and run the skill -export KUBECONFIG=/tmp/cluster-report-kubeconfig -# Then in Claude Code use the skill: /cluster-report -``` - -After the one-time setup, only Steps 2–3 are needed for future report sessions. - -## Manual Setup (Per Cluster) - -If you prefer to set up each cluster individually: - -### 1. Apply RBAC - -> **Required permissions**: The manifest creates cluster-scoped resources (ClusterRole, ClusterRoleBinding), so the user applying it needs `cluster-admin` privileges. This is a one-time setup step. - -```bash -oc login -oc apply -f ocp-admin/scripts/cluster-report/cluster-reporter-rbac.yaml -``` - -This creates: - -- Namespace `cluster-reporter-system` -- ServiceAccount `cluster-reporter` with a read-only ClusterRole -- ClusterRoleBinding `cluster-reporter-binding` (binds the SA to the ClusterRole) -- Token Secret `cluster-reporter-token` (non-expiring) - -### 2. Extract the Token - -```bash -oc get secret cluster-reporter-token -n cluster-reporter-system \ - -o jsonpath='{.data.token}' | base64 -d -``` - -Save this token securely. It grants read-only access to nodes, pods, namespaces, projects, cluster version, and metrics. - -> **AI Safety**: Never display token values in conversation output. Verify tokens are set, but never print or echo their contents. - -### 3. Add to Inventory File - -Create or edit `~/.ocp-clusters/clusters.json`: - -```json -{ - "clusters": [ - { - "name": "prod-us-east", - "api_url": "https://api.prod-us-east.example.com:6443", - "token": "sha256~your-token-here" - } - ] -} -``` - -Set permissions: `chmod 600 ~/.ocp-clusters/clusters.json` - -### 4. Build Kubeconfig - -```bash -python3 ocp-admin/scripts/cluster-report/build-kubeconfig.py \ - build --clusters ~/.ocp-clusters/clusters.json --output ~/.kube/cluster-report-kubeconfig -``` - -## RBAC Permissions - -The `cluster-reporter-readonly` ClusterRole grants the minimum permissions required by the `cluster-report` skill: - - -| Resource | API Group | Verbs | Used By | -| ----------------------- | -------------------- | --------- | ------------------------------------------------------------- | -| nodes, namespaces, pods | core | get, list | `nodes_top`, `resources_list`, `namespaces_list`, `pods_list` | -| clusterversions | config.openshift.io | get | `resources_get` (OpenShift verification) | -| projects | project.openshift.io | list | `projects_list` | -| nodes, pods (metrics) | metrics.k8s.io | get, list | `nodes_top` | - - -No create, update, delete, or watch permissions are granted. - -## Clusters Inventory Format - -The inventory file (`clusters.json`) supports two token modes: - -### Inline Tokens (Simple) - -```json -{ - "clusters": [ - { - "name": "prod-us-east", - "api_url": "https://api.prod-us-east.example.com:6443", - "token": "sha256~abc123..." - } - ] -} -``` - -The file itself contains secrets — keep it out of git and set `chmod 600`. - -### Environment Variable References (More Secure) - -```json -{ - "clusters": [ - { - "name": "prod-us-east", - "api_url": "https://api.prod-us-east.example.com:6443", - "token_env": "CLUSTER_TOKEN_PROD_US_EAST" - } - ] -} -``` - -The file contains no secrets. Load tokens into environment variables from your secrets manager before running `--build`. - -### Optional: CA Certificate - -```json -{ - "clusters": [ - { - "name": "prod-us-east", - "api_url": "https://api.prod-us-east.example.com:6443", - "token": "sha256~abc123...", - "ca_cert": "/path/to/prod-us-east-ca.crt" - } - ] -} -``` - -If `ca_cert` is omitted, TLS verification is skipped (`--insecure-skip-tls-verify`). - -## Script Reference - -### `setup` Subcommand - -```bash -python3 build-kubeconfig.py setup [OPTIONS] -``` - - -| Flag | Description | Default | -| --------------------------- | ----------------------------- | ------------------------------- | -| `--all-contexts` | Setup all kubeconfig contexts | Lists contexts and exits | -| `--contexts ctx1,ctx2` | Setup only specified contexts | — | -| `--output-inventory ` | Inventory file path | `~/.ocp-clusters/clusters.json` | - - -Behavior: - -- Applies `cluster-reporter-rbac.yaml` to each cluster -- Waits up to 15 seconds for the token Secret to populate -- Extracts and saves the token to the inventory file -- Skips unreachable clusters with an error message -- Appends to existing inventory (deduplicates by name) - -### `build` Subcommand - -```bash -python3 build-kubeconfig.py build --clusters [OPTIONS] -``` - - -| Flag | Description | Default | -| ------------------- | -------------------------------- | -------------------------------- | -| `--clusters ` | Inventory file path (required) | — | -| `--output ` | Kubeconfig output path | `/tmp/cluster-report-kubeconfig` | -| `--verify` | Test each context after building | Off | - - -Behavior: - -- Reads inventory, resolves tokens (inline or env var) -- Builds kubeconfig with `kubectl config set-cluster/set-credentials/set-context` -- Partial success: continues on individual failures -- `--verify` tests each context with `cluster-info` -- Outputs JSON summary with success/error counts - -## Token Rotation - -SA token Secrets do not expire, but you may want to rotate them periodically: - -```bash -oc delete secret cluster-reporter-token -n cluster-reporter-system -oc apply -f ocp-admin/scripts/cluster-report/cluster-reporter-rbac.yaml - -oc get secret cluster-reporter-token -n cluster-reporter-system \ - -o jsonpath='{.data.token}' | base64 -d - -python3 build-kubeconfig.py build --clusters ~/.ocp-clusters/clusters.json --verify -``` - -To detect expired or invalid tokens: - -```bash -python3 build-kubeconfig.py build --clusters ~/.ocp-clusters/clusters.json --verify -``` - -## Security Best Practices - -1. **Never commit tokens to git** — add `clusters.json` to `.gitignore` -2. **File permissions** — `chmod 600` on both `clusters.json` and the generated kubeconfig -3. **Prefer `token_env`** — store actual tokens in a secrets manager, not in files -4. **Minimum RBAC** — the ClusterRole grants read-only access only -5. **Dedicated namespace** — the SA lives in `cluster-reporter-system`, not `kube-system` -6. **Generated kubeconfig is ephemeral** — `/tmp/` is fine for session use; for persistent storage use `~/.kube/` with `chmod 600` -7. **Never display tokens in AI conversations** — verify tokens are set but never print, echo, or expose their values in output - -## Troubleshooting - - -| Problem | Cause | Fix | -| ---------------------------------------- | ----------------------------------------- | ------------------------------------------------------------- | -| `--setup` skips a cluster | Not logged in or auth expired | `oc login ` first, then re-run setup | -| `--verify` fails for a cluster | Token expired or Secret deleted | Re-run `--setup --contexts ` for that cluster | -| `cluster-report` shows 401 for a cluster | Token invalid | Same as above — re-run setup for that cluster | -| `cluster-report` shows 403 | SA missing permissions | Re-apply `cluster-reporter-rbac.yaml` on that cluster | -| Token Secret not populated | Token controller slow or SA doesn't exist | Wait and retry; verify SA exists in `cluster-reporter-system` | -| `--build` says "env var not set" | Using `token_env` but env not loaded | Export the token env vars before running `--build` | - - diff --git a/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/aggregate.py b/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/aggregate.py deleted file mode 100644 index 031f0bbe..00000000 --- a/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/aggregate.py +++ /dev/null @@ -1,601 +0,0 @@ -#!/usr/bin/env python3 - -import json -import math -import re -import sys - - -def parse_cpu(value): - if value is None: - return 0.0 - s = str(value).strip() - if s.endswith("m"): - return float(s[:-1]) / 1000.0 - if s.endswith("n"): - return float(s[:-1]) / 1e9 - if s.endswith("u"): - return float(s[:-1]) / 1e6 - return float(s) - - -def parse_memory(value): - if value is None: - return 0.0 - s = str(value).strip() - multipliers = { - "Ki": 1024, - "Mi": 1024 ** 2, - "Gi": 1024 ** 3, - "Ti": 1024 ** 4, - "K": 1000, - "M": 1000 ** 2, - "G": 1000 ** 3, - "T": 1000 ** 4, - } - for suffix, mult in sorted(multipliers.items(), key=lambda x: -len(x[0])): - if s.endswith(suffix): - num = float(s[: -len(suffix)]) - return (num * mult) / (1024 ** 3) - return float(s) / (1024 ** 3) - - -def detect_node_role(labels): - if not labels: - return "worker" - prefix = "node-role.kubernetes.io/" - roles = [] - for key in labels: - if key.startswith(prefix): - role = key[len(prefix):] - if role: - roles.append(role) - if not roles: - return "worker" - priority = ["control-plane", "master", "infra", "worker"] - for p in priority: - if p in roles: - return p - return roles[0] - - -GPU_KEYS = ["nvidia.com/gpu", "amd.com/gpu", "intel.com/gpu"] - - -def detect_gpus(allocatable): - if not allocatable: - return 0, "" - for key in GPU_KEYS: - val = allocatable.get(key) - if val is not None: - count = int(val) - if count > 0: - return count, key - return 0, "" - - -def parse_tabular(text): - if not text or not isinstance(text, str): - return [] - - lines = text.splitlines() - non_blank = [line for line in lines if line.strip()] - if len(non_blank) < 2: - return [] - - header_line = non_blank[0] - data_lines = non_blank[1:] - - starts = [0] - i = 0 - while i < len(header_line): - if header_line[i] == " ": - space_start = i - while i < len(header_line) and header_line[i] == " ": - i += 1 - if i < len(header_line) and (i - space_start) >= 2: - starts.append(i) - else: - i += 1 - - headers = [] - for idx, start in enumerate(starts): - end = starts[idx + 1] if idx + 1 < len(starts) else len(header_line) - headers.append(header_line[start:end].strip()) - - result = [] - for line in data_lines: - row = {} - for idx, start in enumerate(starts): - end = starts[idx + 1] if idx + 1 < len(starts) else len(line) - value = line[start:end].strip() if start < len(line) else "" - row[headers[idx]] = value - result.append(row) - - return result - - -def parse_labels_string(labels_str): - if not labels_str or labels_str == "": - return {} - result = {} - for item in labels_str.split(","): - item = item.strip() - if not item: - continue - if "=" in item: - key, val = item.split("=", 1) - result[key] = val - else: - result[item] = "" - return result - - -def _col(row, name, default=""): - if name in row: - return row[name] - name_lower = name.lower() - for key in row: - if key.lower() == name_lower: - return row[key] - return default - - -def parse_pods_tabular(text): - rows = parse_tabular(text) - result = [] - for row in rows: - result.append({ - "namespace": _col(row, "NAMESPACE", "unknown"), - "name": _col(row, "NAME", "unknown"), - "status": _col(row, "STATUS", "Unknown"), - }) - return result - - -def parse_nodes_list_tabular(text): - rows = parse_tabular(text) - result = [] - for row in rows: - name = _col(row, "NAME", "unknown") - roles_str = _col(row, "ROLES", "") - labels_str = _col(row, "LABELS", "") - - labels = parse_labels_string(labels_str) - if roles_str and roles_str != "": - for role in roles_str.split(","): - role = role.strip() - if role: - label_key = f"node-role.kubernetes.io/{role}" - if label_key not in labels: - labels[label_key] = "" - - result.append({ - "metadata": {"name": name, "labels": labels}, - "status": {}, - }) - return result - - -def parse_nodes_top_tabular(text): - rows = parse_tabular(text) - result = [] - for row in rows: - result.append({ - "name": _col(row, "NAME", "unknown"), - "cpu_usage": _col(row, "CPU(cores)") or None, - "memory_usage": _col(row, "MEMORY(bytes)") or None, - }) - return result - - -def parse_projects_tabular(text): - rows = parse_tabular(text) - return [{"name": _col(row, "NAME", "unknown")} for row in rows] - - -def parse_namespaces_tabular(text): - rows = parse_tabular(text) - return [{"name": _col(row, "NAME", "unknown")} for row in rows] - - -def classify_pod_status(pod): - if isinstance(pod, dict) and "status" in pod and isinstance(pod["status"], str): - return pod["status"] - - status_obj = pod.get("status", {}) - if isinstance(status_obj, str): - return status_obj - - phase = status_obj.get("phase", "Unknown") - - container_statuses = status_obj.get("containerStatuses", []) - if not container_statuses: - container_statuses = status_obj.get("initContainerStatuses", []) - - for cs in container_statuses or []: - state = cs.get("state", {}) - waiting = state.get("waiting", {}) - reason = waiting.get("reason", "") - if reason in ( - "CrashLoopBackOff", - "ImagePullBackOff", - "ErrImagePull", - "CreateContainerError", - "CreateContainerConfigError", - "RunContainerError", - ): - return reason - - if phase == "Completed": - return "Succeeded" - - return phase - - -def aggregate_pods_by_namespace(pods, top_n=10): - if not pods: - return [] - - ns_data = {} - for pod in pods: - if "metadata" in pod: - ns = pod["metadata"].get("namespace", "unknown") - else: - ns = pod.get("namespace", "unknown") - - status = classify_pod_status(pod) - - if ns not in ns_data: - ns_data[ns] = {"namespace": ns, "pods_total": 0, "running": 0, - "pending": 0, "failed": 0, "succeeded": 0, "other": 0} - - ns_data[ns]["pods_total"] += 1 - if status == "Running": - ns_data[ns]["running"] += 1 - elif status == "Pending": - ns_data[ns]["pending"] += 1 - elif status in ("Failed", "Error"): - ns_data[ns]["failed"] += 1 - elif status in ("Succeeded", "Completed"): - ns_data[ns]["succeeded"] += 1 - else: - ns_data[ns]["other"] += 1 - - sorted_ns = sorted(ns_data.values(), key=lambda x: x["pods_total"], reverse=True) - return sorted_ns[:top_n] - - -def process_nodes(nodes_top, nodes_list): - nodes = {} - metrics_available = nodes_top is not None - - if nodes_list: - for node in nodes_list: - if isinstance(node, dict): - meta = node.get("metadata", {}) - name = meta.get("name", node.get("name", "unknown")) - labels = meta.get("labels", node.get("labels", {})) - status = node.get("status", {}) - allocatable = status.get("allocatable", {}) - capacity = status.get("capacity", {}) - - role = detect_node_role(labels) - gpu_count, gpu_type = detect_gpus(allocatable) - - cpu_total = parse_cpu(allocatable.get("cpu") or capacity.get("cpu")) - mem_total = parse_memory(allocatable.get("memory") or capacity.get("memory")) - - nodes[name] = { - "name": name, - "role": role, - "cpu_used": None, - "cpu_total": round(cpu_total, 2), - "memory_used": None, - "memory_total": round(mem_total, 2), - "gpus": gpu_count, - "gpu_type": gpu_type, - } - - if nodes_top: - for entry in nodes_top: - if isinstance(entry, dict): - name = entry.get("name", entry.get("NAME", "unknown")) - cpu_used = entry.get("cpu_usage") or entry.get("CPU(cores)") or entry.get("cpu") - mem_used = entry.get("memory_usage") or entry.get("MEMORY(bytes)") or entry.get("memory") - - if name in nodes: - if cpu_used is not None: - nodes[name]["cpu_used"] = round(parse_cpu(str(cpu_used)), 2) - if mem_used is not None: - nodes[name]["memory_used"] = round(parse_memory(str(mem_used)), 2) - else: - nodes[name] = { - "name": name, - "role": "worker", - "cpu_used": round(parse_cpu(str(cpu_used)), 2) if cpu_used else None, - "cpu_total": None, - "memory_used": round(parse_memory(str(mem_used)), 2) if mem_used else None, - "memory_total": None, - "gpus": 0, - "gpu_type": "", - } - - return list(nodes.values()), metrics_available - - -def process_cluster(cluster_data): - errors = cluster_data.get("errors", []) - nodes_top = cluster_data.get("nodes_top") - nodes_list = cluster_data.get("nodes_list") - projects = cluster_data.get("projects") - namespaces = cluster_data.get("namespaces") - pods = cluster_data.get("pods") - - if isinstance(pods, str): - pods = parse_pods_tabular(pods) - if isinstance(nodes_top, str): - nodes_top = parse_nodes_top_tabular(nodes_top) - if isinstance(nodes_list, str): - nodes_list = parse_nodes_list_tabular(nodes_list) - if isinstance(projects, str): - projects = parse_projects_tabular(projects) - if isinstance(namespaces, str): - namespaces = parse_namespaces_tabular(namespaces) - - nodes_detail, metrics_available = process_nodes(nodes_top, nodes_list) - - cpu_used = None - cpu_total = 0.0 - mem_used = None - mem_total = 0.0 - gpu_total = 0 - - for node in nodes_detail: - if node["cpu_total"] is not None: - cpu_total += node["cpu_total"] - if node["memory_total"] is not None: - mem_total += node["memory_total"] - if node["cpu_used"] is not None: - cpu_used = (cpu_used or 0.0) + node["cpu_used"] - if node["memory_used"] is not None: - mem_used = (mem_used or 0.0) + node["memory_used"] - gpu_total += node["gpus"] - - cpu_percent = None - if cpu_used is not None and cpu_total > 0: - cpu_percent = round((cpu_used / cpu_total) * 100) - - mem_percent = None - if mem_used is not None and mem_total > 0: - mem_percent = round((mem_used / mem_total) * 100) - - project_count = 0 - if projects is not None: - project_count = len(projects) if isinstance(projects, list) else 0 - elif namespaces is not None: - project_count = len(namespaces) if isinstance(namespaces, list) else 0 - - pod_status = { - "Running": 0, - "Pending": 0, - "Succeeded": 0, - "Failed": 0, - "Unknown": 0, - "CrashLoopBackOff": 0, - "ImagePullBackOff": 0, - "ErrImagePull": 0, - "Other": 0, - } - pods_running = 0 - pods_total = 0 - - if pods and isinstance(pods, list): - pods_total = len(pods) - for pod in pods: - status = classify_pod_status(pod) - if status in pod_status: - pod_status[status] += 1 - else: - pod_status["Other"] += 1 - if status == "Running": - pods_running += 1 - - top_namespaces = aggregate_pods_by_namespace(pods or []) - - return { - "overview": { - "cluster": cluster_data.get("context", "unknown"), - "server": cluster_data.get("server", "unknown"), - "node_count": len(nodes_detail), - "cpu_used_cores": round(cpu_used, 1) if cpu_used is not None else None, - "cpu_total_cores": round(cpu_total, 1), - "cpu_percent": cpu_percent, - "memory_used_gib": round(mem_used, 1) if mem_used is not None else None, - "memory_total_gib": round(mem_total, 1), - "memory_percent": mem_percent, - "gpu_total": gpu_total, - "project_count": project_count, - "pods_running": pods_running, - "pods_total": pods_total, - "metrics_available": metrics_available, - }, - "nodes": nodes_detail, - "pod_status": {k: v for k, v in pod_status.items() if v > 0}, - "top_namespaces": top_namespaces, - "errors": errors, - } - - -def compute_totals(overview_list): - totals = { - "node_count": 0, - "cpu_used_cores": None, - "cpu_total_cores": 0.0, - "memory_used_gib": None, - "memory_total_gib": 0.0, - "gpu_total": 0, - "project_count": 0, - "pods_running": 0, - "pods_total": 0, - } - - for ov in overview_list: - totals["node_count"] += ov.get("node_count", 0) - totals["cpu_total_cores"] += ov.get("cpu_total_cores", 0) - totals["memory_total_gib"] += ov.get("memory_total_gib", 0) - totals["gpu_total"] += ov.get("gpu_total", 0) - totals["project_count"] += ov.get("project_count", 0) - totals["pods_running"] += ov.get("pods_running", 0) - totals["pods_total"] += ov.get("pods_total", 0) - - if ov.get("cpu_used_cores") is not None: - totals["cpu_used_cores"] = (totals["cpu_used_cores"] or 0) + ov["cpu_used_cores"] - if ov.get("memory_used_gib") is not None: - totals["memory_used_gib"] = (totals["memory_used_gib"] or 0) + ov["memory_used_gib"] - - totals["cpu_total_cores"] = round(totals["cpu_total_cores"], 1) - totals["memory_total_gib"] = round(totals["memory_total_gib"], 1) - - if totals["cpu_used_cores"] is not None: - totals["cpu_used_cores"] = round(totals["cpu_used_cores"], 1) - if totals["memory_used_gib"] is not None: - totals["memory_used_gib"] = round(totals["memory_used_gib"], 1) - - if totals["cpu_used_cores"] is not None and totals["cpu_total_cores"] > 0: - totals["cpu_percent"] = round((totals["cpu_used_cores"] / totals["cpu_total_cores"]) * 100) - else: - totals["cpu_percent"] = None - - if totals["memory_used_gib"] is not None and totals["memory_total_gib"] > 0: - totals["memory_percent"] = round((totals["memory_used_gib"] / totals["memory_total_gib"]) * 100) - else: - totals["memory_percent"] = None - - return totals - - -def detect_attention_items(overview_list, per_cluster): - items = [] - - for ov in overview_list: - cluster = ov["cluster"] - pc = per_cluster.get(cluster, {}) - - if ov.get("cpu_percent") is not None and ov["cpu_percent"] > 85: - items.append(f"{cluster}: Cluster CPU usage at {ov['cpu_percent']}% (>85% threshold)") - - if ov.get("memory_percent") is not None and ov["memory_percent"] > 85: - items.append(f"{cluster}: Cluster memory usage at {ov['memory_percent']}% (>85% threshold)") - - for node in pc.get("nodes", []): - if (node.get("cpu_used") is not None and node.get("cpu_total") - and node["cpu_total"] > 0): - node_cpu_pct = (node["cpu_used"] / node["cpu_total"]) * 100 - if node_cpu_pct > 85: - items.append( - f"{cluster}: Node {node['name']} CPU at {round(node_cpu_pct)}% (>85%)" - ) - if (node.get("memory_used") is not None and node.get("memory_total") - and node["memory_total"] > 0): - node_mem_pct = (node["memory_used"] / node["memory_total"]) * 100 - if node_mem_pct > 85: - items.append( - f"{cluster}: Node {node['name']} memory at {round(node_mem_pct)}% (>85%)" - ) - - pod_status = pc.get("pod_status", {}) - failed = pod_status.get("Failed", 0) + pod_status.get("Error", 0) - if failed > 0: - items.append(f"{cluster}: {failed} pods in Failed/Error state") - - unknown = pod_status.get("Unknown", 0) - if unknown > 0: - items.append(f"{cluster}: {unknown} pods in Unknown state") - - pending = pod_status.get("Pending", 0) - if pending > 0: - items.append(f"{cluster}: {pending} pods in Pending state (possible resource constraints)") - - crash = pod_status.get("CrashLoopBackOff", 0) - if crash > 0: - items.append(f"{cluster}: {crash} pods in CrashLoopBackOff") - - img_pull = pod_status.get("ImagePullBackOff", 0) + pod_status.get("ErrImagePull", 0) - if img_pull > 0: - items.append(f"{cluster}: {img_pull} pods with image pull errors") - - if not ov.get("metrics_available", True): - items.append(f"{cluster}: Metrics Server not available — no CPU/memory usage data") - - for err in pc.get("errors", []): - items.append(f"{cluster}: {err}") - - return items - - -def main(): - try: - raw = sys.stdin.read() - except Exception as e: - json.dump({"error": f"Failed to read stdin: {e}"}, sys.stdout, indent=2) - sys.exit(1) - - try: - data = json.loads(raw) - except json.JSONDecodeError as e: - json.dump({"error": f"Invalid JSON input: {e}"}, sys.stdout, indent=2) - sys.exit(1) - - clusters_input = data.get("clusters", {}) - if not clusters_input: - json.dump({"error": "No clusters found in input"}, sys.stdout, indent=2) - sys.exit(1) - - overview_list = [] - per_cluster = {} - failed_clusters = [] - - for ctx_name, cluster_data in clusters_input.items(): - cluster_data.setdefault("context", ctx_name) - result = process_cluster(cluster_data) - overview_list.append(result["overview"]) - per_cluster[ctx_name] = { - "nodes": result["nodes"], - "pod_status": result["pod_status"], - "top_namespaces": result["top_namespaces"], - "errors": result["errors"], - } - if result["errors"]: - for err in result["errors"]: - failed_clusters.append({ - "context": ctx_name, - "server": cluster_data.get("server", "unknown"), - "error": err, - }) - - clusters_reported = sum( - 1 for ov in overview_list - if ov["node_count"] > 0 or ov["pods_total"] > 0 or ov["project_count"] > 0 - ) - clusters_failed = len(overview_list) - clusters_reported - - totals = compute_totals(overview_list) - attention = detect_attention_items(overview_list, per_cluster) - - output = { - "generated_at": data.get("generated_at", ""), - "clusters_reported": clusters_reported, - "clusters_failed": clusters_failed, - "overview": overview_list, - "totals": totals, - "per_cluster": per_cluster, - "attention": attention, - "failed_clusters": failed_clusters, - } - - json.dump(output, sys.stdout, indent=2) - - -if __name__ == "__main__": - main() diff --git a/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/assemble.py b/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/assemble.py deleted file mode 100644 index ee0f9f67..00000000 --- a/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/assemble.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env python3 - -import json -import os -import subprocess -import sys - -DATA_FIELDS = ("nodes_top", "nodes_list", "projects", "namespaces", "pods") - - -def unwrap_persisted_output(raw_content): - try: - data = json.loads(raw_content) - except (json.JSONDecodeError, ValueError): - return raw_content - - if isinstance(data, list) and len(data) > 0: - if all(isinstance(item, dict) and "type" in item for item in data): - texts = [] - for item in data: - if item.get("type") == "text" and "text" in item: - texts.append(item["text"]) - if texts: - return "\n".join(texts) - return None - - return data - - -def resolve_file_ref(file_path): - if not os.path.exists(file_path): - return None, f"File not found: {file_path}" - - try: - with open(file_path, "r") as f: - raw = f.read() - except PermissionError: - return None, f"Permission denied reading: {file_path}" - except OSError as e: - return None, f"Error reading {file_path}: {e}" - - if not raw.strip(): - return None, f"Empty file: {file_path}" - - content = unwrap_persisted_output(raw) - - if content is None: - return None, f"No text content in envelope: {file_path}" - - return content, None - - -def resolve_cluster(cluster_data): - errors = list(cluster_data.get("errors", [])) - - for field in DATA_FIELDS: - value = cluster_data.get(field) - if isinstance(value, dict) and "$file" in value: - file_path = value["$file"] - content, error = resolve_file_ref(file_path) - if error: - cluster_data[field] = None - errors.append(error) - else: - cluster_data[field] = content - - cluster_data["errors"] = errors - return cluster_data - - -def main(): - aggregate_mode = "--aggregate" in sys.argv - - try: - raw = sys.stdin.read() - except Exception as e: - json.dump({"error": f"Failed to read stdin: {e}"}, sys.stdout, indent=2) - sys.exit(1) - - try: - manifest = json.loads(raw) - except json.JSONDecodeError as e: - json.dump({"error": f"Invalid manifest JSON: {e}"}, sys.stdout, indent=2) - sys.exit(1) - - clusters = manifest.get("clusters", {}) - for cluster_data in clusters.values(): - resolve_cluster(cluster_data) - - resolved_json = json.dumps(manifest, indent=2) - - if aggregate_mode: - script_dir = os.path.dirname(os.path.abspath(__file__)) - aggregate_script = os.path.join(script_dir, "aggregate.py") - proc = subprocess.run( - [sys.executable, aggregate_script], - input=resolved_json, - capture_output=True, - text=True, - ) - sys.stdout.write(proc.stdout) - if proc.stderr: - sys.stderr.write(proc.stderr) - sys.exit(proc.returncode) - else: - sys.stdout.write(resolved_json) - - -if __name__ == "__main__": - main() diff --git a/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/build-kubeconfig.py b/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/build-kubeconfig.py deleted file mode 100644 index a4e06bc2..00000000 --- a/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/build-kubeconfig.py +++ /dev/null @@ -1,446 +0,0 @@ -#!/usr/bin/env python3 -"""Multi-cluster kubeconfig builder for cluster-report. - -Two subcommands: - setup Apply RBAC and extract SA tokens for clusters you're logged into - build Build a merged kubeconfig from a clusters inventory file - -Usage: - python3 build-kubeconfig.py setup [--all-contexts] [--contexts ctx1,ctx2] - [--output-inventory ] - - python3 build-kubeconfig.py build --clusters - [--output ] [--verify] - -Requires: oc or kubectl, python3 (stdlib only) -""" - -import argparse -import base64 -import json -import os -import shutil -import subprocess -import sys -import time -from pathlib import Path - -SCRIPT_DIR = Path(__file__).resolve().parent -RBAC_MANIFEST = SCRIPT_DIR / "cluster-reporter-rbac.yaml" - -SA_NAMESPACE = "cluster-reporter-system" -SECRET_NAME = "cluster-reporter-token" - -DEFAULT_INVENTORY = Path.home() / ".ocp-clusters" / "clusters.json" -DEFAULT_OUTPUT = Path("/tmp/cluster-report-kubeconfig") - - -def find_kube_cmd(): - """Detect oc (preferred) or kubectl in PATH.""" - if shutil.which("oc"): - return "oc" - if shutil.which("kubectl"): - print("WARNING: 'oc' not found – falling back to 'kubectl'. " - "Install the OpenShift CLI (oc) for full compatibility: " - "https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/", - file=sys.stderr) - return "kubectl" - print('{"error": "Neither oc nor kubectl found in PATH. ' - 'Install oc: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/"}', - file=sys.stderr) - sys.exit(1) - - -# --------------------------------------------------------------------------- -# Setup mode -# --------------------------------------------------------------------------- - -def run_setup(args): - kube_cmd = find_kube_cmd() - inventory_file = Path(args.output_inventory) - - if not args.skip_rbac and not RBAC_MANIFEST.is_file(): - print(f"Error: RBAC manifest not found at {RBAC_MANIFEST}", file=sys.stderr) - sys.exit(1) - - try: - all_ctx = subprocess.check_output( - [kube_cmd, "config", "get-contexts", "-o", "name"], - text=True, stderr=subprocess.DEVNULL - ).strip().splitlines() - except subprocess.CalledProcessError: - all_ctx = [] - - if not all_ctx: - print('{"error": "No kubeconfig contexts found. Log in to at least one cluster first."}', - file=sys.stderr) - sys.exit(1) - - if args.contexts: - contexts = args.contexts.split(",") - unknown = [c for c in contexts if c not in all_ctx] - if unknown: - print(f"Error: unknown context(s): {', '.join(unknown)}", file=sys.stderr) - print(f"Available: {', '.join(all_ctx)}", file=sys.stderr) - sys.exit(1) - elif args.all_contexts: - contexts = all_ctx - else: - print("Available contexts:") - for i, ctx in enumerate(all_ctx, 1): - print(f" {i}. {ctx}") - print() - print("Run with --all-contexts to setup all, or --contexts ctx1,ctx2 to select specific ones.") - sys.exit(0) - - print(f"Pre-flight: checking {len(contexts)} cluster(s)...\n") - reachable = {} - for ctx in contexts: - server = _get_server_url(kube_cmd, ctx) - if not server: - print(f" {ctx}: SKIP (no server URL in kubeconfig)") - continue - try: - subprocess.run( - [kube_cmd, "cluster-info", "--context", ctx], - capture_output=True, text=True, timeout=15, check=True - ) - reachable[ctx] = server - print(f" {ctx}: reachable ({server})") - except (subprocess.CalledProcessError, subprocess.TimeoutExpired): - print(f" {ctx}: SKIP (unreachable – try '{kube_cmd} login {server}' first)") - continue - - if not args.skip_rbac: - try: - result = subprocess.run( - [kube_cmd, "auth", "can-i", "create", "clusterroles", - "--context", ctx], - capture_output=True, text=True, timeout=10 - ) - if result.stdout.strip().lower() != "yes": - print(f" {ctx}: SKIP (insufficient permissions – " - f"cluster-admin required for RBAC setup, " - f"or use --skip-rbac if RBAC is already applied)") - del reachable[ctx] - except (subprocess.CalledProcessError, subprocess.TimeoutExpired): - print(f" {ctx}: SKIP (could not verify permissions)") - del reachable[ctx] - - if not reachable: - print("\nError: no eligible clusters found. Nothing to do.", file=sys.stderr) - sys.exit(1) - - print(f"\n{len(reachable)}/{len(contexts)} cluster(s) ready. " - f"Proceeding with setup...\n") - - inventory_file.parent.mkdir(parents=True, exist_ok=True) - - existing_by_name = {} - if inventory_file.is_file(): - try: - with open(inventory_file) as f: - existing_by_name = {c["name"]: c for c in json.load(f).get("clusters", [])} - except (json.JSONDecodeError, KeyError): - pass - - results = {"setup": [], "errors": []} - - for ctx, server in reachable.items(): - print(f"--- {ctx} ---") - print(f" Server: {server}") - - if args.skip_rbac: - print(" Skipping RBAC apply (--skip-rbac)") - else: - print(" Applying RBAC...") - try: - subprocess.run( - [kube_cmd, "apply", "-f", str(RBAC_MANIFEST), "--context", ctx], - capture_output=True, text=True, timeout=30, check=True - ) - except subprocess.CalledProcessError as e: - results["errors"].append(f"{ctx}: RBAC apply failed: {e.stderr.strip()}") - print(f" FAIL: RBAC apply failed: {e.stderr.strip()}") - continue - - print(" Waiting for token...") - token = _wait_for_token(kube_cmd, ctx) - if not token: - results["errors"].append(f"{ctx}: token not populated after 15s") - print(" FAIL: token Secret not populated") - continue - - try: - decoded_token = base64.b64decode(token).decode("utf-8") - except Exception: - decoded_token = token - - existing_by_name[ctx] = {"name": ctx, "api_url": server, "token": decoded_token} - results["setup"].append(ctx) - print(" OK: token extracted") - - with open(inventory_file, "w") as f: - json.dump({"clusters": list(existing_by_name.values())}, f, indent=2) - os.chmod(inventory_file, 0o600) - - print() - print("=" * 50) - print(f"Setup complete: {len(results['setup'])} succeeded, {len(results['errors'])} failed") - if results["errors"]: - print("Errors:") - for e in results["errors"]: - print(f" - {e}") - print(f"Inventory written to: {inventory_file}") - print() - print("Next step:") - print(f" python3 {__file__} build --clusters {inventory_file} --verify") - - json.dump(results, sys.stderr, indent=2) - - -def _get_server_url(kube_cmd, ctx): - """Resolve the API server URL for a kubeconfig context.""" - try: - server = subprocess.check_output( - [kube_cmd, "config", "view", "-o", - f'jsonpath={{.clusters[?(@.name=="{ctx}")].cluster.server}}'], - text=True, stderr=subprocess.DEVNULL - ).strip() - if server: - return server - - cluster_ref = subprocess.check_output( - [kube_cmd, "config", "view", "-o", - f'jsonpath={{.contexts[?(@.name=="{ctx}")].context.cluster}}'], - text=True, stderr=subprocess.DEVNULL - ).strip() - if cluster_ref: - return subprocess.check_output( - [kube_cmd, "config", "view", "-o", - f'jsonpath={{.clusters[?(@.name=="{cluster_ref}")].cluster.server}}'], - text=True, stderr=subprocess.DEVNULL - ).strip() or None - except subprocess.CalledProcessError: - pass - return None - - -def _wait_for_token(kube_cmd, ctx, timeout_secs=15): - """Poll for the SA token Secret to be populated.""" - for _ in range(timeout_secs): - try: - token = subprocess.check_output( - [kube_cmd, "get", "secret", SECRET_NAME, - "-n", SA_NAMESPACE, "--context", ctx, - "-o", "jsonpath={.data.token}"], - text=True, stderr=subprocess.DEVNULL, timeout=10 - ).strip() - if token: - return token - except (subprocess.CalledProcessError, subprocess.TimeoutExpired): - pass - time.sleep(1) - return None - - -# --------------------------------------------------------------------------- -# Build mode -# --------------------------------------------------------------------------- - -def run_build(args): - kube_cmd = find_kube_cmd() - clusters_file = Path(args.clusters) - output_file = Path(args.output) - - if not clusters_file.is_file(): - print(f'{{"error": "Clusters file not found: {clusters_file}"}}', file=sys.stderr) - sys.exit(1) - - with open(clusters_file) as f: - config = json.load(f) - - clusters = config.get("clusters", []) - if not clusters: - print('{"error": "No clusters in inventory file"}', file=sys.stderr) - sys.exit(1) - - output_file.unlink(missing_ok=True) - output_file.touch(mode=0o600) - - env = {**os.environ, "KUBECONFIG": str(output_file)} - errors = [] - success = 0 - - for c in clusters: - name = c.get("name", "") - api_url = c.get("api_url", "") - - if not name or not api_url: - errors.append(f"Entry missing name or api_url: {c}") - continue - - token = _resolve_token(c, errors) - if token is None: - continue - - ca_args = (["--certificate-authority", c["ca_cert"]] - if c.get("ca_cert") - else ["--insecure-skip-tls-verify=true"]) - try: - subprocess.run( - [kube_cmd, "config", "set-cluster", name, "--server", api_url] + ca_args, - check=True, capture_output=True, env=env - ) - except subprocess.CalledProcessError as e: - errors.append(f"{name}: set-cluster failed: {e.stderr.decode().strip()}") - continue - - try: - subprocess.run( - [kube_cmd, "config", "set-credentials", f"{name}-reporter", "--token", token], - check=True, capture_output=True, env=env - ) - except subprocess.CalledProcessError as e: - errors.append(f"{name}: set-credentials failed: {e.stderr.decode().strip()}") - continue - - try: - subprocess.run( - [kube_cmd, "config", "set-context", name, - "--cluster", name, "--user", f"{name}-reporter"], - check=True, capture_output=True, env=env - ) - except subprocess.CalledProcessError as e: - errors.append(f"{name}: set-context failed: {e.stderr.decode().strip()}") - continue - - if success == 0: - subprocess.run( - [kube_cmd, "config", "use-context", name], - check=False, capture_output=True, env=env - ) - - success += 1 - - verify_results = {} - if args.verify and success > 0: - print(f"Verifying {success} context(s)...") - for c in clusters: - name = c.get("name", "") - if not name: - continue - try: - subprocess.run( - [kube_cmd, "get", "nodes", "--context", name, "-o", "name", "--no-headers"], - capture_output=True, text=True, timeout=15, check=True, env=env - ) - verify_results[name] = "ok" - print(f" {name}: OK") - except subprocess.TimeoutExpired: - verify_results[name] = "timeout" - errors.append(f"{name}: verification timed out") - print(f" {name}: TIMEOUT") - except subprocess.CalledProcessError: - verify_results[name] = "failed" - errors.append(f"{name}: verification failed (likely expired token)") - print(f" {name}: FAILED (re-run setup for this cluster)") - - result = { - "clusters_configured": success, - "clusters_failed": len(errors), - "kubeconfig": str(output_file), - "errors": errors, - } - if args.verify: - result["verification"] = verify_results - - print() - print(json.dumps(result, indent=2)) - print() - print(f"Kubeconfig written to: {output_file}") - print() - print("To use with cluster-report:") - print(f" export KUBECONFIG={output_file}") - - if success == 0: - sys.exit(1) - - -def _resolve_token(cluster_entry, errors): - """Resolve token from inline value or environment variable. Returns None on failure.""" - name = cluster_entry.get("name", "") - if "token_env" in cluster_entry: - token = os.environ.get(cluster_entry["token_env"]) - if not token: - errors.append(f"{name}: env var {cluster_entry['token_env']} not set") - return None - return token - if "token" in cluster_entry: - return cluster_entry["token"] - errors.append(f"{name}: no token or token_env specified") - return None - - -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - -def main(): - parser = argparse.ArgumentParser( - description="Multi-cluster kubeconfig builder for cluster-report", - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - subparsers = parser.add_subparsers(dest="command", required=True) - - # -- setup -- - setup_parser = subparsers.add_parser( - "setup", - help="Apply RBAC to clusters you're logged into, extract SA tokens, " - "and write a clusters inventory file.", - ) - setup_parser.add_argument( - "--all-contexts", action="store_true", - help="Setup all kubeconfig contexts without prompting.", - ) - setup_parser.add_argument( - "--contexts", type=str, default=None, - help="Comma-separated list of contexts to setup.", - ) - setup_parser.add_argument( - "--skip-rbac", action="store_true", - help="Skip RBAC apply and only extract tokens (use when RBAC is already configured).", - ) - setup_parser.add_argument( - "--output-inventory", type=str, default=str(DEFAULT_INVENTORY), - help=f"Path for the clusters inventory file (default: {DEFAULT_INVENTORY}).", - ) - - # -- build -- - build_parser = subparsers.add_parser( - "build", - help="Read a clusters inventory file and build a merged kubeconfig.", - ) - build_parser.add_argument( - "--clusters", type=str, required=True, - help="Path to the clusters inventory JSON file.", - ) - build_parser.add_argument( - "--output", type=str, default=str(DEFAULT_OUTPUT), - help=f"Path for the generated kubeconfig (default: {DEFAULT_OUTPUT}).", - ) - build_parser.add_argument( - "--verify", action="store_true", - help="Test each context after building the kubeconfig.", - ) - - args = parser.parse_args() - - if args.command == "setup": - run_setup(args) - elif args.command == "build": - run_build(args) - - -if __name__ == "__main__": - main() diff --git a/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/cluster-reporter-rbac.yaml b/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/cluster-reporter-rbac.yaml deleted file mode 100644 index 4fe9eeb8..00000000 --- a/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/cluster-reporter-rbac.yaml +++ /dev/null @@ -1,72 +0,0 @@ ---- -# cluster-reporter-rbac.yaml -# One-time per-cluster setup for multi-cluster reporting with SA tokens. -# Apply with: oc apply -f cluster-reporter-rbac.yaml -# -# Creates a read-only ServiceAccount with the minimum permissions required -# by the cluster-report skill. The token Secret does not expire until deleted. -# -# After applying, extract the token: -# oc get secret cluster-reporter-token -n cluster-reporter-system \ -# -o jsonpath='{.data.token}' | base64 -d - -apiVersion: v1 -kind: Namespace -metadata: - name: cluster-reporter-system - labels: - app.kubernetes.io/part-of: cluster-reporter ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cluster-reporter - namespace: cluster-reporter-system - labels: - app.kubernetes.io/part-of: cluster-reporter ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cluster-reporter-readonly - labels: - app.kubernetes.io/part-of: cluster-reporter -rules: - - apiGroups: [""] - resources: ["nodes", "namespaces", "pods"] - verbs: ["get", "list"] - - apiGroups: ["config.openshift.io"] - resources: ["clusterversions"] - verbs: ["get"] - - apiGroups: ["project.openshift.io"] - resources: ["projects"] - verbs: ["list"] - - apiGroups: ["metrics.k8s.io"] - resources: ["nodes", "pods"] - verbs: ["get", "list"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cluster-reporter-binding - labels: - app.kubernetes.io/part-of: cluster-reporter -subjects: - - kind: ServiceAccount - name: cluster-reporter - namespace: cluster-reporter-system -roleRef: - kind: ClusterRole - name: cluster-reporter-readonly - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: Secret -metadata: - name: cluster-reporter-token - namespace: cluster-reporter-system - annotations: - kubernetes.io/service-account.name: cluster-reporter - labels: - app.kubernetes.io/part-of: cluster-reporter -type: kubernetes.io/service-account-token diff --git a/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/test_aggregate.py b/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/test_aggregate.py deleted file mode 100644 index db3fa535..00000000 --- a/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/test_aggregate.py +++ /dev/null @@ -1,863 +0,0 @@ -#!/usr/bin/env python3 - -import json -import subprocess -import sys -import unittest -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).parent)) -import aggregate - - -class TestParseCpu(unittest.TestCase): - def test_whole_cores(self): - self.assertEqual(aggregate.parse_cpu("4"), 4.0) - - def test_millicores(self): - self.assertEqual(aggregate.parse_cpu("500m"), 0.5) - - def test_millicores_whole(self): - self.assertEqual(aggregate.parse_cpu("4000m"), 4.0) - - def test_nanocores(self): - self.assertAlmostEqual(aggregate.parse_cpu("1000000000n"), 1.0) - - def test_microcores(self): - self.assertAlmostEqual(aggregate.parse_cpu("1000000u"), 1.0) - - def test_none(self): - self.assertEqual(aggregate.parse_cpu(None), 0.0) - - def test_integer(self): - self.assertEqual(aggregate.parse_cpu(8), 8.0) - - def test_fractional(self): - self.assertEqual(aggregate.parse_cpu("0.5"), 0.5) - - -class TestParseMemory(unittest.TestCase): - def test_gibibytes(self): - self.assertEqual(aggregate.parse_memory("16Gi"), 16.0) - - def test_mebibytes(self): - self.assertEqual(aggregate.parse_memory("16384Mi"), 16.0) - - def test_kibibytes(self): - self.assertAlmostEqual(aggregate.parse_memory("16777216Ki"), 16.0) - - def test_raw_bytes(self): - self.assertAlmostEqual(aggregate.parse_memory("17179869184"), 16.0, places=0) - - def test_tebibytes(self): - self.assertEqual(aggregate.parse_memory("1Ti"), 1024.0) - - def test_decimal_gigabytes(self): - val = aggregate.parse_memory("16G") - self.assertAlmostEqual(val, 16000000000 / (1024 ** 3), places=1) - - def test_none(self): - self.assertEqual(aggregate.parse_memory(None), 0.0) - - -class TestDetectNodeRole(unittest.TestCase): - def test_worker(self): - labels = {"node-role.kubernetes.io/worker": ""} - self.assertEqual(aggregate.detect_node_role(labels), "worker") - - def test_control_plane(self): - labels = {"node-role.kubernetes.io/control-plane": ""} - self.assertEqual(aggregate.detect_node_role(labels), "control-plane") - - def test_master(self): - labels = {"node-role.kubernetes.io/master": ""} - self.assertEqual(aggregate.detect_node_role(labels), "master") - - def test_infra(self): - labels = {"node-role.kubernetes.io/infra": ""} - self.assertEqual(aggregate.detect_node_role(labels), "infra") - - def test_multiple_roles_prefers_control_plane(self): - labels = { - "node-role.kubernetes.io/worker": "", - "node-role.kubernetes.io/control-plane": "", - } - self.assertEqual(aggregate.detect_node_role(labels), "control-plane") - - def test_no_role_labels(self): - labels = {"kubernetes.io/hostname": "node-1"} - self.assertEqual(aggregate.detect_node_role(labels), "worker") - - def test_empty_labels(self): - self.assertEqual(aggregate.detect_node_role({}), "worker") - - def test_none_labels(self): - self.assertEqual(aggregate.detect_node_role(None), "worker") - - -class TestDetectGpus(unittest.TestCase): - def test_nvidia_gpu(self): - alloc = {"cpu": "8", "memory": "32Gi", "nvidia.com/gpu": "2"} - count, gpu_type = aggregate.detect_gpus(alloc) - self.assertEqual(count, 2) - self.assertEqual(gpu_type, "nvidia.com/gpu") - - def test_amd_gpu(self): - alloc = {"amd.com/gpu": "4"} - count, gpu_type = aggregate.detect_gpus(alloc) - self.assertEqual(count, 4) - self.assertEqual(gpu_type, "amd.com/gpu") - - def test_intel_gpu(self): - alloc = {"intel.com/gpu": "1"} - count, gpu_type = aggregate.detect_gpus(alloc) - self.assertEqual(count, 1) - self.assertEqual(gpu_type, "intel.com/gpu") - - def test_no_gpus(self): - alloc = {"cpu": "8", "memory": "32Gi"} - count, gpu_type = aggregate.detect_gpus(alloc) - self.assertEqual(count, 0) - self.assertEqual(gpu_type, "") - - def test_zero_gpus(self): - alloc = {"nvidia.com/gpu": "0"} - count, gpu_type = aggregate.detect_gpus(alloc) - self.assertEqual(count, 0) - self.assertEqual(gpu_type, "") - - def test_none_allocatable(self): - count, gpu_type = aggregate.detect_gpus(None) - self.assertEqual(count, 0) - self.assertEqual(gpu_type, "") - - -class TestParseTabular(unittest.TestCase): - def test_basic_table(self): - text = "NAME STATUS\nnode-1 Ready\nnode-2 NotReady" - result = aggregate.parse_tabular(text) - self.assertEqual(len(result), 2) - self.assertEqual(result[0]["NAME"], "node-1") - self.assertEqual(result[0]["STATUS"], "Ready") - self.assertEqual(result[1]["NAME"], "node-2") - self.assertEqual(result[1]["STATUS"], "NotReady") - - def test_multiword_header(self): - text = "NAME DISPLAY NAME STATUS\nproj-1 My Project Active" - result = aggregate.parse_tabular(text) - self.assertEqual(len(result), 1) - self.assertIn("DISPLAY NAME", result[0]) - self.assertEqual(result[0]["DISPLAY NAME"], "My Project") - - def test_empty_input(self): - self.assertEqual(aggregate.parse_tabular(""), []) - self.assertEqual(aggregate.parse_tabular(None), []) - - def test_header_only(self): - self.assertEqual(aggregate.parse_tabular("NAME STATUS"), []) - - def test_short_data_line(self): - text = "NAME STATUS LABELS\nnode-1 Ready" - result = aggregate.parse_tabular(text) - self.assertEqual(result[0]["NAME"], "node-1") - self.assertEqual(result[0]["STATUS"], "Ready") - self.assertEqual(result[0]["LABELS"], "") - - def test_varying_column_widths(self): - text = "NAME STATUS AGE\nshort OK 1d\nvery-long Fail 30d" - result = aggregate.parse_tabular(text) - self.assertEqual(result[0]["NAME"], "short") - self.assertEqual(result[1]["NAME"], "very-long") - self.assertEqual(result[0]["STATUS"], "OK") - self.assertEqual(result[1]["STATUS"], "Fail") - - def test_blank_lines_skipped(self): - text = "NAME STATUS\n\nnode-1 Ready\n\n" - result = aggregate.parse_tabular(text) - self.assertEqual(len(result), 1) - - def test_real_mcp_pod_header(self): - text = ( - "NAMESPACE APIVERSION KIND NAME " - "READY STATUS RESTARTS AGE\n" - "openshift-dns v1 Pod dns-default-abc12 " - "1/1 Running 0 5d\n" - "aistor v1 Pod webhook-69496784f7 " - "0/1 ErrImagePull 0 4d" - ) - result = aggregate.parse_tabular(text) - self.assertEqual(len(result), 2) - self.assertEqual(result[0]["NAMESPACE"], "openshift-dns") - self.assertEqual(result[0]["STATUS"], "Running") - self.assertEqual(result[1]["STATUS"], "ErrImagePull") - - -class TestParseLabelsString(unittest.TestCase): - def test_basic_labels(self): - result = aggregate.parse_labels_string( - "node-role.kubernetes.io/worker=,kubernetes.io/hostname=node-1" - ) - self.assertEqual(result["node-role.kubernetes.io/worker"], "") - self.assertEqual(result["kubernetes.io/hostname"], "node-1") - - def test_empty_string(self): - self.assertEqual(aggregate.parse_labels_string(""), {}) - - def test_none(self): - self.assertEqual(aggregate.parse_labels_string(None), {}) - - def test_label_with_value(self): - result = aggregate.parse_labels_string("beta.kubernetes.io/arch=amd64") - self.assertEqual(result["beta.kubernetes.io/arch"], "amd64") - - def test_none_literal(self): - self.assertEqual(aggregate.parse_labels_string(""), {}) - - -class TestParsePodsTabular(unittest.TestCase): - def test_basic_pods(self): - text = ( - "NAMESPACE APIVERSION KIND NAME " - "READY STATUS RESTARTS AGE\n" - "openshift-mon v1 Pod prometheus-0 " - "1/1 Running 0 5d\n" - "default v1 Pod failing-pod " - "0/1 CrashLoopBackOff 15 1d" - ) - result = aggregate.parse_pods_tabular(text) - self.assertEqual(len(result), 2) - self.assertEqual(result[0]["namespace"], "openshift-mon") - self.assertEqual(result[0]["status"], "Running") - self.assertEqual(result[1]["namespace"], "default") - self.assertEqual(result[1]["status"], "CrashLoopBackOff") - - def test_empty_input(self): - self.assertEqual(aggregate.parse_pods_tabular(""), []) - - def test_various_statuses(self): - for status in ["Running", "Pending", "Failed", "Succeeded", - "ErrImagePull", "ImagePullBackOff", "Completed"]: - text = f"NAMESPACE NAME STATUS\ndefault pod-x {status}" - result = aggregate.parse_pods_tabular(text) - self.assertEqual(result[0]["status"], status, f"Failed for {status}") - - -class TestParseNodesListTabular(unittest.TestCase): - def test_basic_nodes(self): - text = ( - "APIVERSION KIND NAME STATUS ROLES AGE " - "VERSION LABELS\n" - "v1 Node worker-0 Ready worker 30d " - "v1.28 node-role.kubernetes.io/worker=,kubernetes.io/hostname=worker-0" - ) - result = aggregate.parse_nodes_list_tabular(text) - self.assertEqual(len(result), 1) - self.assertEqual(result[0]["metadata"]["name"], "worker-0") - self.assertIn("node-role.kubernetes.io/worker", - result[0]["metadata"]["labels"]) - - def test_role_from_roles_column(self): - text = ( - "APIVERSION KIND NAME STATUS ROLES AGE " - "VERSION LABELS\n" - "v1 Node master-0 Ready control-plane 30d " - "v1.28 kubernetes.io/hostname=master-0" - ) - result = aggregate.parse_nodes_list_tabular(text) - labels = result[0]["metadata"]["labels"] - role = aggregate.detect_node_role(labels) - self.assertEqual(role, "control-plane") - - def test_no_allocatable_data(self): - text = "APIVERSION KIND NAME STATUS ROLES LABELS\nv1 Node n1 Ready worker app=test" - result = aggregate.parse_nodes_list_tabular(text) - self.assertEqual(result[0]["status"], {}) - - -class TestParseNodesTopTabular(unittest.TestCase): - def test_basic_top(self): - text = ( - "NAME CPU(cores) CPU(%) MEMORY(bytes) MEMORY(%)\n" - "node-1 4000m 50% 16Gi 50%\n" - "node-2 2000m 25% 8Gi 25%" - ) - result = aggregate.parse_nodes_top_tabular(text) - self.assertEqual(len(result), 2) - self.assertEqual(result[0]["name"], "node-1") - self.assertEqual(result[0]["cpu_usage"], "4000m") - self.assertEqual(result[0]["memory_usage"], "16Gi") - self.assertEqual(result[1]["name"], "node-2") - - def test_empty(self): - self.assertEqual(aggregate.parse_nodes_top_tabular(""), []) - - -class TestParseProjectsTabular(unittest.TestCase): - def test_basic(self): - text = ( - "APIVERSION KIND NAME DISPLAY NAME STATUS LABELS\n" - "project.openshift.io/v1 Project my-project My Project Active app=test\n" - "project.openshift.io/v1 Project default Active " - ) - result = aggregate.parse_projects_tabular(text) - self.assertEqual(len(result), 2) - self.assertEqual(result[0]["name"], "my-project") - self.assertEqual(result[1]["name"], "default") - - def test_empty(self): - self.assertEqual(aggregate.parse_projects_tabular(""), []) - - -class TestParseNamespacesTabular(unittest.TestCase): - def test_basic(self): - text = ( - "APIVERSION KIND NAME STATUS AGE LABELS\n" - "v1 Namespace kube-system Active 90d kubernetes.io/metadata.name=kube-system\n" - "v1 Namespace default Active 90d kubernetes.io/metadata.name=default" - ) - result = aggregate.parse_namespaces_tabular(text) - self.assertEqual(len(result), 2) - self.assertEqual(result[0]["name"], "kube-system") - self.assertEqual(result[1]["name"], "default") - - -class TestClassifyPodStatus(unittest.TestCase): - def test_running(self): - pod = {"status": {"phase": "Running", "containerStatuses": []}} - self.assertEqual(aggregate.classify_pod_status(pod), "Running") - - def test_pending(self): - pod = {"status": {"phase": "Pending"}} - self.assertEqual(aggregate.classify_pod_status(pod), "Pending") - - def test_succeeded(self): - pod = {"status": {"phase": "Succeeded"}} - self.assertEqual(aggregate.classify_pod_status(pod), "Succeeded") - - def test_failed(self): - pod = {"status": {"phase": "Failed"}} - self.assertEqual(aggregate.classify_pod_status(pod), "Failed") - - def test_completed_maps_to_succeeded(self): - pod = {"status": {"phase": "Completed"}} - self.assertEqual(aggregate.classify_pod_status(pod), "Succeeded") - - def test_crashloopbackoff_override(self): - pod = { - "status": { - "phase": "Running", - "containerStatuses": [ - {"state": {"waiting": {"reason": "CrashLoopBackOff"}}} - ], - } - } - self.assertEqual(aggregate.classify_pod_status(pod), "CrashLoopBackOff") - - def test_imagepullbackoff_override(self): - pod = { - "status": { - "phase": "Pending", - "containerStatuses": [ - {"state": {"waiting": {"reason": "ImagePullBackOff"}}} - ], - } - } - self.assertEqual(aggregate.classify_pod_status(pod), "ImagePullBackOff") - - def test_errimagepull(self): - pod = { - "status": { - "phase": "Pending", - "containerStatuses": [ - {"state": {"waiting": {"reason": "ErrImagePull"}}} - ], - } - } - self.assertEqual(aggregate.classify_pod_status(pod), "ErrImagePull") - - def test_flat_status_string(self): - pod = {"name": "my-pod", "namespace": "default", "status": "CrashLoopBackOff"} - self.assertEqual(aggregate.classify_pod_status(pod), "CrashLoopBackOff") - - def test_unknown_default(self): - pod = {"status": {}} - self.assertEqual(aggregate.classify_pod_status(pod), "Unknown") - - -class TestAggregatePodsByNamespace(unittest.TestCase): - def test_basic_aggregation(self): - pods = [ - {"metadata": {"namespace": "ns-a"}, "status": {"phase": "Running"}}, - {"metadata": {"namespace": "ns-a"}, "status": {"phase": "Running"}}, - {"metadata": {"namespace": "ns-b"}, "status": {"phase": "Pending"}}, - ] - result = aggregate.aggregate_pods_by_namespace(pods) - self.assertEqual(len(result), 2) - self.assertEqual(result[0]["namespace"], "ns-a") - self.assertEqual(result[0]["pods_total"], 2) - self.assertEqual(result[0]["running"], 2) - self.assertEqual(result[1]["namespace"], "ns-b") - self.assertEqual(result[1]["pods_total"], 1) - self.assertEqual(result[1]["pending"], 1) - - def test_top_10_limit(self): - pods = [] - for i in range(15): - for j in range(15 - i): - pods.append({ - "metadata": {"namespace": f"ns-{i:02d}"}, - "status": {"phase": "Running"}, - }) - result = aggregate.aggregate_pods_by_namespace(pods) - self.assertEqual(len(result), 10) - self.assertEqual(result[0]["pods_total"], 15) - self.assertEqual(result[9]["pods_total"], 6) - - def test_empty_pods(self): - self.assertEqual(aggregate.aggregate_pods_by_namespace([]), []) - self.assertEqual(aggregate.aggregate_pods_by_namespace(None), []) - - def test_flat_pod_structure(self): - pods = [ - {"namespace": "ns-a", "status": "Running"}, - {"namespace": "ns-a", "status": "Failed"}, - ] - result = aggregate.aggregate_pods_by_namespace(pods) - self.assertEqual(len(result), 1) - self.assertEqual(result[0]["running"], 1) - self.assertEqual(result[0]["failed"], 1) - - -class TestProcessCluster(unittest.TestCase): - def _make_cluster(self, **overrides): - base = { - "context": "test-cluster", - "server": "https://api.test.example.com:6443", - "nodes_top": None, - "nodes_list": None, - "projects": None, - "namespaces": None, - "pods": None, - "errors": [], - } - base.update(overrides) - return base - - def test_full_data(self): - cluster = self._make_cluster( - nodes_top=[ - {"name": "node-1", "cpu_usage": "4000m", "memory_usage": "16Gi"}, - ], - nodes_list=[ - { - "metadata": { - "name": "node-1", - "labels": {"node-role.kubernetes.io/worker": ""}, - }, - "status": { - "allocatable": {"cpu": "8", "memory": "32Gi", "nvidia.com/gpu": "2"}, - }, - } - ], - projects=[{"name": f"proj-{i}"} for i in range(5)], - pods=[ - {"metadata": {"namespace": "default"}, "status": {"phase": "Running"}}, - {"metadata": {"namespace": "default"}, "status": {"phase": "Running"}}, - {"metadata": {"namespace": "kube-system"}, "status": {"phase": "Failed"}}, - ], - ) - result = aggregate.process_cluster(cluster) - - ov = result["overview"] - self.assertEqual(ov["node_count"], 1) - self.assertEqual(ov["cpu_used_cores"], 4.0) - self.assertEqual(ov["cpu_total_cores"], 8.0) - self.assertEqual(ov["cpu_percent"], 50) - self.assertEqual(ov["gpu_total"], 2) - self.assertEqual(ov["project_count"], 5) - self.assertEqual(ov["pods_running"], 2) - self.assertEqual(ov["pods_total"], 3) - self.assertTrue(ov["metrics_available"]) - - self.assertEqual(result["pod_status"]["Running"], 2) - self.assertEqual(result["pod_status"]["Failed"], 1) - self.assertEqual(len(result["top_namespaces"]), 2) - - def test_no_metrics_server(self): - cluster = self._make_cluster( - nodes_top=None, - nodes_list=[ - { - "metadata": {"name": "node-1", "labels": {}}, - "status": {"allocatable": {"cpu": "8", "memory": "32Gi"}}, - } - ], - ) - result = aggregate.process_cluster(cluster) - - ov = result["overview"] - self.assertFalse(ov["metrics_available"]) - self.assertIsNone(ov["cpu_used_cores"]) - self.assertIsNone(ov["cpu_percent"]) - self.assertEqual(ov["cpu_total_cores"], 8.0) - - def test_empty_cluster(self): - cluster = self._make_cluster() - result = aggregate.process_cluster(cluster) - - ov = result["overview"] - self.assertEqual(ov["node_count"], 0) - self.assertEqual(ov["pods_total"], 0) - self.assertEqual(ov["project_count"], 0) - self.assertEqual(result["pod_status"], {}) - self.assertEqual(result["top_namespaces"], []) - - def test_namespaces_fallback(self): - cluster = self._make_cluster( - projects=None, - namespaces=[{"name": f"ns-{i}"} for i in range(3)], - ) - result = aggregate.process_cluster(cluster) - self.assertEqual(result["overview"]["project_count"], 3) - - def test_tabular_pods_input(self): - tabular_pods = ( - "NAMESPACE APIVERSION KIND NAME " - "READY STATUS RESTARTS AGE\n" - "default v1 Pod pod-1 " - "1/1 Running 0 1d\n" - "default v1 Pod pod-2 " - "1/1 Running 0 1d\n" - "kube-system v1 Pod pod-3 " - "0/1 Failed 5 3d" - ) - cluster = self._make_cluster(pods=tabular_pods) - result = aggregate.process_cluster(cluster) - self.assertEqual(result["overview"]["pods_total"], 3) - self.assertEqual(result["overview"]["pods_running"], 2) - self.assertEqual(result["pod_status"]["Running"], 2) - self.assertEqual(result["pod_status"]["Failed"], 1) - self.assertEqual(len(result["top_namespaces"]), 2) - - def test_tabular_projects_input(self): - tabular_projects = ( - "APIVERSION KIND NAME DISPLAY NAME STATUS LABELS\n" - "project.openshift.io/v1 Project proj-1 Project 1 Active \n" - "project.openshift.io/v1 Project proj-2 Project 2 Active \n" - "project.openshift.io/v1 Project proj-3 Active " - ) - cluster = self._make_cluster(projects=tabular_projects) - result = aggregate.process_cluster(cluster) - self.assertEqual(result["overview"]["project_count"], 3) - - def test_mixed_tabular_and_json(self): - tabular_pods = ( - "NAMESPACE NAME STATUS\n" - "default pod-1 Running" - ) - json_nodes = [{ - "metadata": {"name": "n1", "labels": {}}, - "status": {"allocatable": {"cpu": "4", "memory": "16Gi"}}, - }] - cluster = self._make_cluster( - pods=tabular_pods, nodes_list=json_nodes, - projects=[{"name": "proj-1"}]) - result = aggregate.process_cluster(cluster) - self.assertEqual(result["overview"]["pods_total"], 1) - self.assertEqual(result["overview"]["node_count"], 1) - self.assertEqual(result["overview"]["project_count"], 1) - - def test_tabular_nodes_list_known_limitation(self): - tabular_nodes = ( - "APIVERSION KIND NAME STATUS ROLES AGE " - "VERSION LABELS\n" - "v1 Node worker-0 Ready worker 30d " - "v1.28 node-role.kubernetes.io/worker=" - ) - cluster = self._make_cluster(nodes_list=tabular_nodes) - result = aggregate.process_cluster(cluster) - self.assertEqual(result["overview"]["node_count"], 1) - self.assertEqual(result["overview"]["cpu_total_cores"], 0.0) - self.assertEqual(result["overview"]["memory_total_gib"], 0.0) - self.assertEqual(result["overview"]["gpu_total"], 0) - self.assertEqual(result["nodes"][0]["role"], "worker") - - def test_tabular_nodes_top_with_json_nodes_list(self): - tabular_top = ( - "NAME CPU(cores) CPU(%) MEMORY(bytes) MEMORY(%)\n" - "node-1 4000m 50% 16Gi 50%" - ) - json_nodes = [{ - "metadata": {"name": "node-1", - "labels": {"node-role.kubernetes.io/worker": ""}}, - "status": {"allocatable": {"cpu": "8", "memory": "32Gi"}}, - }] - cluster = self._make_cluster( - nodes_top=tabular_top, nodes_list=json_nodes) - result = aggregate.process_cluster(cluster) - self.assertEqual(result["overview"]["cpu_used_cores"], 4.0) - self.assertEqual(result["overview"]["cpu_total_cores"], 8.0) - self.assertTrue(result["overview"]["metrics_available"]) - - -class TestComputeTotals(unittest.TestCase): - def test_two_clusters(self): - overview = [ - { - "node_count": 3, "cpu_used_cores": 10.0, "cpu_total_cores": 24.0, - "memory_used_gib": 40.0, "memory_total_gib": 96.0, - "gpu_total": 2, "project_count": 10, - "pods_running": 50, "pods_total": 60, - }, - { - "node_count": 5, "cpu_used_cores": 20.0, "cpu_total_cores": 40.0, - "memory_used_gib": 60.0, "memory_total_gib": 160.0, - "gpu_total": 4, "project_count": 20, - "pods_running": 100, "pods_total": 120, - }, - ] - totals = aggregate.compute_totals(overview) - self.assertEqual(totals["node_count"], 8) - self.assertEqual(totals["cpu_used_cores"], 30.0) - self.assertEqual(totals["cpu_total_cores"], 64.0) - self.assertEqual(totals["cpu_percent"], 47) - self.assertEqual(totals["memory_used_gib"], 100.0) - self.assertEqual(totals["memory_total_gib"], 256.0) - self.assertEqual(totals["memory_percent"], 39) - self.assertEqual(totals["gpu_total"], 6) - self.assertEqual(totals["project_count"], 30) - self.assertEqual(totals["pods_running"], 150) - self.assertEqual(totals["pods_total"], 180) - - def test_mixed_metrics_availability(self): - overview = [ - {"node_count": 3, "cpu_used_cores": 10.0, "cpu_total_cores": 24.0, - "memory_used_gib": 40.0, "memory_total_gib": 96.0, - "gpu_total": 0, "project_count": 5, "pods_running": 20, "pods_total": 25}, - {"node_count": 2, "cpu_used_cores": None, "cpu_total_cores": 16.0, - "memory_used_gib": None, "memory_total_gib": 64.0, - "gpu_total": 0, "project_count": 3, "pods_running": 10, "pods_total": 15}, - ] - totals = aggregate.compute_totals(overview) - self.assertEqual(totals["cpu_used_cores"], 10.0) - self.assertEqual(totals["cpu_total_cores"], 40.0) - - -class TestDetectAttentionItems(unittest.TestCase): - def test_high_cpu(self): - overview = [{"cluster": "prod", "cpu_percent": 90, "memory_percent": 50, - "metrics_available": True, "server": "x"}] - per_cluster = {"prod": {"nodes": [], "pod_status": {}, "errors": []}} - items = aggregate.detect_attention_items(overview, per_cluster) - self.assertTrue(any("CPU usage at 90%" in i for i in items)) - - def test_failed_pods(self): - overview = [{"cluster": "prod", "cpu_percent": 50, "memory_percent": 50, - "metrics_available": True, "server": "x"}] - per_cluster = {"prod": {"nodes": [], "pod_status": {"Failed": 3}, "errors": []}} - items = aggregate.detect_attention_items(overview, per_cluster) - self.assertTrue(any("3 pods in Failed" in i for i in items)) - - def test_pending_pods(self): - overview = [{"cluster": "dev", "cpu_percent": 30, "memory_percent": 30, - "metrics_available": True, "server": "x"}] - per_cluster = {"dev": {"nodes": [], "pod_status": {"Pending": 5}, "errors": []}} - items = aggregate.detect_attention_items(overview, per_cluster) - self.assertTrue(any("5 pods in Pending" in i for i in items)) - - def test_crashloopbackoff(self): - overview = [{"cluster": "prod", "cpu_percent": None, "memory_percent": None, - "metrics_available": False, "server": "x"}] - per_cluster = {"prod": {"nodes": [], "pod_status": {"CrashLoopBackOff": 2}, "errors": []}} - items = aggregate.detect_attention_items(overview, per_cluster) - self.assertTrue(any("CrashLoopBackOff" in i for i in items)) - self.assertTrue(any("Metrics Server" in i for i in items)) - - def test_no_issues(self): - overview = [{"cluster": "prod", "cpu_percent": 30, "memory_percent": 40, - "metrics_available": True, "server": "x"}] - per_cluster = {"prod": {"nodes": [], "pod_status": {"Running": 10}, "errors": []}} - items = aggregate.detect_attention_items(overview, per_cluster) - self.assertEqual(items, []) - - def test_node_level_high_usage(self): - overview = [{"cluster": "prod", "cpu_percent": 50, "memory_percent": 50, - "metrics_available": True, "server": "x"}] - per_cluster = {"prod": { - "nodes": [{"name": "node-1", "cpu_used": 7.5, "cpu_total": 8.0, - "memory_used": 5.0, "memory_total": 32.0}], - "pod_status": {}, "errors": [], - }} - items = aggregate.detect_attention_items(overview, per_cluster) - self.assertTrue(any("node-1 CPU at 94%" in i for i in items)) - - -class TestFullPipeline(unittest.TestCase): - - def test_two_cluster_report(self): - input_data = { - "generated_at": "2026-03-03T14:30:00Z", - "clusters": { - "prod-us": { - "context": "prod-us", - "server": "https://api.prod-us.example.com:6443", - "nodes_top": [ - {"name": "node-1", "cpu_usage": "4000m", "memory_usage": "16Gi"}, - {"name": "node-2", "cpu_usage": "3000m", "memory_usage": "12Gi"}, - ], - "nodes_list": [ - { - "metadata": {"name": "node-1", - "labels": {"node-role.kubernetes.io/worker": ""}}, - "status": {"allocatable": {"cpu": "8", "memory": "32Gi", - "nvidia.com/gpu": "2"}}, - }, - { - "metadata": {"name": "node-2", - "labels": {"node-role.kubernetes.io/worker": ""}}, - "status": {"allocatable": {"cpu": "8", "memory": "32Gi"}}, - }, - ], - "projects": [{"name": f"proj-{i}"} for i in range(10)], - "pods": [ - {"metadata": {"namespace": "app"}, "status": {"phase": "Running"}} - for _ in range(8) - ] + [ - {"metadata": {"namespace": "app"}, "status": {"phase": "Failed"}} - for _ in range(2) - ], - "errors": [], - }, - "dev-eu": { - "context": "dev-eu", - "server": "https://api.dev-eu.example.com:6443", - "nodes_top": None, - "nodes_list": [ - { - "metadata": {"name": "dev-1", - "labels": {"node-role.kubernetes.io/worker": ""}}, - "status": {"allocatable": {"cpu": "4", "memory": "16Gi"}}, - }, - ], - "projects": [{"name": f"ns-{i}"} for i in range(3)], - "pods": [ - {"metadata": {"namespace": "default"}, "status": {"phase": "Running"}} - for _ in range(5) - ], - "errors": [], - }, - }, - } - - script_path = Path(__file__).parent / "aggregate.py" - proc = subprocess.run( - [sys.executable, str(script_path)], - input=json.dumps(input_data), - capture_output=True, - text=True, - ) - self.assertEqual(proc.returncode, 0, f"Script failed: {proc.stderr}") - - output = json.loads(proc.stdout) - - self.assertEqual(output["generated_at"], "2026-03-03T14:30:00Z") - self.assertEqual(output["clusters_reported"], 2) - self.assertEqual(output["clusters_failed"], 0) - self.assertEqual(len(output["overview"]), 2) - - prod = next(o for o in output["overview"] if o["cluster"] == "prod-us") - self.assertEqual(prod["node_count"], 2) - self.assertEqual(prod["gpu_total"], 2) - self.assertTrue(prod["metrics_available"]) - self.assertEqual(prod["pods_running"], 8) - self.assertEqual(prod["pods_total"], 10) - - dev = next(o for o in output["overview"] if o["cluster"] == "dev-eu") - self.assertFalse(dev["metrics_available"]) - self.assertIsNone(dev["cpu_used_cores"]) - self.assertEqual(dev["node_count"], 1) - - self.assertTrue(any("Failed" in a for a in output["attention"])) - - self.assertEqual(output["totals"]["node_count"], 3) - self.assertEqual(output["totals"]["gpu_total"], 2) - - def test_malformed_input(self): - script_path = Path(__file__).parent / "aggregate.py" - proc = subprocess.run( - [sys.executable, str(script_path)], - input="not valid json{{{", - capture_output=True, - text=True, - ) - self.assertEqual(proc.returncode, 1) - output = json.loads(proc.stdout) - self.assertIn("error", output) - - def test_empty_clusters(self): - script_path = Path(__file__).parent / "aggregate.py" - proc = subprocess.run( - [sys.executable, str(script_path)], - input=json.dumps({"clusters": {}}), - capture_output=True, - text=True, - ) - self.assertEqual(proc.returncode, 1) - output = json.loads(proc.stdout) - self.assertIn("error", output) - - def test_tabular_input_pipeline(self): - input_data = { - "generated_at": "2026-03-03T15:00:00Z", - "clusters": { - "prod": { - "context": "prod", - "server": "https://api.prod.example.com:6443", - "nodes_top": ( - "NAME CPU(cores) CPU(%) MEMORY(bytes) MEMORY(%)\n" - "node-1 4000m 50% 16Gi 50%" - ), - "nodes_list": [ - { - "metadata": {"name": "node-1", - "labels": {"node-role.kubernetes.io/worker": ""}}, - "status": {"allocatable": {"cpu": "8", "memory": "32Gi"}}, - } - ], - "projects": ( - "APIVERSION KIND NAME DISPLAY NAME STATUS LABELS\n" - "project.openshift.io/v1 Project proj-1 Active \n" - "project.openshift.io/v1 Project proj-2 Active " - ), - "pods": ( - "NAMESPACE NAME STATUS\n" - "default pod-1 Running\n" - "default pod-2 Running\n" - "kube-system pod-3 Pending" - ), - "namespaces": None, - "errors": [], - } - }, - } - script_path = Path(__file__).parent / "aggregate.py" - proc = subprocess.run( - [sys.executable, str(script_path)], - input=json.dumps(input_data), - capture_output=True, text=True, - ) - self.assertEqual(proc.returncode, 0, f"Script failed: {proc.stderr}") - output = json.loads(proc.stdout) - self.assertEqual(output["clusters_reported"], 1) - prod = output["overview"][0] - self.assertEqual(prod["pods_total"], 3) - self.assertEqual(prod["pods_running"], 2) - self.assertEqual(prod["project_count"], 2) - self.assertEqual(prod["cpu_used_cores"], 4.0) - self.assertEqual(prod["cpu_total_cores"], 8.0) - self.assertTrue(prod["metrics_available"]) - self.assertTrue(any("Pending" in a for a in output["attention"])) - - -if __name__ == "__main__": - unittest.main() diff --git a/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/test_assemble.py b/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/test_assemble.py deleted file mode 100644 index 0cd3e9f8..00000000 --- a/evaluation/with_skills/ocp-admin__cluster-report/environment/scripts/cluster-report/test_assemble.py +++ /dev/null @@ -1,490 +0,0 @@ -#!/usr/bin/env python3 - -import json -import os -import stat -import subprocess -import sys -import tempfile -import unittest -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).parent)) -import assemble - - -class TestUnwrapPersistedOutput(unittest.TestCase): - - def test_single_text_entry(self): - raw = json.dumps([{"type": "text", "text": "NAME STATUS\nnode-1 Ready"}]) - result = assemble.unwrap_persisted_output(raw) - self.assertEqual(result, "NAME STATUS\nnode-1 Ready") - - def test_multiple_text_entries(self): - raw = json.dumps([ - {"type": "text", "text": "part1"}, - {"type": "text", "text": "part2"}, - ]) - result = assemble.unwrap_persisted_output(raw) - self.assertEqual(result, "part1\npart2") - - def test_non_envelope_json_array(self): - data = [{"name": "proj-1"}, {"name": "proj-2"}] - raw = json.dumps(data) - result = assemble.unwrap_persisted_output(raw) - self.assertEqual(result, data) - - def test_envelope_with_no_text_type(self): - raw = json.dumps([{"type": "image", "data": "base64..."}]) - result = assemble.unwrap_persisted_output(raw) - self.assertIsNone(result) - - def test_mixed_types_in_envelope(self): - raw = json.dumps([ - {"type": "text", "text": "hello"}, - {"type": "image", "data": "..."}, - {"type": "text", "text": "world"}, - ]) - result = assemble.unwrap_persisted_output(raw) - self.assertEqual(result, "hello\nworld") - - def test_empty_list(self): - raw = "[]" - result = assemble.unwrap_persisted_output(raw) - self.assertEqual(result, []) - - def test_string_value(self): - raw = json.dumps("just a string") - result = assemble.unwrap_persisted_output(raw) - self.assertEqual(result, "just a string") - - def test_dict_value(self): - data = {"key": "value"} - raw = json.dumps(data) - result = assemble.unwrap_persisted_output(raw) - self.assertEqual(result, data) - - def test_non_json_returns_raw_string(self): - raw = "not valid json{{{" - result = assemble.unwrap_persisted_output(raw) - self.assertEqual(result, raw) - - def test_plain_text_tabular_passthrough(self): - raw = ( - "NAMESPACE APIVERSION KIND NAME READY STATUS\n" - "default v1 Pod web-1 1/1 Running\n" - "default v1 Pod web-2 0/1 Pending\n" - ) - result = assemble.unwrap_persisted_output(raw) - self.assertEqual(result, raw) - - def test_plain_text_oc_format_passthrough(self): - raw = "NAME STATUS ROLES AGE\nnode-1 Ready worker 5d\n" - result = assemble.unwrap_persisted_output(raw) - self.assertEqual(result, raw) - - def test_large_text_content(self): - big_text = "LINE\n" * 10000 - raw = json.dumps([{"type": "text", "text": big_text}]) - result = assemble.unwrap_persisted_output(raw) - self.assertEqual(result, big_text) - - -class TestResolveFileRef(unittest.TestCase): - - def test_valid_envelope_file(self): - content = json.dumps([{"type": "text", "text": "pod data here"}]) - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - f.write(content) - path = f.name - try: - result, error = assemble.resolve_file_ref(path) - self.assertIsNone(error) - self.assertEqual(result, "pod data here") - finally: - os.unlink(path) - - def test_valid_plain_json_file(self): - data = [{"name": "ns-1"}, {"name": "ns-2"}] - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(data, f) - path = f.name - try: - result, error = assemble.resolve_file_ref(path) - self.assertIsNone(error) - self.assertEqual(result, data) - finally: - os.unlink(path) - - def test_missing_file(self): - result, error = assemble.resolve_file_ref("/nonexistent/path/file.json") - self.assertIsNone(result) - self.assertIn("not found", error.lower()) - - def test_empty_file(self): - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - path = f.name - try: - result, error = assemble.resolve_file_ref(path) - self.assertIsNone(result) - self.assertIn("Empty", error) - finally: - os.unlink(path) - - def test_non_json_file_returns_content(self): - with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: - f.write("not json{{{") - path = f.name - try: - result, error = assemble.resolve_file_ref(path) - self.assertIsNone(error) - self.assertEqual(result, "not json{{{") - finally: - os.unlink(path) - - def test_plain_text_tabular_file(self): - text = ( - "NAMESPACE APIVERSION KIND NAME READY STATUS RESTARTS AGE\n" - "default v1 Pod web-1 1/1 Running 0 1d\n" - ) - with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: - f.write(text) - path = f.name - try: - result, error = assemble.resolve_file_ref(path) - self.assertIsNone(error) - self.assertEqual(result, text) - finally: - os.unlink(path) - - def test_envelope_with_no_text(self): - content = json.dumps([{"type": "image", "data": "..."}]) - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - f.write(content) - path = f.name - try: - result, error = assemble.resolve_file_ref(path) - self.assertIsNone(result) - self.assertIn("No text content", error) - finally: - os.unlink(path) - - @unittest.skipIf(os.getuid() == 0, "Cannot test permission denied as root") - def test_permission_denied(self): - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - f.write("data") - path = f.name - try: - os.chmod(path, 0o000) - result, error = assemble.resolve_file_ref(path) - self.assertIsNone(result) - self.assertIn("Permission denied", error) - finally: - os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) - os.unlink(path) - - -class TestResolveCluster(unittest.TestCase): - - def test_all_inline_passthrough(self): - cluster = { - "context": "test", - "server": "https://test:6443", - "nodes_top": "NAME CPU\nnode1 100m", - "nodes_list": "NAME STATUS\nnode1 Ready", - "projects": [{"name": "p1"}], - "namespaces": None, - "pods": "NS NAME STATUS\ndefault pod1 Running", - "errors": [], - } - original = json.loads(json.dumps(cluster)) - assemble.resolve_cluster(cluster) - self.assertEqual(cluster, original) - - def test_file_ref_resolved(self): - content = json.dumps([{"type": "text", "text": "pod data"}]) - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - f.write(content) - path = f.name - try: - cluster = { - "pods": {"$file": path}, - "nodes_top": None, - "nodes_list": None, - "projects": None, - "namespaces": None, - "errors": [], - } - assemble.resolve_cluster(cluster) - self.assertEqual(cluster["pods"], "pod data") - self.assertEqual(cluster["errors"], []) - finally: - os.unlink(path) - - def test_failed_file_ref_adds_error(self): - cluster = { - "pods": {"$file": "/nonexistent/file.json"}, - "nodes_top": None, - "nodes_list": None, - "projects": None, - "namespaces": None, - "errors": [], - } - assemble.resolve_cluster(cluster) - self.assertIsNone(cluster["pods"]) - self.assertEqual(len(cluster["errors"]), 1) - self.assertIn("not found", cluster["errors"][0].lower()) - - def test_preserves_existing_errors(self): - cluster = { - "pods": {"$file": "/nonexistent/file.json"}, - "nodes_top": None, - "nodes_list": None, - "projects": None, - "namespaces": None, - "errors": ["Metrics Server not available"], - } - assemble.resolve_cluster(cluster) - self.assertEqual(len(cluster["errors"]), 2) - self.assertEqual(cluster["errors"][0], "Metrics Server not available") - - def test_mixed_inline_file_null(self): - content = json.dumps([{"type": "text", "text": "node data"}]) - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - f.write(content) - path = f.name - try: - cluster = { - "nodes_top": None, - "nodes_list": {"$file": path}, - "projects": [{"name": "p1"}], - "namespaces": None, - "pods": "NS NAME STATUS\ndefault pod1 Running", - "errors": [], - } - assemble.resolve_cluster(cluster) - self.assertIsNone(cluster["nodes_top"]) - self.assertEqual(cluster["nodes_list"], "node data") - self.assertEqual(cluster["projects"], [{"name": "p1"}]) - self.assertEqual(cluster["pods"], "NS NAME STATUS\ndefault pod1 Running") - finally: - os.unlink(path) - - def test_non_data_fields_ignored(self): - cluster = { - "context": {"$file": "/should/not/resolve"}, - "nodes_top": None, - "nodes_list": None, - "projects": None, - "namespaces": None, - "pods": None, - "errors": [], - } - assemble.resolve_cluster(cluster) - self.assertEqual(cluster["context"], {"$file": "/should/not/resolve"}) - - -class TestFullPipeline(unittest.TestCase): - - SCRIPT = str(Path(__file__).parent / "assemble.py") - - def _run(self, input_data, extra_args=None): - cmd = [sys.executable, self.SCRIPT] - if extra_args: - cmd.extend(extra_args) - proc = subprocess.run( - cmd, - input=json.dumps(input_data), - capture_output=True, text=True, - ) - return proc - - def test_inline_passthrough(self): - manifest = { - "generated_at": "2026-01-01T00:00:00Z", - "clusters": { - "test": { - "context": "test", - "server": "https://test:6443", - "nodes_top": None, - "nodes_list": None, - "projects": [{"name": "default"}], - "namespaces": None, - "pods": "NS NAME STATUS\ndefault pod1 Running", - "errors": [], - } - }, - } - proc = self._run(manifest) - self.assertEqual(proc.returncode, 0, f"Failed: {proc.stderr}") - output = json.loads(proc.stdout) - cluster = output["clusters"]["test"] - self.assertEqual(cluster["pods"], "NS NAME STATUS\ndefault pod1 Running") - self.assertEqual(cluster["projects"], [{"name": "default"}]) - - def test_file_ref_resolution(self): - content = json.dumps([{"type": "text", "text": "resolved content"}]) - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - f.write(content) - path = f.name - try: - manifest = { - "generated_at": "2026-01-01T00:00:00Z", - "clusters": { - "test": { - "context": "test", - "server": "https://test:6443", - "nodes_top": None, - "nodes_list": None, - "projects": None, - "namespaces": None, - "pods": {"$file": path}, - "errors": [], - } - }, - } - proc = self._run(manifest) - self.assertEqual(proc.returncode, 0, f"Failed: {proc.stderr}") - output = json.loads(proc.stdout) - self.assertEqual(output["clusters"]["test"]["pods"], "resolved content") - finally: - os.unlink(path) - - def test_aggregate_flag(self): - manifest = { - "generated_at": "2026-01-01T00:00:00Z", - "clusters": { - "test": { - "context": "test", - "server": "https://test:6443", - "nodes_top": None, - "nodes_list": None, - "projects": [{"name": "default"}, {"name": "kube-system"}], - "namespaces": None, - "pods": [ - {"namespace": "default", "name": "pod1", "status": "Running"}, - {"namespace": "default", "name": "pod2", "status": "Pending"}, - ], - "errors": [], - } - }, - } - proc = self._run(manifest, extra_args=["--aggregate"]) - self.assertEqual(proc.returncode, 0, f"Failed: {proc.stderr}") - output = json.loads(proc.stdout) - self.assertIn("clusters_reported", output) - self.assertIn("overview", output) - self.assertEqual(output["clusters_reported"], 1) - self.assertEqual(output["overview"][0]["project_count"], 2) - self.assertEqual(output["overview"][0]["pods_total"], 2) - self.assertEqual(output["overview"][0]["pods_running"], 1) - - def test_malformed_manifest(self): - proc = subprocess.run( - [sys.executable, self.SCRIPT], - input="not valid json{{{", - capture_output=True, text=True, - ) - self.assertEqual(proc.returncode, 1) - output = json.loads(proc.stdout) - self.assertIn("error", output) - - def test_file_ref_error_in_pipeline(self): - manifest = { - "generated_at": "2026-01-01T00:00:00Z", - "clusters": { - "test": { - "context": "test", - "server": "https://test:6443", - "nodes_top": None, - "nodes_list": None, - "projects": None, - "namespaces": None, - "pods": {"$file": "/nonexistent/file.json"}, - "errors": [], - } - }, - } - proc = self._run(manifest) - self.assertEqual(proc.returncode, 0) - output = json.loads(proc.stdout) - cluster = output["clusters"]["test"] - self.assertIsNone(cluster["pods"]) - self.assertTrue(len(cluster["errors"]) > 0) - - def test_end_to_end_with_file_ref_and_aggregate(self): - pods_text = ( - "NAMESPACE APIVERSION KIND NAME READY STATUS RESTARTS AGE\n" - "default v1 Pod web-1 1/1 Running 0 1d\n" - "default v1 Pod web-2 1/1 Running 0 1d\n" - "kube-sys v1 Pod dns-1 0/1 Failed 3 2d\n" - ) - content = json.dumps([{"type": "text", "text": pods_text}]) - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - f.write(content) - path = f.name - try: - manifest = { - "generated_at": "2026-01-01T00:00:00Z", - "clusters": { - "prod": { - "context": "prod", - "server": "https://prod:6443", - "nodes_top": None, - "nodes_list": None, - "projects": [{"name": "default"}, {"name": "kube-sys"}], - "namespaces": None, - "pods": {"$file": path}, - "errors": [], - } - }, - } - proc = self._run(manifest, extra_args=["--aggregate"]) - self.assertEqual(proc.returncode, 0, f"Failed: {proc.stderr}") - output = json.loads(proc.stdout) - self.assertEqual(output["clusters_reported"], 1) - ov = output["overview"][0] - self.assertEqual(ov["pods_total"], 3) - self.assertEqual(ov["pods_running"], 2) - self.assertEqual(ov["project_count"], 2) - finally: - os.unlink(path) - - def test_plain_text_file_ref_with_aggregate(self): - pods_text = ( - "NAMESPACE APIVERSION KIND NAME READY STATUS RESTARTS AGE\n" - "default v1 Pod web-1 1/1 Running 0 1d\n" - "default v1 Pod web-2 0/1 Pending 0 1h\n" - ) - with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: - f.write(pods_text) - path = f.name - try: - manifest = { - "generated_at": "2026-01-01T00:00:00Z", - "clusters": { - "prod": { - "context": "prod", - "server": "https://prod:6443", - "nodes_top": None, - "nodes_list": None, - "projects": [{"name": "default"}], - "namespaces": None, - "pods": {"$file": path}, - "errors": [], - } - }, - } - proc = self._run(manifest, extra_args=["--aggregate"]) - self.assertEqual(proc.returncode, 0, f"Failed: {proc.stderr}") - output = json.loads(proc.stdout) - self.assertEqual(output["clusters_reported"], 1) - ov = output["overview"][0] - self.assertEqual(ov["pods_total"], 2) - self.assertEqual(ov["pods_running"], 1) - finally: - os.unlink(path) - - -if __name__ == "__main__": - unittest.main() diff --git a/evaluation/with_skills/ocp-admin__cluster-report/environment/skills/cluster-report/SKILL.md b/evaluation/with_skills/ocp-admin__cluster-report/environment/skills/cluster-report/SKILL.md deleted file mode 100644 index d9e6a429..00000000 --- a/evaluation/with_skills/ocp-admin__cluster-report/environment/skills/cluster-report/SKILL.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -name: cluster-report -description: | - Generate a consolidated health report across multiple OpenShift clusters. - Verifies each kubeconfig context is a genuine OpenShift cluster before - reporting. Non-OpenShift contexts are skipped by default. - Collects node resources (CPU, memory, GPUs), namespace counts, and pod - status into a single comparison view. - Use when: - - "Show me a report across all clusters" - - "Compare cluster health" - - "Multi-cluster status overview" - - "How are my clusters doing?" - - "Include all clusters including non-OpenShift" (override default filter) - NOT for single-cluster deep-dives or troubleshooting specific pods. -model: inherit -color: cyan -metadata: - user_invocable: "true" ---- - -# Multi-Cluster Report Skill - -Generate a unified health and resource report across multiple OpenShift/Kubernetes clusters using the OpenShift MCP server's multi-cluster capabilities. - -## Prerequisites - -**Required MCP Servers**: `openshift` (configured in [.mcp.json](../../.mcp.json)) - -**Required MCP Tools** (all from `openshift` server): -- `configuration_contexts_list` — list all kubeconfig contexts and server URLs -- `resources_get` — get a single Kubernetes resource by apiVersion/kind/name -- `nodes_top` — node CPU and memory usage from Metrics Server -- `resources_list` — list Kubernetes resources by apiVersion/kind -- `namespaces_list` — list all namespaces in a cluster -- `projects_list` — list all OpenShift projects -- `pods_list` — list all pods across namespaces - -**Required Environment Variables**: `KUBECONFIG` — must contain at least one cluster context. Two or more recommended for comparison. - -**Multi-Cluster Setup**: For large-scale deployments using service account tokens instead of interactive `oc login`, see [multi-cluster-auth.md](../../docs/multi-cluster-auth.md) and the [build-kubeconfig.py](../../scripts/cluster-report/build-kubeconfig.py) helper script. - -**Helper Scripts** (Python 3, stdlib only — treat as black boxes): -- [`assemble.py`](../../scripts/cluster-report/assemble.py) — resolves `$file` references into complete raw data JSON -- [`aggregate.py`](../../scripts/cluster-report/aggregate.py) — aggregates raw data into structured report JSON - -**CRITICAL Script Rules**: -- **NEVER** read the source code of `aggregate.py` or `assemble.py` -- **NEVER** write ad-hoc Python to parse or transform MCP output -- **NEVER** manually reconstruct data already available in MCP output - -**Verification Steps:** -1. Confirm `openshift` MCP server is available in `.mcp.json` -2. Verify `KUBECONFIG` is set: `test -n "$KUBECONFIG"` (never expose path or contents) -3. If either check fails → Human Notification Protocol - -**Human Notification Protocol:** - -When prerequisites fail: -1. **Stop immediately** — do not make any MCP tool calls -2. **Report error:** - ``` - Cannot execute skill: [specific failure] - Setup: [instructions + link to .mcp.json or KUBECONFIG docs] - ``` -3. **Request decision:** "How to proceed? (setup/skip/abort)" -4. **Wait for user input** - -**Security:** Never display KUBECONFIG path, contents, or any credential values. - -## When to Use This Skill - -**Use when**: -- Comparing resource utilization across clusters -- Getting a fleet-wide health overview -- Preparing capacity planning reports - -**Do NOT use when**: -- Debugging a specific pod or workload (use `/debug-pod`) - -## Workflow - -### Step 0: Validate Environment - -Check that `KUBECONFIG` is set. **Never expose the path or contents** — only confirm it is set. If not set, stop and instruct the user to `export KUBECONFIG=/path/to/kubeconfig`. - -### Step 1: Discover and Verify Clusters - -#### Step 1a: List Contexts - -**MCP Tool**: `configuration_contexts_list` - -Collect all context names and server URLs. Do NOT present results to the user yet. - -**Expected Output**: List of context names with associated server URLs. - -**Error Handling**: -- If no contexts found: Stop and instruct user to verify KUBECONFIG points to a valid file with cluster contexts -- If tool call fails: Report MCP server connectivity issue, suggest checking `.mcp.json` configuration - -#### Step 1b: Verify OpenShift Clusters - -For **each** context discovered in Step 1a, probe for the OpenShift `ClusterVersion` resource: - -**MCP Tool**: `resources_get` - -| Parameter | Value | -|---|---| -| `apiVersion` | `config.openshift.io/v1` | -| `kind` | `ClusterVersion` | -| `name` | `version` | -| `context` | `` | - -**Classification rules**: - -| Probe Result | Classification | Default Behavior | -|---|---|---| -| Success (resource returned) | **OpenShift** — extract version from `.status.desired.version` | Include | -| 403 Forbidden | **OpenShift (unverified)** — API group exists, RBAC restricts access | Include (version shown as "unknown") | -| 404 / resource not found | **Non-OpenShift** (vanilla Kubernetes or other distribution) | Exclude | -| Timeout / connection refused / 401 | **Unreachable** | Always exclude | - -**Performance**: Issue all `resources_get` calls in parallel (one per context) since they are independent. - -#### Step 1c: Present Verification Results - -Present a categorized summary to the user: - -```markdown -## Cluster Discovery Results - -### OpenShift Clusters (will be included in report) - -| Context | Server | OpenShift Version | -|---------|--------|-------------------| -| prod-us | https://api.prod-us.example.com:6443 | 4.16.3 | -| staging | https://api.staging.example.com:6443 | 4.15.12 | - -### Non-OpenShift Clusters (excluded by default) - -| Context | Server | Reason | -|---------|--------|--------| -| dev-k8s | https://dev-k8s.example.com:6443 | No ClusterVersion resource (vanilla Kubernetes) | - -### Unreachable Clusters (excluded) - -| Context | Server | Error | -|---------|--------|-------| -| old-cluster | https://old.example.com:6443 | Connection refused | - -**Proceeding with 2 OpenShift clusters.** To include non-OpenShift clusters, say "include all clusters". -``` - -**Presentation rules**: -- Omit any section that has no entries (e.g., skip "Non-OpenShift" section if all contexts are OpenShift). -- If ALL contexts are OpenShift, simplify to: "All N contexts are verified OpenShift clusters." -- If ALL contexts are non-OpenShift, inform the user: "No OpenShift clusters found. To include non-OpenShift clusters, say 'include all clusters'." - -**User override handling**: - -If the user responds with "include all clusters", "include non-OpenShift", "report on all clusters", or any clear intent to include non-OpenShift contexts, add them back into the selected set. Unreachable clusters are always excluded. - -If the user's **original prompt** (before the skill started) already contains phrases like "all clusters", "including non-OpenShift", or "all contexts", pre-select the override and present verification results as: "Including all clusters as requested." - -Proceed automatically with the discovered OpenShift clusters unless the user explicitly requests changes. - -### Step 2: Collect Cluster Data - -For each selected cluster, pass `context=` to every tool call. Collect data using: - -| Manifest Key | MCP Tool | Extra Parameters | Fallback | -|---|---|---|---| -| `nodes_top` | `nodes_top` | — | Set null if Metrics Server unavailable | -| `nodes_list` | `resources_list` | `apiVersion=v1`, `kind=Node` | — | -| `projects` | `projects_list` | — | Use `namespaces_list` if fails | -| `pods` | `pods_list` | — | — | - -**Error policy**: Skip unreachable clusters. Set failed fields to `null` and append the error to the cluster's `errors` array. Never abort the entire report. - -#### Persist MCP Output to Files - -For each MCP tool call, **immediately save the output to a file** under `/tmp/cluster-report/`. -This ensures data is available for the assembly pipeline regardless of output size. - -**Naming convention**: `/tmp/cluster-report/-.txt` - -Use a sanitized short name for the context (e.g., `prod-us`, `dev-eu`). Create the directory first: - -```bash -mkdir -p /tmp/cluster-report -``` - -**How to save**: After each MCP tool call, use Bash to write the output to disk. `$file` references -accept **both plain text and JSON files** — no special formatting is required. - -If Claude Code auto-persisted the output to a file (shown as `persisted-output` in the tool result), -reference that file path directly. - -#### Assemble Manifest - -Write the manifest to `/tmp/cluster-report-manifest.json` with `$file` references to the saved files: - -```json -{ - "generated_at": "2026-03-03T14:30:00Z", - "clusters": { - "": { - "context": "", - "server": "", - "cluster_type": "openshift", - "openshift_version": "4.16.3", - "nodes_top": {"$file": "/tmp/cluster-report/-nodes_top.txt"} or null, - "nodes_list": {"$file": "/tmp/cluster-report/-nodes_list.txt"} or null, - "projects": {"$file": "/tmp/cluster-report/-projects.txt"} or null, - "namespaces": {"$file": "/tmp/cluster-report/-namespaces.txt"} or null, - "pods": {"$file": "/tmp/cluster-report/-pods.txt"} or null, - "errors": [""] - } - } -} -``` - -**Manifest fields from verification**: -- `cluster_type`: `"openshift"` or `"kubernetes"`. Determined during Step 1b verification. -- `openshift_version`: The OpenShift version string (e.g., `"4.16.3"`) or `null` for non-OpenShift clusters. - -Fields may also be inlined as raw text strings or set to `null` for failed/unavailable data. - -### Step 3: Aggregate Data - -Run the assembly and aggregation pipeline: - -```bash -python3 ocp-admin/scripts/cluster-report/assemble.py --aggregate < /tmp/cluster-report-manifest.json -``` - -If the pipeline exits with code 1, display the error JSON to the user and stop. - -### Step 4: Render Report - -Render the structured JSON output as markdown using this template: - -```markdown -# Multi-Cluster Report - -**Generated**: YYYY-MM-DDTHH:MM:SSZ -**Clusters**: clusters reporting - ---- - -## Cluster Overview - -| Cluster | Version | Nodes | CPU (used/total) | Memory (used/total) | GPUs | Projects | Pods (Running/Total) | -|---------|---------|-------|-------------------|---------------------|------|----------|---------------------| -| prod-us | OCP 4.16.3 | 12 | 48/96 cores (50%) | 192/384 GiB (50%) | 8 | 45 | 312/320 | -| dev-eu | OCP 4.15.12 | 4 | 8/32 cores (25%) | 32/128 GiB (25%) | 0 | 12 | 87/92 | -| **Total** | | **16** | **56/128 cores (44%)** | **224/512 GiB (44%)** | **8** | **57** | **399/412** | - ---- - -## Per-Cluster Details - -### () — OpenShift - -#### Node Resources - -| Node | Role | CPU Used | CPU Total | Memory Used | Memory Total | GPUs | -|------|------|----------|-----------|-------------|--------------|------| -| node-1 | worker | 4 cores | 8 cores | 16 GiB | 32 GiB | 2 | - -#### Pod Status - -| Status | Count | -|--------|-------| -| Running | 312 | -| Pending | 5 | -| Succeeded | 0 | -| Failed | 3 | -| Unknown | 0 | - -#### Top Namespaces (by pod count) - -| Namespace | Pods | Running | Pending | Failed | -|-----------|------|---------|---------|--------| -| openshift-monitoring | 24 | 24 | 0 | 0 | - -[Repeat for each cluster] - ---- - -## Attention Required - -[Render each item from the `attention` array] -``` - -### Step 5: Offer Next Steps - -```markdown -## Next Steps - -Would you like to: -1. **Drill down** into a specific cluster or namespace -2. **Check alerts** — query Prometheus/Alertmanager for active alerts -3. **Refresh** — re-run the report with updated data -``` - -## Dependencies - -### Required MCP Servers -- `openshift` — with multi-cluster support enabled - -### Required MCP Tools -- `configuration_contexts_list` (from openshift) — list all kubeconfig contexts and server URLs -- `resources_get` (from openshift) — get a single Kubernetes resource by apiVersion/kind/name - - Parameters: `apiVersion`, `kind`, `name`, `context` -- `nodes_top` (from openshift) — node CPU and memory usage from Metrics Server - - Parameters: `context` -- `resources_list` (from openshift) — list Kubernetes resources by apiVersion/kind - - Parameters: `apiVersion`, `kind`, `context` -- `namespaces_list` (from openshift) — list all namespaces in a cluster - - Parameters: `context` -- `projects_list` (from openshift) — list all OpenShift projects - - Parameters: `context` -- `pods_list` (from openshift) — list all pods across namespaces - - Parameters: `context` - -### Helper Scripts -- [`ocp-admin/scripts/cluster-report/assemble.py`](../../scripts/cluster-report/assemble.py) -- [`ocp-admin/scripts/cluster-report/aggregate.py`](../../scripts/cluster-report/aggregate.py) - -### Related Skills -- None currently - -### Reference Documentation -- [OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server) -- [Kubernetes MCP Server Tools](https://github.com/containers/kubernetes-mcp-server#tools) - -## Error Handling - -| Error | Behavior | -|---|---| -| ClusterVersion probe succeeds | Classify as OpenShift, include by default | -| ClusterVersion probe 404/not found | Classify as non-OpenShift, exclude by default | -| ClusterVersion probe 403 Forbidden | Classify as OpenShift (unverified), include by default with version "unknown" | -| ClusterVersion probe timeout/unreachable | Classify as unreachable, always exclude | -| All contexts are non-OpenShift | Inform user, suggest "include all clusters" override | -| User overrides to include non-OpenShift | Proceed normally; `projects_list` may fail (use `namespaces_list` fallback) | -| Cluster unreachable | Skip, continue with remaining clusters | -| Metrics Server missing | Set `nodes_top` to null, show N/A for CPU/memory usage | -| Auth expired (401) | Skip cluster, suggest: re-run `build-kubeconfig.py build --verify` or `oc login ` | -| No GPUs found | Display 0 (not an error) | -| Empty cluster | Report with all zeros (valid data) | - -## Example Usage - -### Multi-Cluster Report (Default: OpenShift Only) - -**User**: "Show me a report across all clusters" - -**Execution**: -1. Validate KUBECONFIG — OK -2. `configuration_contexts_list()` discovers: prod-us, dev-eu, dev-k8s -3. Verify each context with `resources_get(apiVersion="config.openshift.io/v1", kind="ClusterVersion", name="version", context=)` -4. Results: prod-us (OCP 4.16.3), dev-eu (OCP 4.15.12), dev-k8s (non-OpenShift) -5. Present: "2 OpenShift clusters found. dev-k8s excluded (non-OpenShift). Include all?" -6. User confirms default selection -7. Collect data for prod-us and dev-eu only -8. Write manifest with `cluster_type` and `openshift_version` fields -9. Run `assemble.py --aggregate` pipeline -10. Render report with OpenShift version column -11. Flag attention items - -### Multi-Cluster Report (Include All) - -**User**: "Report on all my clusters including non-OpenShift" - -**Execution**: -1. Validate KUBECONFIG — OK -2. `configuration_contexts_list()` discovers: prod-us, dev-eu, dev-k8s -3. Verify each context (same as above) -4. Results: prod-us (OCP 4.16.3), dev-eu (OCP 4.15.12), dev-k8s (non-OpenShift) -5. User's initial message indicates "include all" — present verification results and confirm -6. User confirms all clusters including dev-k8s -7. Collect data for all three clusters (`projects_list` fails on dev-k8s, falls back to `namespaces_list`) -8. Write manifest; dev-k8s has `cluster_type: "kubernetes"`, `openshift_version: null` -9. Run pipeline, render report -10. dev-k8s shown as "K8s" in version column diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/common-issues.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/examples/model-deploy.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/examples/model-deploy.md deleted file mode 100644 index dcd79e44..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/examples/model-deploy.md +++ /dev/null @@ -1,160 +0,0 @@ -# /model-deploy Examples - -Example interactions demonstrating the model-deploy skill workflow. - -## Example 1: Deploy Llama 3.1 8B with vLLM - -``` -User: "Deploy Llama 3.1 8B on my cluster" - -Agent: [Invokes /model-deploy skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand runtime capabilities. - -Step 1 of 10: Gathering deployment information. - -| Setting | Value | Source | -|---------|-------|--------| -| Model | Llama 3.1 8B Instruct | user input | -| Runtime | TBD | auto-detect | -| Namespace | ? | needs input | -| Model Source | ? | needs input | -| Deployment Mode | Serverless | default | - -What namespace should I deploy to, and where are the model weights stored? -(e.g., S3 bucket, OCI registry, PVC) - -User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" - -Agent: Step 2 of 10: Determining runtime. - -I consulted supported-runtimes.md to understand runtime capabilities. - -Based on Llama 3.1 8B (open-source HuggingFace model): - -Recommended runtime: vLLM -Reason: Default runtime for open-source LLMs. Broad compatibility, - OpenAI-compatible API, no additional setup required. - -Proceed with vLLM? (yes / no / suggest alternative) - -User: "yes" - -Agent: Step 3 of 10: Looking up hardware profile. - -I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. - -| Requirement | Value | -|-------------|-------| -| GPUs | 1x A10G/L4/A100/H100 | -| VRAM | 16GB | -| Key Args | --max-model-len=4096 --tool-call-parser hermes | - -Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). -Ensure your cluster has at least 1 GPU with 16GB VRAM. - -Step 5 of 10: Skipped (NIM platform check not needed for vLLM). - -Step 6-7 of 10: Generating InferenceService YAML. - -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - name: llama-3-1-8b - namespace: ai-models - annotations: - serving.kserve.io/deploymentMode: Serverless -spec: - predictor: - model: - modelFormat: - name: vLLM - runtime: vllm-runtime - storageUri: s3://my-bucket/llama-3.1-8b - resources: - limits: - nvidia.com/gpu: "1" - requests: - cpu: "4" - memory: "16Gi" - args: - - --max-model-len=4096 - - --tool-call-parser=hermes - -| Setting | Value | -|---------|-------| -| Model | llama-3-1-8b | -| Runtime | vllm-runtime | -| GPUs | 1 | -| Namespace | ai-models | -| Mode | Serverless | - -Proceed with creating this InferenceService? (yes/no/modify) - -User: "yes" - -Agent: Step 8 of 10: Creating InferenceService... - [Creates via resources_create_or_update] - - InferenceService created. Monitoring rollout... - -Step 9 of 10: Monitoring rollout. - -| Stage | Status | -|-------|--------| -| Pod Scheduled | Complete | -| Image Pulled | Complete | -| Container Started | Complete | -| Model Loaded | Complete | -| Ready | Complete | - -Step 10 of 10: Deployment complete! - -Model Deployment Successful - -Model: llama-3-1-8b -Runtime: vLLM -Namespace: ai-models -GPUs: 1 - -Inference Endpoint: - URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com - API: OpenAI-compatible REST - -Quick Test: - curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' - -Next Steps: - Monitor performance: "/ai-observability llama-3-1-8b" -``` - -## Example 2: Deploy a Model with NIM (NIM Not Set Up) - -``` -User: "Deploy Llama 3.1 70B with NIM" - -Agent: [Invokes /model-deploy skill] - ... - Step 2: User requested NIM runtime. - - Step 5: Checking NIM platform... - -NIM platform is not set up in namespace ai-models. - -The NIM runtime requires a one-time platform setup. Would you like to: -1. Run /nim-setup now to configure the NIM platform -2. Switch to vLLM runtime instead (no setup required) -3. Abort deployment - -User: "1" - -Agent: [Invokes /nim-setup skill — walks through NGC credential setup] - ... - NIM Platform Setup Complete. - - Resuming model deployment... - [Continues from Step 6 with NIM runtime] -``` diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/examples/nim-setup.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/examples/nim-setup.md deleted file mode 100644 index ccfc1a5d..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/examples/nim-setup.md +++ /dev/null @@ -1,115 +0,0 @@ -# /nim-setup Examples - -Example interactions demonstrating the nim-setup skill workflow. - -## Example 1: First-Time NIM Setup - -``` -User: "Set up NIM on my cluster" - -Agent: [Invokes /nim-setup skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand NIM platform requirements. - -Let me verify the prerequisites for NIM setup. - -Step 1 of 7: Checking GPU Operator and NFD Operator... - -GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) -NFD Operator: nfd.v4.19.0 (Succeeded) - -Both operators are installed and healthy. - -Step 2 of 7: I need your NGC credentials. - -To set up NIM, I need your NVIDIA NGC API key. -You can generate one at: https://ngc.nvidia.com/setup/api-key - -Please provide: -1. Your NGC API key -2. The target namespace for NIM resources - -User: "Here's my key: [user provides NGC API key], namespace: ai-models" - -Agent: Step 3 of 7: Creating NGC image pull secret... - -I'll create a docker-registry secret for pulling NIM images from nvcr.io: - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-image-pull-secret - namespace: ai-models - type: kubernetes.io/dockerconfigjson - data: - .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] - -Should I create this image pull secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - Image pull secret created successfully. - -Step 4 of 7: Creating NGC API key secret... - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-api-key - namespace: ai-models - type: Opaque - stringData: - NGC_API_KEY: "[REDACTED]" - -Should I create this API key secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - API key secret created successfully. - -Step 5 of 7: Creating NIM Account CR... - - apiVersion: nim.opendatahub.io/v1 - kind: Account - metadata: - name: nim-account - namespace: ai-models - spec: - apiKeySecret: - name: ngc-api-key - imagePullSecret: - name: ngc-image-pull-secret - -Should I create this NIM Account CR in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates Account CR via resources_create_or_update] - Account CR created successfully. - -Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) - -User: "no" - -Agent: Step 7 of 7: Validating NIM platform readiness... - -NIM Platform Setup Complete - -Account CR: `nim-account` (namespace: ai-models) - Status: Ready - -NGC Credentials: - Image Pull Secret: ngc-image-pull-secret [created] - API Key Secret: ngc-api-key [created] - -NIM ServingRuntimes Available: - - nim-llama-3-1-8b-instruct - - nim-llama-3-1-70b-instruct - - nim-mistral-7b-instruct - -Next Steps: - To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" -``` diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/references/known-model-profiles.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/references/known-model-profiles.md deleted file mode 100644 index 40148981..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/references/known-model-profiles.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Known Model Hardware Profiles -category: references -tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] -semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] -use_cases: [model-deploy, debug-inference, ai-observability] -last_updated: 2026-03-10 ---- - -# Known Model Hardware Profiles - -Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). - -**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. - -## Model Source Conventions - -Each model lists a recommended `storageUri` with its authentication requirements: -- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** -- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). -- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. - -When the user does not specify a model source, use the `hf://` URI listed in the profile below. - -## Llama 3.x (Meta) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | -| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | -| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | -| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | - -- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) -- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly - -## Granite 3.x (IBM/Red Hat) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | -| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | - -- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) -- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser granite --chat-template` -- Red Hat-supported model family on RHOAI - -## Mixtral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | -| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) -- Mixture-of-Experts architecture: only ~13B/45B parameters active per token - -## Mistral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | -| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) - -## When a Model Is Not Listed - -If the requested model is not in this file, the agent MUST use the live doc lookup protocol: - -1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure -2. Fetch hardware specs from the appropriate source -3. Report findings to the user before proceeding with deployment - -Common cases requiring live lookup: -- Newly released models (after this file's last update) -- Domain-specific fine-tuned models -- Models with custom quantization -- NIM-specific optimized profiles diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/references/supported-runtimes.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/references/supported-runtimes.md deleted file mode 100644 index 9d018651..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/references/supported-runtimes.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: Supported Serving Runtimes -category: references -tags: [runtimes, vllm, nim, caikit, tgis, serving] -semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] -use_cases: [model-deploy, serving-runtime-config, nim-setup] -last_updated: 2026-02-26 ---- - -# Supported Serving Runtimes - -This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. - -## Runtime Comparison - -| Runtime | API Type | Model Format | GPU Support | License | Setup Required | -|---------|----------|-------------|-------------|---------|----------------| -| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | -| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | -| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | -| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | - -## vLLM - -**Default runtime for most open-source models.** - -- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) -- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized -- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) -- **Model source**: S3-compatible storage, OCI registry, PVC, URI -- **Key features**: - - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) - - PagedAttention for efficient memory management - - Tensor parallelism for multi-GPU inference - - Tool/function calling support (`--tool-call-parser`) - - Continuous batching for high throughput -- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name - -## NVIDIA NIM - -**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** - -- **API**: OpenAI-compatible REST -- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog -- **GPU support**: NVIDIA only (requires specific GPU models per model profile) -- **Model source**: NGC model catalog (pulled automatically via NGC credentials) -- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) -- **Key features**: - - TensorRT-LLM optimization for lower latency - - Pre-compiled model engines (no compilation on first load) - - Optimized and generic profiles per GPU type - - Automatic model download from NGC -- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. -- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM -- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. -- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state - -## Caikit+TGIS - -**Red Hat's Caikit format with Text Generation Inference Server.** - -- **API**: gRPC (not REST) -- **Model formats**: Caikit format (requires conversion from HuggingFace) -- **GPU support**: NVIDIA -- **Model source**: S3-compatible storage -- **Key features**: - - Red Hat-supported runtime - - gRPC API for streaming inference - - Integrated with RHOAI model serving platform -- **When to choose**: When the model is already in Caikit format or when gRPC API is required. -- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name - -## Custom Runtimes - -**User-provided ServingRuntime CRs for unsupported frameworks.** - -- **API**: Defined by the custom runtime -- **Model formats**: Defined by the custom runtime -- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. -- **How to create**: Use `/serving-runtime-config` skill -- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility - -## Runtime Selection Decision Tree - -``` -Is the user's preferred runtime explicitly stated? -├── Yes → Use that runtime -└── No → Continue - -Is the model available in the NGC NIM catalog? -├── Yes → Suggest NIM (with vLLM as fallback) -│ Note: Requires /nim-setup and NVIDIA GPUs -└── No → Continue - -Is the model in Caikit format? -├── Yes → Caikit+TGIS -└── No → Continue - -Is the model a standard open-source LLM (HuggingFace-compatible)? -├── Yes → vLLM (default) -└── No → Custom runtime via /serving-runtime-config -``` diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/docs/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/ai-observability/SKILL.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/ai-observability/SKILL.md deleted file mode 100644 index d93861ec..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/ai-observability/SKILL.md +++ /dev/null @@ -1,420 +0,0 @@ ---- -name: ai-observability -description: | - Analyze AI model performance, GPU utilization, and cluster health on OpenShift AI. - - Use when: - - "How is my model performing?" - - "What GPUs are available in the cluster?" - - "Show me inference latency for Llama" - - "Check OpenShift cluster health metrics" - - "Trace a slow inference request" - - "Correlate errors across my inference stack" - - Query-driven, read-only analysis. Routes to the appropriate observability domain based on user intent. - - NOT for deploying models (use /model-deploy). - NOT for debugging failed deployments (use /debug-inference). -model: inherit -color: cyan ---- - -# /ai-observability Skill - -Analyze AI model inference performance, GPU utilization, OpenShift cluster health, and distributed traces on Red Hat OpenShift AI. This is a query-driven, read-only skill: the user asks a question, and the skill routes to the appropriate observability domain (vLLM metrics, OpenShift health, Tempo traces, or cross-domain correlation via Korrel8r). - -## Prerequisites - -**Required MCP Server**: `ai-observability` ([AI Observability MCP](https://github.com/rh-ai-quickstart/ai-observability-summarizer)) - -**Required MCP Tools**: -- `list_models` (from ai-observability) - Discover served models -- `list_vllm_namespaces` (from ai-observability) - List monitored namespaces -- `get_gpu_info` (from ai-observability) - GPU inventory and utilization -- `get_deployment_info` (from ai-observability) - Deployment health status -- `analyze_vllm` (from ai-observability) - Model performance analysis -- `chat_vllm` (from ai-observability) - Conversational follow-up on vLLM metrics -- `analyze_openshift` (from ai-observability) - Cluster/namespace health metrics -- `list_openshift_metric_groups` (from ai-observability) - Available metric categories -- `list_openshift_namespaces` (from ai-observability) - Namespaces in Prometheus -- `query_tempo_tool` (from ai-observability) - Distributed trace queries -- `get_trace_details_tool` (from ai-observability) - Trace span details -- `search_metrics` (from ai-observability) - Metric discovery by pattern -- `execute_promql` (from ai-observability) - Custom PromQL queries -- `korrel8r_get_correlated` (from ai-observability) - Cross-domain signal correlation - -**Optional MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) - -**Optional MCP Tools** (from rhoai): -- `list_data_science_projects` - Discover RHOAI projects for scope selection -- `list_inference_services` - List deployed models with structured status for context -- `get_inference_service` - Get InferenceService status for context - -**Optional MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Optional MCP Tools** (from openshift): -- `resources_get` (from openshift) - Get raw resource details for context -- `pods_list` (from openshift) - List predictor pods for correlation context - -**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). - -**Additional environment variables**: -- `AI_OBSERVABILITY_MCP_URL` - URL for the AI Observability MCP server (e.g., `http://aiobs-mcp.apps.cluster.example.com`) - -**Additional cluster requirements**: -- AI Observability MCP server deployed on-cluster (from `quay.io/ecosystem-appeng/aiobs-mcp-server`) -- Prometheus configured with vLLM and OpenShift metrics -- Tempo configured for distributed tracing (optional, for trace analysis) -- Korrel8r deployed (optional, for cross-domain correlation) - -## When to Use This Skill - -**Use this skill when you need to:** -- Check model inference performance (latency, throughput, error rates) -- View GPU inventory and utilization across the cluster -- Analyze OpenShift cluster health metrics by category -- Trace slow inference requests with distributed tracing (Tempo) -- Correlate signals across logs, metrics, traces, and alerts (Korrel8r) -- Run custom PromQL queries against cluster Prometheus - -**Do NOT use this skill when:** -- You need to troubleshoot a failed deployment (use `/debug-inference`) -- You want to deploy or redeploy a model (use `/model-deploy`) -- You need to create or modify a ServingRuntime (use `/serving-runtime-config`) - -## Workflow - -### Step 1: Verify MCP and Triage Intent - -**Verify ai-observability MCP server is reachable.** If any tool call fails with a connection error: - -``` -Cannot execute /ai-observability: ai-observability MCP server is not reachable. - -Setup Instructions: -1. Deploy the server on your cluster from quay.io/ecosystem-appeng/aiobs-mcp-server - See: https://github.com/rh-ai-quickstart/ai-observability-summarizer -2. Set AI_OBSERVABILITY_MCP_URL to the server route URL -3. Restart Claude Code to reload MCP servers - -Options: setup (show deployment guide) / abort -``` - -Proceed automatically without waiting for user input. - -**Classify user query** into one of these domains: - -| Domain | Trigger Phrases | Primary Tool(s) | -|--------|----------------|-----------------| -| Model Discovery | "what models", "list models", "what's deployed" | `list_models`, `list_vllm_namespaces` | -| GPU Inventory | "GPU", "GPU utilization", "what GPUs", "available hardware" | `get_gpu_info` | -| vLLM Performance | "latency", "throughput", "performance", "how is [model]", "slow" | `analyze_vllm` | -| OpenShift Health | "cluster health", "namespace metrics", "node health", "pods" | `analyze_openshift` | -| Tracing | "trace", "latency trace", "slow request", "spans" | `query_tempo_tool` | -| Correlation | "correlate", "root cause", "what's related to" | `korrel8r_get_correlated` | -| Custom PromQL | "PromQL", "custom query", "specific metric" | `execute_promql` | - -If the intent is ambiguous, present the domain options and ask the user to choose. - -If the user specifies a model name, use `list_models` first to verify it exists and get the correct identifier. If the user does not specify a namespace, use `list_vllm_namespaces` or `list_openshift_namespaces` to discover available namespaces and present them. - -**Project context** (if `rhoai` MCP available): For "what's running" or "what's deployed" queries, use `list_data_science_projects` (from rhoai) to provide project-level overview. Use `list_inference_services` (from rhoai) per project to show deployed models with status. - -Proceed automatically without waiting for user input. - -### Step 2: Execute Analysis - -Branch based on the domain determined in Step 1. - -#### Step 2a: Model Discovery - -**MCP Tool**: `list_models` (from ai-observability) - -**Parameters**: None - -**MCP Tool**: `list_vllm_namespaces` (from ai-observability) - -**Parameters**: None - -Present results: - -| Model Name | Namespace | -|------------|-----------| -| [model] | [namespace] | - -**Offer**: "Would you like to analyze performance for a specific model, or check GPU inventory?" - -Proceed automatically without waiting for user input. - -#### Step 2b: GPU Inventory - -**MCP Tool**: `get_gpu_info` (from ai-observability) - -**Parameters**: None - -Present results: - -| Node | GPU Type | Count | Temperature | Power Usage | -|------|----------|-------|-------------|-------------| -| [node] | [type] | [count] | [temp] | [watts] | - -If GPUs are near capacity, note: "Some GPUs are heavily utilized. Check model performance or consider scaling." - -**Offer**: "Would you like to check which models are using these GPUs, or analyze a specific model's performance?" - -Proceed automatically without waiting for user input. - -#### Step 2c: vLLM Performance Analysis - -Requires: model name (from user or discovered via `list_models` in Step 1). - -**MCP Tool**: `get_deployment_info` (from ai-observability) - -**Parameters**: -- `namespace`: model's namespace - REQUIRED -- `model`: model name - REQUIRED - -Report deployment status (is_new_deployment, deployment_date). - -**MCP Tool**: `analyze_vllm` (from ai-observability) - -**Parameters**: -- `model_name`: vLLM model identifier - REQUIRED -- `summarize_model_id`: LLM for analysis (use server default if not specified) - REQUIRED -- `time_range`: natural language time range, e.g., `"15m"`, `"1h"`, `"24h"` - OPTIONAL (default: `"15m"`) -- `start_datetime`: ISO datetime string - OPTIONAL (alternative to time_range) -- `end_datetime`: ISO datetime string - OPTIONAL (alternative to time_range) - -Present the LLM-generated analysis covering: latency (p50/p95/p99), throughput (requests/sec), token rates (input/output tokens/sec), error rate, queue depth. - -**Offer**: -``` -Would you like to: -1. Ask a follow-up question about these metrics -2. Trace a slow inference request -3. Correlate with other signals (logs, alerts) -4. Check a different time range -5. Exit analysis -``` - -Proceed automatically without waiting for user input. - -If user asks a follow-up question: - -**MCP Tool**: `chat_vllm` (from ai-observability) - -**Parameters**: -- `model_name`: same model name - REQUIRED -- `prompt_summary`: the analysis output from `analyze_vllm` - REQUIRED -- `question`: the user's follow-up question - REQUIRED -- `summarize_model_id`: LLM for response - REQUIRED - -#### Step 2d: OpenShift Health Analysis - -**MCP Tool**: `list_openshift_metric_groups` (from ai-observability) - -**Parameters**: None - -Present available metric categories to user if they did not specify one. - -Proceed automatically without waiting for user input. - -**MCP Tool**: `analyze_openshift` (from ai-observability) - -**Parameters**: -- `metric_category`: the selected category (e.g., `"Fleet Overview"`, `"GPU & Accelerators"`, `"Workloads & Pods"`, `"Storage & Networking"`) - REQUIRED -- `scope`: `"cluster_wide"` or `"namespace_scoped"` - OPTIONAL (default: `"cluster_wide"`) -- `namespace`: required when scope is `"namespace_scoped"` - CONDITIONAL -- `time_range`: natural language time range - OPTIONAL -- `start_datetime`: ISO datetime string - OPTIONAL -- `end_datetime`: ISO datetime string - OPTIONAL - -Present the health assessment and key metrics. - -**Offer**: "Would you like to check another metric category, drill into a specific namespace, or exit?" - -Proceed automatically without waiting for user input. - -#### Step 2e: Distributed Tracing - -Requires: service name or operation name, and time range. - -**MCP Tool**: `query_tempo_tool` (from ai-observability) - -**Parameters**: -- `query`: TraceQL query string (e.g., `"{resource.service.name=\"[service]\"}"`) - REQUIRED -- `start_time`: ISO datetime string (e.g., `"2024-01-01T00:00:00Z"`) - REQUIRED -- `end_time`: ISO datetime string - REQUIRED -- `limit`: max traces to return - OPTIONAL (default: 10) - -Present traces: - -| Trace ID | Duration (ms) | Root Service | Span Count | Start Time | -|----------|--------------|--------------|------------|------------| -| [id] | [duration] | [service] | [spans] | [time] | - -**Ask**: "Would you like to drill into a specific trace? Enter a Trace ID." - -Proceed automatically without waiting for user input. - -If user selects a trace: - -**MCP Tool**: `get_trace_details_tool` (from ai-observability) - -**Parameters**: -- `trace_id`: the trace ID string - REQUIRED - -Present span waterfall: - -| Span | Service | Operation | Duration (ms) | Status | -|------|---------|-----------|---------------|--------| -| [span-id] | [service] | [operation] | [duration] | [ok/error] | - -**Offer**: "Would you like to view another trace, correlate this trace with logs/metrics, or exit?" - -Proceed automatically without waiting for user input. - -#### Step 2f: Cross-Domain Correlation (Korrel8r) - -Requires: a starting point (pod name and namespace, or other Korrel8r domain query). - -**MCP Tool**: `korrel8r_get_correlated` (from ai-observability) - -**Parameters**: -- `query`: Korrel8r domain query string - REQUIRED - - Example: `k8s:Pod:{"namespace":"llm-serving","name":"vllm-predictor-abc"}` -- `goals`: array of target domain class names - REQUIRED - - Example: `["log:application", "metric:metric", "trace:span", "alert:alert"]` - -Present correlated signals grouped by domain: - -**Related Logs**: [count] log entries found -**Related Metrics**: [count] metric series -**Related Traces**: [count] trace spans -**Related Alerts**: [count] active alerts - -**Offer**: "Would you like to drill into any of these correlated signals?" - -Proceed automatically without waiting for user input. - -#### Step 2g: Custom PromQL Query - -For advanced users who want to run specific PromQL. - -**MCP Tool**: `search_metrics` (from ai-observability) - -**Parameters**: -- `pattern`: search string (e.g., `"vllm latency"`) - OPTIONAL (default: `""`) -- `limit`: max results, 1-1000 - OPTIONAL (default: 50) - -Present matching metrics with their descriptions. Let user select or compose a query. - -**MCP Tool**: `execute_promql` (from ai-observability) - -**Parameters**: -- `query`: PromQL query string - REQUIRED -- `time_range`: relative time range (e.g., `"5m"`, `"1h"`) - OPTIONAL -- `start_datetime`: ISO datetime string - OPTIONAL -- `end_datetime`: ISO datetime string - OPTIONAL - -Present query results. - -**Offer**: "Would you like to run another query, or exit?" - -Proceed automatically without waiting for user input. - -### Step 3: Follow-Up and Drill-Down - -After presenting initial results, offer domain-appropriate follow-up options: - -- For vLLM analysis: use `chat_vllm` for conversational follow-up -- For traces: allow drilling into specific trace IDs via `get_trace_details_tool` -- For correlation: allow drilling into correlated signals -- For any domain: offer to switch to a different analysis domain - -Present options and Proceed automatically without waiting for user input. - -### Step 4: Summary and Next Steps - -When the user chooses to exit: - -Summarize key findings from the analysis session. - -**If issues were found**, suggest: -- `/debug-inference` for deployment or pod-level problems -- `/model-deploy` to redeploy with different configuration -- Custom PromQL queries for ongoing monitoring - -**If everything looks healthy**, confirm: "All monitored metrics are within normal ranges." - -## Common Issues - -### Issue 1: AI Observability MCP Server Not Deployed - -**Error**: Connection refused or timeout when reaching `AI_OBSERVABILITY_MCP_URL` - -**Cause**: The AI Observability MCP server is not deployed on the cluster, or the route/service is not accessible. - -**Solution:** -1. Deploy the server from `quay.io/ecosystem-appeng/aiobs-mcp-server` -- see https://github.com/rh-ai-quickstart/ai-observability-summarizer -2. Verify the route is accessible: `oc get route -n [namespace] aiobs-mcp` -3. Set `AI_OBSERVABILITY_MCP_URL` to the route URL -4. Restart Claude Code to reload MCP servers - -### Issue 2: No Models Found in Monitoring - -**Error**: `list_models` returns empty results - -**Cause**: vLLM metrics are not being scraped by Prometheus, or no InferenceServices are deployed. - -**Solution:** -1. Verify InferenceServices exist: use `resources_list` from `openshift` MCP -2. Check that Prometheus ServiceMonitor is configured for vLLM metrics -3. Verify the vLLM serving container exposes `/metrics` endpoint - -### Issue 3: Tempo Traces Not Available - -**Error**: `query_tempo_tool` returns empty or connection error - -**Cause**: Tempo is not deployed, or distributed tracing is not configured for the inference stack. - -**Solution:** -1. Verify Tempo is deployed in the cluster -2. Check OpenTelemetry instrumentation on the inference endpoints -3. Verify Tempo datasource is configured in the MCP server - -### Issue 4: Korrel8r Correlation Returns No Results - -**Error**: `korrel8r_get_correlated` returns empty correlation - -**Cause**: Korrel8r is not deployed, or the query format is incorrect. - -**Solution:** -1. Verify Korrel8r is deployed and accessible -2. Check the query format matches Korrel8r domain syntax (e.g., `k8s:Pod:{"namespace":"[ns]","name":"[pod]"}`) -3. Ensure the target pod/namespace exists and has generated observability signals - -## Dependencies - -### MCP Tools -See [Prerequisites](#prerequisites) for the complete list of required and optional MCP tools. - -### Related Skills -- `/debug-inference` - Troubleshoot deployment issues found during analysis -- `/model-deploy` - Redeploy models with different configuration based on findings -- `/serving-runtime-config` - Adjust runtime parameters if performance issues are runtime-related - -### Reference Documentation -- [known-model-profiles.md](../../docs/references/known-model-profiles.md) - Expected performance baselines for common models -- [supported-runtimes.md](../../docs/references/supported-runtimes.md) - Runtime capabilities and known limitations - -## Critical: Human-in-the-Loop Requirements - -See [skill-conventions.md](../references/skill-conventions.md) for general HITL and security conventions. - -**Skill-specific checkpoints:** -- After triage (Step 1): confirm analysis scope (model, namespace, time range) before running queries -- After initial analysis (Step 2): present follow-up options, wait for user choice -- After correlation (Step 2f): confirm before drilling into correlated signals -- **NEVER** expose raw Prometheus/Tempo credentials or internal cluster endpoints in output -- **NEVER** execute unbounded PromQL queries (no time limit, extremely wide label selectors) without confirming with the user diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/common-issues.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ai-observability/environment/skills/references/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/common-issues.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/examples/model-deploy.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/examples/model-deploy.md deleted file mode 100644 index dcd79e44..00000000 --- a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/examples/model-deploy.md +++ /dev/null @@ -1,160 +0,0 @@ -# /model-deploy Examples - -Example interactions demonstrating the model-deploy skill workflow. - -## Example 1: Deploy Llama 3.1 8B with vLLM - -``` -User: "Deploy Llama 3.1 8B on my cluster" - -Agent: [Invokes /model-deploy skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand runtime capabilities. - -Step 1 of 10: Gathering deployment information. - -| Setting | Value | Source | -|---------|-------|--------| -| Model | Llama 3.1 8B Instruct | user input | -| Runtime | TBD | auto-detect | -| Namespace | ? | needs input | -| Model Source | ? | needs input | -| Deployment Mode | Serverless | default | - -What namespace should I deploy to, and where are the model weights stored? -(e.g., S3 bucket, OCI registry, PVC) - -User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" - -Agent: Step 2 of 10: Determining runtime. - -I consulted supported-runtimes.md to understand runtime capabilities. - -Based on Llama 3.1 8B (open-source HuggingFace model): - -Recommended runtime: vLLM -Reason: Default runtime for open-source LLMs. Broad compatibility, - OpenAI-compatible API, no additional setup required. - -Proceed with vLLM? (yes / no / suggest alternative) - -User: "yes" - -Agent: Step 3 of 10: Looking up hardware profile. - -I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. - -| Requirement | Value | -|-------------|-------| -| GPUs | 1x A10G/L4/A100/H100 | -| VRAM | 16GB | -| Key Args | --max-model-len=4096 --tool-call-parser hermes | - -Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). -Ensure your cluster has at least 1 GPU with 16GB VRAM. - -Step 5 of 10: Skipped (NIM platform check not needed for vLLM). - -Step 6-7 of 10: Generating InferenceService YAML. - -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - name: llama-3-1-8b - namespace: ai-models - annotations: - serving.kserve.io/deploymentMode: Serverless -spec: - predictor: - model: - modelFormat: - name: vLLM - runtime: vllm-runtime - storageUri: s3://my-bucket/llama-3.1-8b - resources: - limits: - nvidia.com/gpu: "1" - requests: - cpu: "4" - memory: "16Gi" - args: - - --max-model-len=4096 - - --tool-call-parser=hermes - -| Setting | Value | -|---------|-------| -| Model | llama-3-1-8b | -| Runtime | vllm-runtime | -| GPUs | 1 | -| Namespace | ai-models | -| Mode | Serverless | - -Proceed with creating this InferenceService? (yes/no/modify) - -User: "yes" - -Agent: Step 8 of 10: Creating InferenceService... - [Creates via resources_create_or_update] - - InferenceService created. Monitoring rollout... - -Step 9 of 10: Monitoring rollout. - -| Stage | Status | -|-------|--------| -| Pod Scheduled | Complete | -| Image Pulled | Complete | -| Container Started | Complete | -| Model Loaded | Complete | -| Ready | Complete | - -Step 10 of 10: Deployment complete! - -Model Deployment Successful - -Model: llama-3-1-8b -Runtime: vLLM -Namespace: ai-models -GPUs: 1 - -Inference Endpoint: - URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com - API: OpenAI-compatible REST - -Quick Test: - curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' - -Next Steps: - Monitor performance: "/ai-observability llama-3-1-8b" -``` - -## Example 2: Deploy a Model with NIM (NIM Not Set Up) - -``` -User: "Deploy Llama 3.1 70B with NIM" - -Agent: [Invokes /model-deploy skill] - ... - Step 2: User requested NIM runtime. - - Step 5: Checking NIM platform... - -NIM platform is not set up in namespace ai-models. - -The NIM runtime requires a one-time platform setup. Would you like to: -1. Run /nim-setup now to configure the NIM platform -2. Switch to vLLM runtime instead (no setup required) -3. Abort deployment - -User: "1" - -Agent: [Invokes /nim-setup skill — walks through NGC credential setup] - ... - NIM Platform Setup Complete. - - Resuming model deployment... - [Continues from Step 6 with NIM runtime] -``` diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/examples/nim-setup.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/examples/nim-setup.md deleted file mode 100644 index ccfc1a5d..00000000 --- a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/examples/nim-setup.md +++ /dev/null @@ -1,115 +0,0 @@ -# /nim-setup Examples - -Example interactions demonstrating the nim-setup skill workflow. - -## Example 1: First-Time NIM Setup - -``` -User: "Set up NIM on my cluster" - -Agent: [Invokes /nim-setup skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand NIM platform requirements. - -Let me verify the prerequisites for NIM setup. - -Step 1 of 7: Checking GPU Operator and NFD Operator... - -GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) -NFD Operator: nfd.v4.19.0 (Succeeded) - -Both operators are installed and healthy. - -Step 2 of 7: I need your NGC credentials. - -To set up NIM, I need your NVIDIA NGC API key. -You can generate one at: https://ngc.nvidia.com/setup/api-key - -Please provide: -1. Your NGC API key -2. The target namespace for NIM resources - -User: "Here's my key: [user provides NGC API key], namespace: ai-models" - -Agent: Step 3 of 7: Creating NGC image pull secret... - -I'll create a docker-registry secret for pulling NIM images from nvcr.io: - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-image-pull-secret - namespace: ai-models - type: kubernetes.io/dockerconfigjson - data: - .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] - -Should I create this image pull secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - Image pull secret created successfully. - -Step 4 of 7: Creating NGC API key secret... - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-api-key - namespace: ai-models - type: Opaque - stringData: - NGC_API_KEY: "[REDACTED]" - -Should I create this API key secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - API key secret created successfully. - -Step 5 of 7: Creating NIM Account CR... - - apiVersion: nim.opendatahub.io/v1 - kind: Account - metadata: - name: nim-account - namespace: ai-models - spec: - apiKeySecret: - name: ngc-api-key - imagePullSecret: - name: ngc-image-pull-secret - -Should I create this NIM Account CR in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates Account CR via resources_create_or_update] - Account CR created successfully. - -Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) - -User: "no" - -Agent: Step 7 of 7: Validating NIM platform readiness... - -NIM Platform Setup Complete - -Account CR: `nim-account` (namespace: ai-models) - Status: Ready - -NGC Credentials: - Image Pull Secret: ngc-image-pull-secret [created] - API Key Secret: ngc-api-key [created] - -NIM ServingRuntimes Available: - - nim-llama-3-1-8b-instruct - - nim-llama-3-1-70b-instruct - - nim-mistral-7b-instruct - -Next Steps: - To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" -``` diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/references/known-model-profiles.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/references/known-model-profiles.md deleted file mode 100644 index 40148981..00000000 --- a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/references/known-model-profiles.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Known Model Hardware Profiles -category: references -tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] -semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] -use_cases: [model-deploy, debug-inference, ai-observability] -last_updated: 2026-03-10 ---- - -# Known Model Hardware Profiles - -Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). - -**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. - -## Model Source Conventions - -Each model lists a recommended `storageUri` with its authentication requirements: -- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** -- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). -- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. - -When the user does not specify a model source, use the `hf://` URI listed in the profile below. - -## Llama 3.x (Meta) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | -| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | -| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | -| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | - -- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) -- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly - -## Granite 3.x (IBM/Red Hat) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | -| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | - -- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) -- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser granite --chat-template` -- Red Hat-supported model family on RHOAI - -## Mixtral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | -| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) -- Mixture-of-Experts architecture: only ~13B/45B parameters active per token - -## Mistral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | -| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) - -## When a Model Is Not Listed - -If the requested model is not in this file, the agent MUST use the live doc lookup protocol: - -1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure -2. Fetch hardware specs from the appropriate source -3. Report findings to the user before proceeding with deployment - -Common cases requiring live lookup: -- Newly released models (after this file's last update) -- Domain-specific fine-tuned models -- Models with custom quantization -- NIM-specific optimized profiles diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/references/supported-runtimes.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/references/supported-runtimes.md deleted file mode 100644 index 9d018651..00000000 --- a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/references/supported-runtimes.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: Supported Serving Runtimes -category: references -tags: [runtimes, vllm, nim, caikit, tgis, serving] -semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] -use_cases: [model-deploy, serving-runtime-config, nim-setup] -last_updated: 2026-02-26 ---- - -# Supported Serving Runtimes - -This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. - -## Runtime Comparison - -| Runtime | API Type | Model Format | GPU Support | License | Setup Required | -|---------|----------|-------------|-------------|---------|----------------| -| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | -| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | -| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | -| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | - -## vLLM - -**Default runtime for most open-source models.** - -- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) -- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized -- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) -- **Model source**: S3-compatible storage, OCI registry, PVC, URI -- **Key features**: - - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) - - PagedAttention for efficient memory management - - Tensor parallelism for multi-GPU inference - - Tool/function calling support (`--tool-call-parser`) - - Continuous batching for high throughput -- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name - -## NVIDIA NIM - -**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** - -- **API**: OpenAI-compatible REST -- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog -- **GPU support**: NVIDIA only (requires specific GPU models per model profile) -- **Model source**: NGC model catalog (pulled automatically via NGC credentials) -- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) -- **Key features**: - - TensorRT-LLM optimization for lower latency - - Pre-compiled model engines (no compilation on first load) - - Optimized and generic profiles per GPU type - - Automatic model download from NGC -- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. -- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM -- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. -- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state - -## Caikit+TGIS - -**Red Hat's Caikit format with Text Generation Inference Server.** - -- **API**: gRPC (not REST) -- **Model formats**: Caikit format (requires conversion from HuggingFace) -- **GPU support**: NVIDIA -- **Model source**: S3-compatible storage -- **Key features**: - - Red Hat-supported runtime - - gRPC API for streaming inference - - Integrated with RHOAI model serving platform -- **When to choose**: When the model is already in Caikit format or when gRPC API is required. -- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name - -## Custom Runtimes - -**User-provided ServingRuntime CRs for unsupported frameworks.** - -- **API**: Defined by the custom runtime -- **Model formats**: Defined by the custom runtime -- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. -- **How to create**: Use `/serving-runtime-config` skill -- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility - -## Runtime Selection Decision Tree - -``` -Is the user's preferred runtime explicitly stated? -├── Yes → Use that runtime -└── No → Continue - -Is the model available in the NGC NIM catalog? -├── Yes → Suggest NIM (with vLLM as fallback) -│ Note: Requires /nim-setup and NVIDIA GPUs -└── No → Continue - -Is the model in Caikit format? -├── Yes → Caikit+TGIS -└── No → Continue - -Is the model a standard open-source LLM (HuggingFace-compatible)? -├── Yes → vLLM (default) -└── No → Custom runtime via /serving-runtime-config -``` diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/docs/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/debug-inference/SKILL.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/debug-inference/SKILL.md deleted file mode 100644 index 6a9a2d2a..00000000 --- a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/debug-inference/SKILL.md +++ /dev/null @@ -1,348 +0,0 @@ ---- -name: debug-inference -description: | - Troubleshoot failed or slow InferenceService deployments on OpenShift AI. - - Use when: - - "My InferenceService won't start" - - "Model deployment is stuck" - - "Inference endpoint returns errors" - - "Model is slow / high latency" - - "GPU scheduling failed for my model" - - Progressive diagnosis: status conditions, events, pod logs, GPU health, and observability analysis. - - NOT for deploying models (use /model-deploy). - NOT for creating runtimes (use /serving-runtime-config). -model: inherit -color: yellow ---- - -# /debug-inference Skill - -Troubleshoot failed, stuck, or slow InferenceService deployments on Red Hat OpenShift AI. Performs progressive diagnosis through status conditions, events, pod logs, related resources, and optional observability analysis. Follows a 6-step diagnosis pattern with human-in-the-loop confirmation at each step. - -## Prerequisites - -**Required MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) - -**Required MCP Tools** (from rhoai): -- `list_inference_services` - List deployed models with structured status data -- `get_inference_service` - Get detailed deployment status (conditions, endpoint, ready state) -- `get_model_endpoint` - Quick check if endpoint is available (early diagnostic) - -**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools** (from openshift): -- `resources_get` (from openshift) - Get ServingRuntime, NIM Account CR details -- `pods_list` (from openshift) - Find predictor/transformer pods -- `pods_log` (from openshift) - Retrieve container logs -- `events_list` (from openshift) - Check events for errors - -**Optional MCP Server**: `ai-observability` ([AI Observability MCP](https://github.com/rh-ai-quickstart/ai-observability-summarizer)) - -**Optional MCP Tools** (from ai-observability): -- `get_deployment_info` - Check model initialization status -- `analyze_vllm` - Analyze vLLM performance bottlenecks (latency, throughput, errors, token rates) -- `chat_vllm` - Conversational follow-up on vLLM metrics during diagnosis -- `get_gpu_info` - GPU inventory and utilization -- `analyze_openshift` - Check GPU health with "GPU & Accelerators" category -- `query_tempo_tool` - Trace request latency by service/operation/time range -- `get_trace_details_tool` - Get detailed span-level info for a specific trace ID -- `execute_promql` - Run custom PromQL queries for metrics not covered by standard analysis -- `korrel8r_get_correlated` - Correlate signals (logs, traces, metrics, alerts) across a pod/namespace for root cause analysis - -**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, KServe, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). - -**Additional cluster requirements**: -- An existing InferenceService deployment to debug - -## When to Use This Skill - -**Use this skill when you need to:** -- Troubleshoot an InferenceService that won't start, is stuck, or shows errors -- Diagnose slow inference latency or high error rates -- Investigate GPU scheduling failures or OOMKilled pods -- Perform root cause analysis on model deployment issues - -**Do NOT use this skill when:** -- You want to deploy a new model (use `/model-deploy`) -- You want to analyze ongoing model performance (use `/ai-observability`) -- You need to create or fix a ServingRuntime (use `/serving-runtime-config`) -- You need to set up NIM credentials (use `/nim-setup`) - -## Workflow - -### Step 1: Identify Target InferenceService - -**Ask the user:** -- Which InferenceService is having issues? (name or "list all") -- What namespace is it in? -- What is the symptom? (won't start / slow / errors / other) - -If user says "list all" or is unsure: - -**MCP Tool**: `list_inference_services` (from rhoai) - -**Parameters**: -- `namespace`: user-specified namespace - REQUIRED -- `verbosity`: `"standard"` - OPTIONAL - -Present InferenceServices with their status: - -| Name | Runtime | Ready | URL | Age | -|------|---------|-------|-----|-----| -| [name] | [runtime] | [True/False/Unknown] | [url or "N/A"] | [age] | - -Proceed automatically without waiting for user input. - -### Step 2: Status Overview - -**MCP Tool**: `get_inference_service` (from rhoai) - -**Parameters**: -- `name`: the InferenceService name - REQUIRED -- `namespace`: user-specified namespace - REQUIRED -- `verbosity`: `"full"` - REQUIRED - -**Early endpoint check:** - -**MCP Tool**: `get_model_endpoint` (from rhoai) -- `name`: the InferenceService name, `namespace`: user-specified namespace - -An empty or error URL indicates deployment issues. Report endpoint availability status. - -Present status conditions: - -| Condition | Status | Reason | Message | -|-----------|--------|--------|---------| -| Ready | [True/False/Unknown] | [reason] | [message] | -| PredictorReady | [True/False/Unknown] | [reason] | [message] | -| IngressReady | [True/False/Unknown] | [reason] | [message] | - -**Quick Assessment**: Based on conditions, provide initial assessment (e.g., "PredictorReady is False -- the model container is not running. Likely a pod-level issue.") - -**Ask**: "Continue with deep analysis of events and pods? (yes/no)" - -Proceed automatically without waiting for user input. - -### Step 3: Events and Pod Analysis - -**MCP Tool**: `events_list` (from openshift) - -**Parameters**: -- `namespace`: user-specified namespace - REQUIRED - -Filter events related to the InferenceService name. - -**MCP Tool**: `pods_list` (from openshift) - -**Parameters**: -- `namespace`: user-specified namespace - REQUIRED -- `labelSelector`: `"serving.kserve.io/inferenceservice=[isvc-name]"` - REQUIRED - -Present findings: - -**Events:** - -| Time | Type | Reason | Message | -|------|------|--------|---------| -| [time] | [Normal/Warning] | [reason] | [message] | - -**Predictor Pods:** - -| Pod | Status | Restarts | Node | GPU | -|-----|--------|----------|------|-----| -| [pod-name] | [status] | [count] | [node] | [gpu-count] | - -**Issues Found:** -- [Issue from events or pod status] - -**Ask**: "Continue to view pod logs? (yes/no)" - -Proceed automatically without waiting for user input. - -### Step 4: Pod Logs Review - -**MCP Tool**: `pods_log` (from openshift) - -**Parameters**: -- `namespace`: user-specified namespace - REQUIRED -- `name`: predictor pod name from Step 3 - REQUIRED -- `container`: `"kserve-container"` - REQUIRED (main serving container) - -If the container has restarted, also retrieve previous logs. - -Present log analysis: - -**Log Analysis:** -- [Error pattern identified, e.g., "CUDA out of memory", "S3 access denied", "Model not found"] -- [Relevant log line with explanation] - -**For NIM-specific deployments**, also check: -- NGC authentication errors in logs -- TensorRT engine compilation status -- GPU compatibility messages - -**If the error is unrecognized -> Trigger live doc lookup:** -1. **Action**: Read [live-doc-lookup.md](../references/live-doc-lookup.md) using the Read tool -2. Use **WebFetch** to look up the error message in RHOAI documentation -3. **Output to user**: "I looked up this error on [source]: [explanation and fix]" - -**Ask**: "Continue to check related resources and observability? (yes/no)" - -Proceed automatically without waiting for user input. - -### Step 5: Related Resources and Observability - -**Check ServingRuntime:** - -**MCP Tool**: `resources_get` (from openshift) - -**Parameters**: -- `apiVersion`: `"serving.kserve.io/v1alpha1"` - REQUIRED -- `kind`: `"ServingRuntime"` - REQUIRED -- `namespace`: user-specified namespace - REQUIRED -- `name`: runtime name from the InferenceService spec - REQUIRED - -Verify the runtime exists and its model format matches the InferenceService. - -**For NIM deployments -- Check Account CR:** - -**MCP Tool**: `resources_get` (from openshift) - -**Parameters**: -- `apiVersion`: `"nim.opendatahub.io/v1alpha1"` - REQUIRED -- `kind`: `"Account"` - REQUIRED -- `namespace`: user-specified namespace - REQUIRED -- `name`: `"nim-account"` - REQUIRED - -**If ai-observability MCP is available:** - -- `get_deployment_info`: Check if the model appears in monitoring and its initialization status -- `analyze_vllm`: Analyze performance metrics for slow inference (latency, throughput, errors, token rates) -- `chat_vllm`: Ask follow-up questions about analyzed metrics (e.g., "Why is latency spiking?") -- `analyze_openshift` with category `"GPU & Accelerators"`: Check GPU health and utilization -- `query_tempo_tool`: Trace request latency if the symptom is slow responses -- `get_trace_details_tool`: Drill into a specific trace ID to see span-level timing -- `execute_promql`: Run custom PromQL queries for deeper metric investigation (e.g., `vllm:request_success:ratio`, GPU memory utilization) -- `korrel8r_get_correlated`: Correlate signals across the inference stack -- find related logs, traces, metrics, and alerts for the failing pod/namespace (query example: `k8s:Pod:{"namespace":"[ns]","name":"[pod-name]"}`, goals: `["log:application", "metric:metric", "trace:span"]`) - -**If ai-observability not available**: Skip with note: "Observability analysis skipped (ai-observability MCP not configured)." - -**Present findings:** -- ServingRuntime status and compatibility -- NIM Account CR status (if applicable) -- Observability insights (if available) - -**Ask**: "Continue to diagnosis summary? (yes/no)" - -Proceed automatically without waiting for user input. - -### Step 6: Diagnosis Summary - -Present a structured diagnosis: - -``` -## Diagnosis Summary: [isvc-name] - -### Root Cause - -**Primary Issue:** [Categorized root cause] - -| Category | Status | Details | -|----------|--------|---------| -| ServingRuntime | [OK/FAIL] | [details] | -| Pod Scheduling | [OK/FAIL] | [details] | -| Container Start | [OK/FAIL] | [details] | -| Model Loading | [OK/FAIL] | [details] | -| GPU Access | [OK/FAIL] | [details] | -| Endpoint Health | [OK/FAIL] | [details] | - -### Evidence - -- [Evidence 1 from events/logs/status] -- [Evidence 2] - -### Recommended Actions - -1. **[Action 1]** - [description] -2. **[Action 2]** - [description] -3. **[Action 3]** - [description] - -### Verification Steps - -After applying fixes: -1. Check InferenceService status: `resources_get` for the InferenceService -2. Verify pod is running: `pods_list` with label selector -3. Test endpoint: curl command to the inference URL -``` - -**End with options:** - -``` -Would you like me to: -1. Execute a recommended fix -2. Dig deeper into a specific area -3. Debug a related resource (ServingRuntime, pod, NIM Account) -4. Invoke /serving-runtime-config to fix runtime issues -5. Exit debugging -``` - -Proceed automatically without waiting for user input. - -## Common Issues - -For common issues (GPU scheduling, OOMKilled, image pull errors, RBAC), see [common-issues.md](../references/common-issues.md). - -### Issue 1: S3 Storage Access Denied - -**Error**: Pod logs show "Access Denied" or "NoSuchBucket" when loading model weights - -**Cause**: S3 credentials are missing, expired, or the bucket/path is incorrect. - -**Solution:** -1. Verify the `storageUri` in the InferenceService spec -2. Check that the S3 credential Secret exists in the namespace -3. Verify the Secret is referenced by the ServiceAccount or data connection -4. Test S3 access independently to confirm credentials are valid - -### Issue 2: NIM Authentication / GPU Incompatibility - -**Error**: NIM pod logs show NGC authentication failure, or TensorRT engine fails to compile for the available GPU - -**Cause**: NGC API key is invalid/expired, or the GPU type is not supported by the NIM model profile. - -**Solution:** -1. Check Account CR status for credential errors: `resources_get` for `accounts.nim.opendatahub.io` -2. Verify NGC API key is valid at https://ngc.nvidia.com -3. Check NIM supported GPU matrix via live doc lookup against [NVIDIA NIM supported models](https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html) -4. Re-run `/nim-setup` to refresh credentials if expired - -## Dependencies - -### MCP Tools -See [Prerequisites](#prerequisites) for the complete list of required and optional MCP tools. - -### Related Skills -- `/model-deploy` - Redeploy or modify the InferenceService after fixing issues -- `/serving-runtime-config` - Fix or create ServingRuntime if runtime is the issue -- `/nim-setup` - Re-run NIM platform setup if NIM credentials are the issue - -### Reference Documentation -- [known-model-profiles.md](../../docs/references/known-model-profiles.md) - Correct resource sizing for common models -- [supported-runtimes.md](../../docs/references/supported-runtimes.md) - Runtime capabilities and known limitations -- [live-doc-lookup.md](../references/live-doc-lookup.md) - Protocol for looking up unrecognized errors - -## Critical: Human-in-the-Loop Requirements - -See [skill-conventions.md](../references/skill-conventions.md) for general HITL and security conventions. - -**Skill-specific checkpoints:** -- After identifying target (Step 1): confirm which InferenceService to debug -- After status overview (Step 2): confirm before deep analysis -- After events/pod analysis (Step 3): confirm before viewing logs -- After log review (Step 4): confirm before checking related resources -- After diagnosis summary (Step 6): present options, wait for user decision -- **NEVER** auto-delete or auto-modify InferenceService resources without user confirmation -- **NEVER** execute remediation actions without presenting the plan and getting explicit approval diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/common-issues.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/with_skills/rh-ai-engineer__debug-inference/environment/skills/references/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/common-issues.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/model-deploy.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/model-deploy.md deleted file mode 100644 index dcd79e44..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/model-deploy.md +++ /dev/null @@ -1,160 +0,0 @@ -# /model-deploy Examples - -Example interactions demonstrating the model-deploy skill workflow. - -## Example 1: Deploy Llama 3.1 8B with vLLM - -``` -User: "Deploy Llama 3.1 8B on my cluster" - -Agent: [Invokes /model-deploy skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand runtime capabilities. - -Step 1 of 10: Gathering deployment information. - -| Setting | Value | Source | -|---------|-------|--------| -| Model | Llama 3.1 8B Instruct | user input | -| Runtime | TBD | auto-detect | -| Namespace | ? | needs input | -| Model Source | ? | needs input | -| Deployment Mode | Serverless | default | - -What namespace should I deploy to, and where are the model weights stored? -(e.g., S3 bucket, OCI registry, PVC) - -User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" - -Agent: Step 2 of 10: Determining runtime. - -I consulted supported-runtimes.md to understand runtime capabilities. - -Based on Llama 3.1 8B (open-source HuggingFace model): - -Recommended runtime: vLLM -Reason: Default runtime for open-source LLMs. Broad compatibility, - OpenAI-compatible API, no additional setup required. - -Proceed with vLLM? (yes / no / suggest alternative) - -User: "yes" - -Agent: Step 3 of 10: Looking up hardware profile. - -I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. - -| Requirement | Value | -|-------------|-------| -| GPUs | 1x A10G/L4/A100/H100 | -| VRAM | 16GB | -| Key Args | --max-model-len=4096 --tool-call-parser hermes | - -Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). -Ensure your cluster has at least 1 GPU with 16GB VRAM. - -Step 5 of 10: Skipped (NIM platform check not needed for vLLM). - -Step 6-7 of 10: Generating InferenceService YAML. - -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - name: llama-3-1-8b - namespace: ai-models - annotations: - serving.kserve.io/deploymentMode: Serverless -spec: - predictor: - model: - modelFormat: - name: vLLM - runtime: vllm-runtime - storageUri: s3://my-bucket/llama-3.1-8b - resources: - limits: - nvidia.com/gpu: "1" - requests: - cpu: "4" - memory: "16Gi" - args: - - --max-model-len=4096 - - --tool-call-parser=hermes - -| Setting | Value | -|---------|-------| -| Model | llama-3-1-8b | -| Runtime | vllm-runtime | -| GPUs | 1 | -| Namespace | ai-models | -| Mode | Serverless | - -Proceed with creating this InferenceService? (yes/no/modify) - -User: "yes" - -Agent: Step 8 of 10: Creating InferenceService... - [Creates via resources_create_or_update] - - InferenceService created. Monitoring rollout... - -Step 9 of 10: Monitoring rollout. - -| Stage | Status | -|-------|--------| -| Pod Scheduled | Complete | -| Image Pulled | Complete | -| Container Started | Complete | -| Model Loaded | Complete | -| Ready | Complete | - -Step 10 of 10: Deployment complete! - -Model Deployment Successful - -Model: llama-3-1-8b -Runtime: vLLM -Namespace: ai-models -GPUs: 1 - -Inference Endpoint: - URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com - API: OpenAI-compatible REST - -Quick Test: - curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' - -Next Steps: - Monitor performance: "/ai-observability llama-3-1-8b" -``` - -## Example 2: Deploy a Model with NIM (NIM Not Set Up) - -``` -User: "Deploy Llama 3.1 70B with NIM" - -Agent: [Invokes /model-deploy skill] - ... - Step 2: User requested NIM runtime. - - Step 5: Checking NIM platform... - -NIM platform is not set up in namespace ai-models. - -The NIM runtime requires a one-time platform setup. Would you like to: -1. Run /nim-setup now to configure the NIM platform -2. Switch to vLLM runtime instead (no setup required) -3. Abort deployment - -User: "1" - -Agent: [Invokes /nim-setup skill — walks through NGC credential setup] - ... - NIM Platform Setup Complete. - - Resuming model deployment... - [Continues from Step 6 with NIM runtime] -``` diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/nim-setup.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/nim-setup.md deleted file mode 100644 index ccfc1a5d..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/nim-setup.md +++ /dev/null @@ -1,115 +0,0 @@ -# /nim-setup Examples - -Example interactions demonstrating the nim-setup skill workflow. - -## Example 1: First-Time NIM Setup - -``` -User: "Set up NIM on my cluster" - -Agent: [Invokes /nim-setup skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand NIM platform requirements. - -Let me verify the prerequisites for NIM setup. - -Step 1 of 7: Checking GPU Operator and NFD Operator... - -GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) -NFD Operator: nfd.v4.19.0 (Succeeded) - -Both operators are installed and healthy. - -Step 2 of 7: I need your NGC credentials. - -To set up NIM, I need your NVIDIA NGC API key. -You can generate one at: https://ngc.nvidia.com/setup/api-key - -Please provide: -1. Your NGC API key -2. The target namespace for NIM resources - -User: "Here's my key: [user provides NGC API key], namespace: ai-models" - -Agent: Step 3 of 7: Creating NGC image pull secret... - -I'll create a docker-registry secret for pulling NIM images from nvcr.io: - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-image-pull-secret - namespace: ai-models - type: kubernetes.io/dockerconfigjson - data: - .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] - -Should I create this image pull secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - Image pull secret created successfully. - -Step 4 of 7: Creating NGC API key secret... - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-api-key - namespace: ai-models - type: Opaque - stringData: - NGC_API_KEY: "[REDACTED]" - -Should I create this API key secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - API key secret created successfully. - -Step 5 of 7: Creating NIM Account CR... - - apiVersion: nim.opendatahub.io/v1 - kind: Account - metadata: - name: nim-account - namespace: ai-models - spec: - apiKeySecret: - name: ngc-api-key - imagePullSecret: - name: ngc-image-pull-secret - -Should I create this NIM Account CR in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates Account CR via resources_create_or_update] - Account CR created successfully. - -Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) - -User: "no" - -Agent: Step 7 of 7: Validating NIM platform readiness... - -NIM Platform Setup Complete - -Account CR: `nim-account` (namespace: ai-models) - Status: Ready - -NGC Credentials: - Image Pull Secret: ngc-image-pull-secret [created] - API Key Secret: ngc-api-key [created] - -NIM ServingRuntimes Available: - - nim-llama-3-1-8b-instruct - - nim-llama-3-1-70b-instruct - - nim-mistral-7b-instruct - -Next Steps: - To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" -``` diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/known-model-profiles.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/known-model-profiles.md deleted file mode 100644 index 40148981..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/known-model-profiles.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Known Model Hardware Profiles -category: references -tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] -semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] -use_cases: [model-deploy, debug-inference, ai-observability] -last_updated: 2026-03-10 ---- - -# Known Model Hardware Profiles - -Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). - -**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. - -## Model Source Conventions - -Each model lists a recommended `storageUri` with its authentication requirements: -- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** -- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). -- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. - -When the user does not specify a model source, use the `hf://` URI listed in the profile below. - -## Llama 3.x (Meta) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | -| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | -| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | -| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | - -- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) -- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly - -## Granite 3.x (IBM/Red Hat) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | -| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | - -- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) -- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser granite --chat-template` -- Red Hat-supported model family on RHOAI - -## Mixtral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | -| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) -- Mixture-of-Experts architecture: only ~13B/45B parameters active per token - -## Mistral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | -| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) - -## When a Model Is Not Listed - -If the requested model is not in this file, the agent MUST use the live doc lookup protocol: - -1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure -2. Fetch hardware specs from the appropriate source -3. Report findings to the user before proceeding with deployment - -Common cases requiring live lookup: -- Newly released models (after this file's last update) -- Domain-specific fine-tuned models -- Models with custom quantization -- NIM-specific optimized profiles diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/supported-runtimes.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/supported-runtimes.md deleted file mode 100644 index 9d018651..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/supported-runtimes.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: Supported Serving Runtimes -category: references -tags: [runtimes, vllm, nim, caikit, tgis, serving] -semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] -use_cases: [model-deploy, serving-runtime-config, nim-setup] -last_updated: 2026-02-26 ---- - -# Supported Serving Runtimes - -This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. - -## Runtime Comparison - -| Runtime | API Type | Model Format | GPU Support | License | Setup Required | -|---------|----------|-------------|-------------|---------|----------------| -| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | -| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | -| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | -| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | - -## vLLM - -**Default runtime for most open-source models.** - -- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) -- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized -- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) -- **Model source**: S3-compatible storage, OCI registry, PVC, URI -- **Key features**: - - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) - - PagedAttention for efficient memory management - - Tensor parallelism for multi-GPU inference - - Tool/function calling support (`--tool-call-parser`) - - Continuous batching for high throughput -- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name - -## NVIDIA NIM - -**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** - -- **API**: OpenAI-compatible REST -- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog -- **GPU support**: NVIDIA only (requires specific GPU models per model profile) -- **Model source**: NGC model catalog (pulled automatically via NGC credentials) -- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) -- **Key features**: - - TensorRT-LLM optimization for lower latency - - Pre-compiled model engines (no compilation on first load) - - Optimized and generic profiles per GPU type - - Automatic model download from NGC -- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. -- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM -- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. -- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state - -## Caikit+TGIS - -**Red Hat's Caikit format with Text Generation Inference Server.** - -- **API**: gRPC (not REST) -- **Model formats**: Caikit format (requires conversion from HuggingFace) -- **GPU support**: NVIDIA -- **Model source**: S3-compatible storage -- **Key features**: - - Red Hat-supported runtime - - gRPC API for streaming inference - - Integrated with RHOAI model serving platform -- **When to choose**: When the model is already in Caikit format or when gRPC API is required. -- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name - -## Custom Runtimes - -**User-provided ServingRuntime CRs for unsupported frameworks.** - -- **API**: Defined by the custom runtime -- **Model formats**: Defined by the custom runtime -- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. -- **How to create**: Use `/serving-runtime-config` skill -- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility - -## Runtime Selection Decision Tree - -``` -Is the user's preferred runtime explicitly stated? -├── Yes → Use that runtime -└── No → Continue - -Is the model available in the NGC NIM catalog? -├── Yes → Suggest NIM (with vLLM as fallback) -│ Note: Requires /nim-setup and NVIDIA GPUs -└── No → Continue - -Is the model in Caikit format? -├── Yes → Caikit+TGIS -└── No → Continue - -Is the model a standard open-source LLM (HuggingFace-compatible)? -├── Yes → vLLM (default) -└── No → Custom runtime via /serving-runtime-config -``` diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/docs/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/ds-project-setup/SKILL.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/ds-project-setup/SKILL.md deleted file mode 100644 index af660b2b..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/ds-project-setup/SKILL.md +++ /dev/null @@ -1,336 +0,0 @@ ---- -name: ds-project-setup -description: | - Create and configure Data Science Projects on OpenShift AI with namespace setup, S3 data connections, pipeline server, and model serving enablement. - - Use when: - - "Create a data science project" - - "Set up a new namespace for ML work" - - "Add an S3 data connection to my project" - - "Configure the pipeline server" - - "Enable model serving on my project" - - Bootstraps an RHOAI Data Science Project with proper labels, data connections, pipeline infrastructure, and model serving configuration. - - NOT for deploying models (use /model-deploy). - NOT for creating workbenches (use /workbench-manage). - NOT for managing pipelines after setup (use /pipeline-manage). -color: green -model: inherit -metadata: - author: "Red Hat Ecosystem Engineering" - version: "1.0" ---- - -# /ds-project-setup Skill - -Bootstrap a Red Hat OpenShift AI Data Science Project from scratch. Creates a namespace with RHOAI dashboard labels, configures S3-compatible data connections, sets up the pipeline server with external storage, and enables model serving on the project. - -## Prerequisites - -**Required MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) - -**Required MCP Tools** (from rhoai): -- `list_data_science_projects` - List existing RHOAI projects to check for duplicates -- `create_data_science_project` - Create namespace with RHOAI labels and dashboard integration -- `get_project_details` - Verify project creation and inspect configuration -- `get_project_status` - Get comprehensive project status including components -- `create_s3_data_connection` - Create S3-compatible data connection secret -- `list_data_connections` - List existing data connections in the project -- `get_pipeline_server` - Check pipeline server configuration -- `create_pipeline_server` - Configure pipeline server with S3 data connection -- `set_model_serving_mode` - Enable single-model or multi-model serving - -**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools** (from openshift): -- `resources_get` (from openshift) - Inspect namespace labels, LimitRange, ResourceQuota - -**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). - -**Additional cluster requirements**: -- Cluster admin or namespace creation privileges for the user - -## When to Use This Skill - -**Use this skill when you need to:** -- Create a new Data Science Project namespace for an ML team -- Add S3 data connections to an existing project -- Configure the pipeline server on a project -- Enable or change the model serving mode (single vs multi-model) -- Bootstrap a complete project environment before deploying models or workbenches - -**Do NOT use this skill when:** -- You want to deploy a model (use `/model-deploy`) -- You need to create a notebook workbench (use `/workbench-manage`) -- You need to manage pipeline runs (use `/pipeline-manage`) -- You need to configure a custom serving runtime (use `/serving-runtime-config`) - -## Workflow - -### Step 1: Gather Requirements - -**Ask the user for:** -- **Project name**: DNS-compatible name for the namespace (lowercase, no spaces, max 63 chars) -- **Display name**: Human-readable project name for the RHOAI dashboard -- **Description**: Optional project description -- **Data connections**: Whether to configure S3 data connections (yes/no) -- **Pipeline server**: Whether to configure the pipeline server (yes/no, requires data connection) -- **Model serving mode**: "single" (default, one model per endpoint) or "multi" (multiple models per endpoint) - -**Present configuration table:** - -| Setting | Value | -|---------|-------| -| Project name | [name] | -| Display name | [display_name] | -| Description | [description] | -| Data connections | [yes/no] | -| Pipeline server | [yes/no] | -| Model serving mode | [single/multi] | - -Proceed automatically without waiting for user input. - -### Step 2: Check Existing Projects - -**MCP Tool**: `list_data_science_projects` (from rhoai) - -**Parameters**: none - -Check if the project name already exists in the cluster. - -- If project **exists**: Report to user and offer options: "Project `[name]` already exists. Would you like to: (a) configure additional components on it, or (b) choose a different name?" -- If project **does not exist**: Proceed to Step 3 - -Proceed automatically without waiting for user input. - -### Step 3: Create Data Science Project - -**MCP Tool**: `create_data_science_project` (from rhoai) - -**Parameters**: -- `name`: project name from Step 1 - REQUIRED (DNS-compatible: lowercase alphanumeric and hyphens, max 63 chars) -- `display_name`: human-readable display name - REQUIRED -- `description`: project description - OPTIONAL - -**Verify creation:** - -**MCP Tool**: `get_project_details` (from rhoai) - -**Parameters**: -- `name`: the created project name - REQUIRED - -Confirm the project was created with proper RHOAI labels (`opendatahub.io/dashboard: "true"`). - -**Error Handling**: -- If name already taken -> Offer alternative name or configure existing project -- If RBAC error -> Report: "Insufficient permissions to create namespaces. Contact your cluster administrator." -- If name invalid -> Report DNS naming constraints and suggest a valid name - -**Output to user**: "Data Science Project `[name]` created successfully." - -### Step 4: Configure Data Connections (Optional) - -Skip this step if user declined data connections in Step 1. - -**Ask the user for S3 connection details:** -- **Connection name**: Identifier for this data connection -- **S3 bucket**: Target bucket name -- **S3 endpoint**: S3-compatible endpoint URL (e.g., `https://s3.amazonaws.com`, MinIO endpoint) -- **Access key**: AWS access key ID or S3-compatible access key -- **Secret key**: AWS secret access key or S3-compatible secret key -- **Region**: AWS region or empty for non-AWS S3 - -**Display connection configuration** (credentials REDACTED): - -| Setting | Value | -|---------|-------| -| Connection name | [name] | -| Bucket | [bucket] | -| Endpoint | [endpoint] | -| Access key | [first-4-chars]**** | -| Secret key | ******** | -| Region | [region] | - -Proceed automatically without waiting for user input. - -**MCP Tool**: `create_s3_data_connection` (from rhoai) - -**Parameters**: -- `namespace`: project name from Step 3 - REQUIRED -- `name`: connection name - REQUIRED -- `bucket`: S3 bucket name - REQUIRED -- `endpoint`: S3 endpoint URL - REQUIRED -- `access_key`: access key ID - REQUIRED -- `secret_key`: secret access key - REQUIRED -- `region`: AWS region - OPTIONAL (omit for non-AWS S3) - -**Verify creation:** - -**MCP Tool**: `list_data_connections` (from rhoai) - -**Parameters**: -- `namespace`: project name - REQUIRED - -Confirm the data connection appears in the list. - -**Error Handling**: -- If connection name already exists -> Ask: "Data connection `[name]` already exists. Create with a different name?" -- If RBAC error -> Report insufficient permissions to create Secrets in namespace - -**Output to user**: "Data connection `[name]` created in project `[namespace]`." - -**Repeat this step** if user wants to create multiple data connections. - -### Step 5: Configure Pipeline Server (Optional) - -Skip this step if user declined pipeline server in Step 1. - -**Prerequisite check**: A data connection must exist in the project (from Step 4 or pre-existing). If no data connections exist, inform user: "Pipeline server requires an S3 data connection for artifact storage. Would you like to create one now?" and return to Step 4. - -**MCP Tool**: `get_pipeline_server` (from rhoai) - -**Parameters**: -- `namespace`: project name - REQUIRED - -If pipeline server already exists, report its status and ask if user wants to reconfigure. - -**Display pipeline server configuration:** - -| Setting | Value | -|---------|-------| -| Namespace | [namespace] | -| Data connection | [data_connection_name] | - -Proceed automatically without waiting for user input. - -**MCP Tool**: `create_pipeline_server` (from rhoai) - -**Parameters**: -- `namespace`: project name - REQUIRED -- `data_connection`: name of the S3 data connection to use for pipeline artifacts - REQUIRED - -**Verify creation:** - -**MCP Tool**: `get_pipeline_server` (from rhoai) - -**Parameters**: -- `namespace`: project name - REQUIRED - -Confirm the pipeline server is configured and initializing. - -**Error Handling**: -- If data connection not found -> Report: "Data connection `[name]` not found in namespace. Create it first." -- If pipeline server already exists -> Ask user whether to reconfigure or keep existing -- If RBAC error -> Report insufficient permissions - -**Output to user**: "Pipeline server configured in project `[namespace]` using data connection `[data_connection]`." - -### Step 6: Enable Model Serving and Report - -**MCP Tool**: `set_model_serving_mode` (from rhoai) - -**Parameters**: -- `namespace`: project name - REQUIRED -- `mode`: "single" or "multi" - REQUIRED (default: "single") - -**Final validation:** - -**MCP Tool**: `get_project_status` (from rhoai) - -**Parameters**: -- `namespace`: project name - REQUIRED - -**Report project summary:** - -| Component | Status | -|-----------|--------| -| Project | [name] (created / existing) | -| Data connections | [count] configured | -| Pipeline server | [configured / not configured] | -| Model serving | [single / multi] mode enabled | - -**Suggest next steps:** -- `/workbench-manage` - Create a notebook workbench in this project -- `/model-deploy` - Deploy a model to this project -- `/pipeline-manage` - Create and run data science pipelines - -## Common Issues - -### Issue 1: Project Name Already Exists - -**Error**: `create_data_science_project` returns conflict error - -**Cause**: A namespace with the same name already exists in the cluster, either as an RHOAI project or a regular OpenShift project. - -**Solution:** -1. Use `list_data_science_projects` to check if it is an existing RHOAI project -2. If it is an RHOAI project, offer to configure additional components on it -3. If it is a regular namespace (not an RHOAI project), suggest a different name or advise converting it by adding the `opendatahub.io/dashboard: "true"` label - -### Issue 2: S3 Endpoint Unreachable - -**Error**: Data connection created but pipeline server or model serving cannot access storage - -**Cause**: The S3 endpoint URL is malformed, unreachable from the cluster, or requires TLS configuration. - -**Solution:** -1. Verify the endpoint URL format includes the protocol (`https://`) -2. For MinIO: use the internal cluster service URL (e.g., `http://minio.minio-ns.svc:9000`) -3. For AWS: use the regional endpoint (e.g., `https://s3.us-east-1.amazonaws.com`) -4. Check if the cluster has network egress restrictions that block external S3 access - -### Issue 3: Pipeline Server Fails to Initialize - -**Error**: Pipeline server status remains unhealthy or pods crash - -**Cause**: Usually caused by an invalid data connection (wrong credentials or unreachable bucket), or insufficient cluster resources. - -**Solution:** -1. Verify the data connection credentials are correct (re-create if needed) -2. Check that the S3 bucket exists and is accessible with the provided credentials -3. Check namespace ResourceQuota for pod limits -4. Review pipeline server pod logs via `pods_log` (from openshift) for specific error messages - -### Issue 4: Namespace Quota Exceeded - -**Error**: Resource creation fails with quota exceeded error - -**Cause**: The cluster has ResourceQuota or LimitRange policies that restrict resource creation in the namespace. - -**Solution:** -1. Use `resources_get` (from openshift) to inspect ResourceQuota in the namespace -2. Report the quota limits to the user -3. Suggest contacting the cluster administrator to increase quotas or clean up unused resources - -## Dependencies - -### MCP Tools -See [Prerequisites](#prerequisites) for the complete list of required and optional MCP tools. - -### Related Skills -- `/workbench-manage` - Create notebook workbenches in the project -- `/model-deploy` - Deploy models to the project -- `/pipeline-manage` - Create and manage pipeline runs -- `/serving-runtime-config` - Configure custom serving runtimes in the project - -### Reference Documentation -- [skill-conventions.md](../references/skill-conventions.md) - Shared prerequisite, HITL, and security conventions - -## Example Usage - -**User**: "Create a data science project called fraud-detection with an S3 connection and pipeline server" - -**Skill response**: Gathers requirements, presents configuration table, creates project `fraud-detection`, configures S3 data connection (credentials redacted in display), sets up pipeline server, enables single-model serving, and reports final project status with next steps. - -## Critical: Human-in-the-Loop Requirements - -See [skill-conventions.md](../references/skill-conventions.md) for general HITL and security conventions. - -**Skill-specific checkpoints:** -- After gathering requirements (Step 1): confirm project configuration table -- Before creating data connections (Step 4): display connection config with credentials REDACTED, confirm -- Before configuring pipeline server (Step 5): confirm data connection selection -- If project already exists (Step 2): confirm whether to configure existing or choose new name -- **NEVER** create data connections without user confirming credential details -- **NEVER** display actual S3 access keys or secret keys in output diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/common-issues.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/with_skills/rh-ai-engineer__ds-project-setup/environment/skills/references/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/common-issues.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/examples/model-deploy.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/examples/model-deploy.md deleted file mode 100644 index dcd79e44..00000000 --- a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/examples/model-deploy.md +++ /dev/null @@ -1,160 +0,0 @@ -# /model-deploy Examples - -Example interactions demonstrating the model-deploy skill workflow. - -## Example 1: Deploy Llama 3.1 8B with vLLM - -``` -User: "Deploy Llama 3.1 8B on my cluster" - -Agent: [Invokes /model-deploy skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand runtime capabilities. - -Step 1 of 10: Gathering deployment information. - -| Setting | Value | Source | -|---------|-------|--------| -| Model | Llama 3.1 8B Instruct | user input | -| Runtime | TBD | auto-detect | -| Namespace | ? | needs input | -| Model Source | ? | needs input | -| Deployment Mode | Serverless | default | - -What namespace should I deploy to, and where are the model weights stored? -(e.g., S3 bucket, OCI registry, PVC) - -User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" - -Agent: Step 2 of 10: Determining runtime. - -I consulted supported-runtimes.md to understand runtime capabilities. - -Based on Llama 3.1 8B (open-source HuggingFace model): - -Recommended runtime: vLLM -Reason: Default runtime for open-source LLMs. Broad compatibility, - OpenAI-compatible API, no additional setup required. - -Proceed with vLLM? (yes / no / suggest alternative) - -User: "yes" - -Agent: Step 3 of 10: Looking up hardware profile. - -I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. - -| Requirement | Value | -|-------------|-------| -| GPUs | 1x A10G/L4/A100/H100 | -| VRAM | 16GB | -| Key Args | --max-model-len=4096 --tool-call-parser hermes | - -Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). -Ensure your cluster has at least 1 GPU with 16GB VRAM. - -Step 5 of 10: Skipped (NIM platform check not needed for vLLM). - -Step 6-7 of 10: Generating InferenceService YAML. - -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - name: llama-3-1-8b - namespace: ai-models - annotations: - serving.kserve.io/deploymentMode: Serverless -spec: - predictor: - model: - modelFormat: - name: vLLM - runtime: vllm-runtime - storageUri: s3://my-bucket/llama-3.1-8b - resources: - limits: - nvidia.com/gpu: "1" - requests: - cpu: "4" - memory: "16Gi" - args: - - --max-model-len=4096 - - --tool-call-parser=hermes - -| Setting | Value | -|---------|-------| -| Model | llama-3-1-8b | -| Runtime | vllm-runtime | -| GPUs | 1 | -| Namespace | ai-models | -| Mode | Serverless | - -Proceed with creating this InferenceService? (yes/no/modify) - -User: "yes" - -Agent: Step 8 of 10: Creating InferenceService... - [Creates via resources_create_or_update] - - InferenceService created. Monitoring rollout... - -Step 9 of 10: Monitoring rollout. - -| Stage | Status | -|-------|--------| -| Pod Scheduled | Complete | -| Image Pulled | Complete | -| Container Started | Complete | -| Model Loaded | Complete | -| Ready | Complete | - -Step 10 of 10: Deployment complete! - -Model Deployment Successful - -Model: llama-3-1-8b -Runtime: vLLM -Namespace: ai-models -GPUs: 1 - -Inference Endpoint: - URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com - API: OpenAI-compatible REST - -Quick Test: - curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' - -Next Steps: - Monitor performance: "/ai-observability llama-3-1-8b" -``` - -## Example 2: Deploy a Model with NIM (NIM Not Set Up) - -``` -User: "Deploy Llama 3.1 70B with NIM" - -Agent: [Invokes /model-deploy skill] - ... - Step 2: User requested NIM runtime. - - Step 5: Checking NIM platform... - -NIM platform is not set up in namespace ai-models. - -The NIM runtime requires a one-time platform setup. Would you like to: -1. Run /nim-setup now to configure the NIM platform -2. Switch to vLLM runtime instead (no setup required) -3. Abort deployment - -User: "1" - -Agent: [Invokes /nim-setup skill — walks through NGC credential setup] - ... - NIM Platform Setup Complete. - - Resuming model deployment... - [Continues from Step 6 with NIM runtime] -``` diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/examples/nim-setup.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/examples/nim-setup.md deleted file mode 100644 index ccfc1a5d..00000000 --- a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/examples/nim-setup.md +++ /dev/null @@ -1,115 +0,0 @@ -# /nim-setup Examples - -Example interactions demonstrating the nim-setup skill workflow. - -## Example 1: First-Time NIM Setup - -``` -User: "Set up NIM on my cluster" - -Agent: [Invokes /nim-setup skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand NIM platform requirements. - -Let me verify the prerequisites for NIM setup. - -Step 1 of 7: Checking GPU Operator and NFD Operator... - -GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) -NFD Operator: nfd.v4.19.0 (Succeeded) - -Both operators are installed and healthy. - -Step 2 of 7: I need your NGC credentials. - -To set up NIM, I need your NVIDIA NGC API key. -You can generate one at: https://ngc.nvidia.com/setup/api-key - -Please provide: -1. Your NGC API key -2. The target namespace for NIM resources - -User: "Here's my key: [user provides NGC API key], namespace: ai-models" - -Agent: Step 3 of 7: Creating NGC image pull secret... - -I'll create a docker-registry secret for pulling NIM images from nvcr.io: - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-image-pull-secret - namespace: ai-models - type: kubernetes.io/dockerconfigjson - data: - .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] - -Should I create this image pull secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - Image pull secret created successfully. - -Step 4 of 7: Creating NGC API key secret... - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-api-key - namespace: ai-models - type: Opaque - stringData: - NGC_API_KEY: "[REDACTED]" - -Should I create this API key secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - API key secret created successfully. - -Step 5 of 7: Creating NIM Account CR... - - apiVersion: nim.opendatahub.io/v1 - kind: Account - metadata: - name: nim-account - namespace: ai-models - spec: - apiKeySecret: - name: ngc-api-key - imagePullSecret: - name: ngc-image-pull-secret - -Should I create this NIM Account CR in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates Account CR via resources_create_or_update] - Account CR created successfully. - -Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) - -User: "no" - -Agent: Step 7 of 7: Validating NIM platform readiness... - -NIM Platform Setup Complete - -Account CR: `nim-account` (namespace: ai-models) - Status: Ready - -NGC Credentials: - Image Pull Secret: ngc-image-pull-secret [created] - API Key Secret: ngc-api-key [created] - -NIM ServingRuntimes Available: - - nim-llama-3-1-8b-instruct - - nim-llama-3-1-70b-instruct - - nim-mistral-7b-instruct - -Next Steps: - To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" -``` diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/references/known-model-profiles.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/references/known-model-profiles.md deleted file mode 100644 index 40148981..00000000 --- a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/references/known-model-profiles.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Known Model Hardware Profiles -category: references -tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] -semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] -use_cases: [model-deploy, debug-inference, ai-observability] -last_updated: 2026-03-10 ---- - -# Known Model Hardware Profiles - -Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). - -**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. - -## Model Source Conventions - -Each model lists a recommended `storageUri` with its authentication requirements: -- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** -- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). -- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. - -When the user does not specify a model source, use the `hf://` URI listed in the profile below. - -## Llama 3.x (Meta) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | -| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | -| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | -| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | - -- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) -- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly - -## Granite 3.x (IBM/Red Hat) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | -| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | - -- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) -- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser granite --chat-template` -- Red Hat-supported model family on RHOAI - -## Mixtral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | -| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) -- Mixture-of-Experts architecture: only ~13B/45B parameters active per token - -## Mistral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | -| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) - -## When a Model Is Not Listed - -If the requested model is not in this file, the agent MUST use the live doc lookup protocol: - -1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure -2. Fetch hardware specs from the appropriate source -3. Report findings to the user before proceeding with deployment - -Common cases requiring live lookup: -- Newly released models (after this file's last update) -- Domain-specific fine-tuned models -- Models with custom quantization -- NIM-specific optimized profiles diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/references/supported-runtimes.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/references/supported-runtimes.md deleted file mode 100644 index 9d018651..00000000 --- a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/references/supported-runtimes.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: Supported Serving Runtimes -category: references -tags: [runtimes, vllm, nim, caikit, tgis, serving] -semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] -use_cases: [model-deploy, serving-runtime-config, nim-setup] -last_updated: 2026-02-26 ---- - -# Supported Serving Runtimes - -This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. - -## Runtime Comparison - -| Runtime | API Type | Model Format | GPU Support | License | Setup Required | -|---------|----------|-------------|-------------|---------|----------------| -| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | -| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | -| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | -| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | - -## vLLM - -**Default runtime for most open-source models.** - -- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) -- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized -- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) -- **Model source**: S3-compatible storage, OCI registry, PVC, URI -- **Key features**: - - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) - - PagedAttention for efficient memory management - - Tensor parallelism for multi-GPU inference - - Tool/function calling support (`--tool-call-parser`) - - Continuous batching for high throughput -- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name - -## NVIDIA NIM - -**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** - -- **API**: OpenAI-compatible REST -- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog -- **GPU support**: NVIDIA only (requires specific GPU models per model profile) -- **Model source**: NGC model catalog (pulled automatically via NGC credentials) -- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) -- **Key features**: - - TensorRT-LLM optimization for lower latency - - Pre-compiled model engines (no compilation on first load) - - Optimized and generic profiles per GPU type - - Automatic model download from NGC -- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. -- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM -- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. -- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state - -## Caikit+TGIS - -**Red Hat's Caikit format with Text Generation Inference Server.** - -- **API**: gRPC (not REST) -- **Model formats**: Caikit format (requires conversion from HuggingFace) -- **GPU support**: NVIDIA -- **Model source**: S3-compatible storage -- **Key features**: - - Red Hat-supported runtime - - gRPC API for streaming inference - - Integrated with RHOAI model serving platform -- **When to choose**: When the model is already in Caikit format or when gRPC API is required. -- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name - -## Custom Runtimes - -**User-provided ServingRuntime CRs for unsupported frameworks.** - -- **API**: Defined by the custom runtime -- **Model formats**: Defined by the custom runtime -- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. -- **How to create**: Use `/serving-runtime-config` skill -- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility - -## Runtime Selection Decision Tree - -``` -Is the user's preferred runtime explicitly stated? -├── Yes → Use that runtime -└── No → Continue - -Is the model available in the NGC NIM catalog? -├── Yes → Suggest NIM (with vLLM as fallback) -│ Note: Requires /nim-setup and NVIDIA GPUs -└── No → Continue - -Is the model in Caikit format? -├── Yes → Caikit+TGIS -└── No → Continue - -Is the model a standard open-source LLM (HuggingFace-compatible)? -├── Yes → vLLM (default) -└── No → Custom runtime via /serving-runtime-config -``` diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/docs/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/model-deploy/SKILL.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/model-deploy/SKILL.md deleted file mode 100644 index 26a4fcb5..00000000 --- a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/model-deploy/SKILL.md +++ /dev/null @@ -1,382 +0,0 @@ ---- -name: model-deploy -description: | - Deploy AI/ML models on OpenShift AI using KServe with vLLM, NVIDIA NIM, or Caikit+TGIS runtimes. - - Use when: - - "Deploy Llama 3 on my cluster" - - "Set up a vLLM inference endpoint" - - "Deploy a model with NIM" - - "Create an InferenceService for Granite" - - "I need to serve a model on OpenShift AI" - - Handles runtime selection, GPU validation, InferenceService CR creation, and rollout monitoring. - - NOT for NIM platform setup (use /nim-setup first). - NOT for custom runtime creation (use /serving-runtime-config). -model: inherit -color: green ---- - -# /model-deploy Skill - -Deploy AI/ML models on Red Hat OpenShift AI using KServe. Supports vLLM, NVIDIA NIM, and Caikit+TGIS serving runtimes. Handles runtime selection, hardware profile lookup (with live doc fallback), GPU pre-flight checks, InferenceService CR creation, rollout monitoring, and post-deployment validation. - -## Prerequisites - -**Required MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) - -**Required MCP Tools** (from rhoai): -- `deploy_model` - Create InferenceService with high-level parameters (no YAML construction needed) -- `list_inference_services` - List deployed models with structured status data -- `get_inference_service` - Get detailed model deployment status (conditions, endpoint, ready state) -- `get_model_endpoint` - Get inference endpoint URL directly -- `list_serving_runtimes` - List available runtimes including platform templates with supported model formats -- `list_data_science_projects` - Discover RHOAI projects for namespace validation -- `list_data_connections` - Verify model storage access (S3 data connections) - -**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools** (from openshift): -- `resources_get` (from openshift) - Check NIM Account CR, LimitRange, GPU node taints -- `resources_list` (from openshift) - Check Knative availability, GPU nodes, existing deployments -- `pods_list` (from openshift) - Check predictor pod status during rollout -- `pods_log` (from openshift) - Retrieve pod logs for debugging -- `events_list` (from openshift) - Check events for errors - -**Optional MCP Server**: `ai-observability` ([AI Observability MCP](https://github.com/rh-ai-quickstart/ai-observability-summarizer)) - -**Optional MCP Tools** (from ai-observability): -- `get_gpu_info` - Pre-flight GPU inventory check -- `get_deployment_info` - Post-deployment validation -- `analyze_vllm` - Verify metrics are flowing after deployment - -**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, KServe, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). - -**Additional cluster requirements**: -- For NIM runtime: NIM platform set up via `/nim-setup` -- For vLLM/NIM: NVIDIA GPU nodes available in the cluster - -## When to Use This Skill - -**Use this skill when you need to:** -- Deploy an AI/ML model on OpenShift AI (KServe InferenceService) -- Set up vLLM, NIM, or Caikit+TGIS inference endpoints -- Look up hardware profiles and GPU requirements for a model -- Perform pre-flight validation before model deployment (GPU availability, namespace readiness, LimitRange conflicts) - -**Do NOT use this skill when:** -- You need to set up the NIM platform first (use `/nim-setup`) -- You need to create or customize a ServingRuntime (use `/serving-runtime-config`) -- You need to troubleshoot a failed or slow deployment (use `/debug-inference`) -- You need to analyze model performance or GPU metrics (use `/ai-observability`) - -## Workflow - -### Step 1: Gather Deployment Information - -Collect the following from the user. Use defaults where sensible, but always confirm. - -**Ask the user for:** -- **Model name**: Which model to deploy (e.g., "Llama 3.1 8B", "Granite 3.1 8B") -- **Runtime preference**: vLLM (default), NIM, or Caikit+TGIS (auto-detect if not specified) -- **Namespace**: Target namespace (must have model serving enabled) -- **Model source**: Where the model weights are stored (S3, OCI registry, PVC, or NGC for NIM) -- **Deployment mode**: Serverless (Knative, default) or RawDeployment - -**Present configuration table for review:** - -| Setting | Value | Source | -|---------|-------|--------| -| Model | [model-name] | user input | -| Runtime | [to be determined in Step 2] | auto-detect / user input | -| Namespace | [namespace] | user input | -| Model Source | [source-uri] | user input | -| Deployment Mode | [Serverless/RawDeployment] | user input / default: Serverless | - -Proceed automatically without waiting for user input. - -### Step 1.5: Pre-flight Environment Validation - -**CRITICAL**: Run these checks BEFORE deploying to avoid repeated deployment failures. - -Read [model-deploy-preflight-checklist.md](references/model-deploy-preflight-checklist.md) for the full pre-flight protocol. The checklist validates: -- Namespace is an RHOAI Data Science Project -- Model storage access (S3 data connections) -- Deployment mode support (Knative availability) -- Namespace resource constraints (LimitRange conflicts with KServe sidecars) -- GPU node taints (auto-generate tolerations) -- Existing deployments (reference configuration) -- Model source accessibility (OCI registry entitlements) - -**Present pre-flight results** in a summary table and note any adjustments made. Proceed automatically without waiting for user input. - -### Step 2: Determine Runtime - -**Document Consultation** (read before selecting runtime): -1. **Action**: Read [supported-runtimes.md](../../docs/references/supported-runtimes.md) using the Read tool to understand runtime capabilities and selection criteria -2. **Output to user**: "I consulted [supported-runtimes.md](../../docs/references/supported-runtimes.md) to understand runtime capabilities." - -**Runtime Selection Logic:** - -- User explicitly requested a runtime -> Use that runtime -- Model available in NGC NIM catalog -> Suggest NIM (with vLLM as fallback) -- Model is a standard open-source LLM (HuggingFace-compatible) -> Default to vLLM -- Model is in Caikit format -> Caikit+TGIS -- None of the above -> Suggest custom runtime via `/serving-runtime-config` - -**Present recommendation** with rationale. Proceed automatically without waiting for user input. - -### Step 3: Look Up Model Hardware Profile - -**Document Consultation** (read before determining hardware requirements): -1. **Action**: Read [known-model-profiles.md](../../docs/references/known-model-profiles.md) using the Read tool to find hardware profile for the requested model -2. **Output to user**: "I consulted [known-model-profiles.md](../../docs/references/known-model-profiles.md) to find hardware requirements for [model-name]." - -**If model IS in known-model-profiles.md:** -- Extract: GPU count, GPU type, VRAM, key vLLM args -- Present to user - -**If model is NOT in known-model-profiles.md -> Trigger live doc lookup:** -1. **Action**: Read [live-doc-lookup.md](../references/live-doc-lookup.md) using the Read tool for the lookup protocol -2. **Output to user**: "Model [model-name] is not in my cached profiles. I'll look up its hardware requirements." -3. Use **WebFetch** tool to retrieve specs from: - - For NIM models: `https://build.nvidia.com/models` or `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - - For other models: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` -4. Extract: GPU requirements, model-specific args, known issues -5. **Output to user**: "I looked up [model-name] on [source] to confirm its hardware requirements: [summary]" - -**Present hardware requirements** in a table (GPUs, VRAM, Key Args). - -### Step 4: Pre-flight GPU Check (Optional) - -**Condition**: Only if `ai-observability` MCP server is available. - -**MCP Tool**: `get_gpu_info` (from ai-observability) - -Compare available GPUs against model requirements from Step 3: -- If sufficient GPUs available -> Report match and proceed -- If insufficient -> Warn user with options: smaller model, quantized version, different cluster, or proceed at user's risk - -**If ai-observability not available**: Skip with note: "GPU pre-flight check skipped (ai-observability MCP not configured)." - -### Step 5: Verify NIM Platform (NIM Runtime Only) - -**Condition**: Only when the selected runtime is NIM. - -**MCP Tool**: `resources_get` (from openshift) - -**Parameters**: -- `apiVersion`: `"nim.opendatahub.io/v1alpha1"` - REQUIRED -- `kind`: `"Account"` - REQUIRED -- `namespace`: target namespace - REQUIRED -- `name`: `"nim-account"` - REQUIRED - -**If Account CR not found or not ready:** -Offer options: (1) Run `/nim-setup` now, (2) Switch to vLLM, (3) Abort. Proceed automatically without waiting for user input. - -### Step 6: Select ServingRuntime and Prepare Deployment Parameters - -**Verify available ServingRuntimes:** - -**MCP Tool**: `list_serving_runtimes` (from rhoai) - -**Parameters**: -- `namespace`: target namespace - REQUIRED -- `include_templates`: `true` - REQUIRED (shows both existing runtimes and platform templates) - -The response shows existing runtimes and available templates with their supported model formats and `requires_instantiation` flag. - -If the needed runtime shows `requires_instantiation: true`, it must first be instantiated via `/serving-runtime-config` or the rhoai `create_serving_runtime` tool. - -Use the runtime list to select the correct `runtime` name for the deployment. - -**Prepare deployment parameters** from Steps 1-3 and environment data from Step 1.5: - -| Parameter | Value | Source | -|-----------|-------|--------| -| `name` | [model-deployment-name] | user input (DNS-compatible) | -| `namespace` | [namespace] | user input | -| `runtime` | [serving-runtime-name] | selected from `list_serving_runtimes` | -| `model_format` | [vLLM/pytorch/onnx/caikit/etc.] | runtime selection | -| `storage_uri` | [model-source-uri] | user input (prefer `hf://` for public models) | -| `gpu_count` | [gpu-count] | from hardware profile (Step 3) | -| `cpu_request` | [cpu] | from profile, adjusted for LimitRange | -| `memory_request` | [memory] | from profile, adjusted for LimitRange | -| `min_replicas` | [1] | default 1 (0 for scale-to-zero) | -| `max_replicas` | [1] | default 1 | - -**Model sizing guide** for LLMs: -- 7B models: 1x 24GB GPU (e.g., A10G) or 1x 16GB GPU with quantization -- 13B models: 2x 24GB GPUs -- 70B models: 4+ 80GB GPUs (A100/H100) or quantized on fewer GPUs - -**Scale-to-zero note**: Setting `min_replicas=0` saves resources but introduces cold start latency (30s-2min for model loading). - -### Step 7: User Review and Confirmation - -**Display the deployment parameters table** and a configuration summary to the user. - -**Ask**: "Proceed with deploying this model? (yes/no/modify)" - -Proceed automatically without waiting for user input. - -- If **yes** -> Proceed to Step 8 -- If **no** -> Abort -- If **modify** -> Ask what to change, update parameters, return to this step - -### Step 8: Deploy Model - -**MCP Tool**: `deploy_model` (from rhoai) - -**Parameters**: -- `name`: deployment name (DNS-compatible) - REQUIRED -- `namespace`: target namespace - REQUIRED -- `runtime`: serving runtime name from Step 6 - REQUIRED -- `model_format`: model format string (e.g., `"vLLM"`, `"pytorch"`, `"onnx"`) - REQUIRED -- `storage_uri`: model location (e.g., `"hf://ibm-granite/granite-3.1-2b-instruct"`, `"s3://bucket/path"`, `"pvc://pvc-name/path"`) - REQUIRED -- `display_name`: human-readable display name - OPTIONAL -- `min_replicas`: minimum replicas (default: 1, 0 for scale-to-zero) - OPTIONAL -- `max_replicas`: maximum replicas (default: 1) - OPTIONAL -- `cpu_request`: CPU request per replica (default: `"1"`) - OPTIONAL -- `cpu_limit`: CPU limit per replica (default: `"2"`) - OPTIONAL -- `memory_request`: memory request per replica (default: `"4Gi"`) - OPTIONAL -- `memory_limit`: memory limit per replica (default: `"8Gi"`) - OPTIONAL -- `gpu_count`: number of GPUs per replica (default: 0) - OPTIONAL - -**Note**: For NIM deployments, ensure the NGC API key secret is referenced. If `deploy_model` does not support NIM-specific env vars, fall back to `resources_create_or_update` (from openshift) with a NIM InferenceService YAML that includes `spec.predictor.env` referencing the `ngc-api-key` secretKeyRef. - -**Error Handling**: -- If namespace not found -> Report error, suggest creating namespace or using `/ds-project-setup` -- If ServingRuntime not found -> Report error, verify runtime name, suggest `/serving-runtime-config` -- If quota exceeded -> Report error, suggest reducing resource requests -- If RBAC error -> Report insufficient permissions - -### Step 9: Monitor Rollout - -Poll InferenceService status until ready or timeout (10 minutes). - -**MCP Tool**: `get_inference_service` (from rhoai) -- `name`: deployment name, `namespace`: target namespace, `verbosity`: `"full"` - -Check the Ready condition and status. Repeat every 15-30 seconds until Ready=True or timeout. - -**Check predictor pod status:** - -**MCP Tool**: `pods_list` (from openshift) -- `namespace`: target namespace, `labelSelector`: `"serving.kserve.io/inferenceservice=[model-name]"` - -Show deployment progress tracking: Pod Scheduled, Image Pulled, Container Started, Model Loaded, Ready. Include pod name, status, and restart count. - -**On failure:** Check pod logs (`pods_log`) and events (`events_list`) for diagnostics. Present options: (1) View full pod logs, (2) Check namespace events, (3) Invoke `/debug-inference`, (4) Delete and retry, (5) Continue waiting. Proceed automatically without waiting for user input. - -### Step 10: Deployment Complete - -**Get endpoint URL:** - -**MCP Tool**: `get_model_endpoint` (from rhoai) -- `name`: deployment name, `namespace`: target namespace - -**Report success** showing: model name, runtime, namespace, GPUs, inference endpoint URL, API type (OpenAI-compatible REST), and next steps (`/ai-observability`, `/model-monitor`, `/guardrails-config`). - -**Provide test commands** based on runtime: -- **vLLM (OpenAI-compatible)**: `curl -X POST [endpoint]/v1/completions -H "Content-Type: application/json" -d '{"model":"[model-name]","prompt":"Hello","max_tokens":100}'` -- **KServe v2**: `curl -X POST [endpoint]/v2/models/[model-name]/infer -H "Content-Type: application/json" -d '{"inputs":[...]}'` - -**Post-deployment validation** (if ai-observability MCP available): -- `get_deployment_info` to confirm model appears in monitoring -- `analyze_vllm` with a short time range to verify initial metrics are flowing -- Report findings to user - -## Common Issues - -For common issues (GPU scheduling, OOMKilled, image pull errors, RBAC), see [common-issues.md](../references/common-issues.md). - -### Issue 1: InferenceService Stuck in "Unknown" - -**Error**: InferenceService `status.conditions` shows "Unknown" state - -**Cause**: ServingRuntime not found in the namespace, or model serving platform not enabled. - -**Solution:** -1. Verify ServingRuntime exists: `resources_list` for `servingruntimes` in namespace -2. Ensure model serving is enabled: namespace has label `opendatahub.io/dashboard: "true"` -3. Check the runtime name in the InferenceService matches an available ServingRuntime -4. If no matching runtime, use `/serving-runtime-config` to create one - -### Issue 2: Model Download Timeout - -**Error**: Pod starts but times out while downloading model weights from S3 or OCI registry - -**Cause**: Large model size combined with slow network connection to storage. - -**Solution:** -1. Add `serving.knative.dev/progress-deadline` annotation with a longer timeout (e.g., `"1800s"`) -2. Verify S3/storage credentials are valid -3. Consider using a PVC with pre-downloaded model weights instead -4. Check network connectivity between the pod and storage endpoint - -### Issue 3: LimitRange Conflicts with KServe Sidecars - -**Error**: Pod rejected with `minimum cpu usage per Container is 50m, but request is 10m` or `minimum memory usage per Container is 64Mi, but request is 15Mi` - -**Cause**: The namespace has a LimitRange with minimum resource constraints that exceed the hardcoded resource requests of KServe-injected sidecar containers (oauth-proxy, queue-proxy, or modelcar containers request 10m CPU / 15Mi memory). These sidecar resource values cannot be controlled through the InferenceService spec. - -**Solution:** -1. Check LimitRange: `resources_list` for `LimitRange` in the namespace -2. If LimitRange minimum CPU > 10m or minimum memory > 15Mi, the LimitRange must be adjusted -3. Options: (a) Lower LimitRange minimums to accommodate sidecars (min CPU ≤ 10m, min memory ≤ 15Mi), (b) Remove the LimitRange entirely, (c) Deploy in a different namespace without restrictive LimitRanges -4. **Prevention**: Step 1.5 pre-flight validation now checks for this conflict before deployment - -### Issue 4: GPU Node Taints Prevent Scheduling - -**Error**: Pod stuck in Pending with events showing `node(s) had untolerated taint {ai-app: true}` or similar custom taint messages, while also showing `Insufficient nvidia.com/gpu` on remaining nodes - -**Cause**: GPU nodes are tainted with custom taints (e.g., `ai-app=true:NoSchedule`) to reserve them for AI workloads. The InferenceService predictor pod does not have matching tolerations, so it cannot be scheduled on GPU nodes. - -**Solution:** -1. Identify GPU node taints: `resources_get` for GPU nodes, check `.spec.taints` -2. Add matching tolerations to the InferenceService predictor spec: - ```yaml - spec: - predictor: - tolerations: - - key: "ai-app" - operator: "Equal" - value: "true" - effect: "NoSchedule" - ``` -3. **Prevention**: Step 1.5 pre-flight validation now auto-discovers GPU node taints and generates tolerations - -## Dependencies - -### MCP Tools -See [Prerequisites](#prerequisites) for the complete list of required and optional MCP tools. - -### Related Skills -- `/nim-setup` - Prerequisite for NIM runtime deployments -- `/debug-inference` - Troubleshoot InferenceService failures -- `/ai-observability` - Analyze deployed model performance -- `/serving-runtime-config` - Create custom ServingRuntime CRs -- `/ds-project-setup` - Create a namespace with model serving enabled - -### Reference Documentation -- [known-model-profiles.md](../../docs/references/known-model-profiles.md) - Hardware profiles for common models -- [supported-runtimes.md](../../docs/references/supported-runtimes.md) - Runtime capabilities and selection criteria -- [live-doc-lookup.md](../references/live-doc-lookup.md) - Protocol for fetching specs for unknown models - -## Critical: Human-in-the-Loop Requirements - -See [skill-conventions.md](../references/skill-conventions.md) for general HITL and security conventions. - -**Skill-specific checkpoints:** -- After gathering settings (Step 1): confirm configuration table -- After pre-flight validation (Step 1.5): confirm if significant adjustments were needed (deployment mode, tolerations, resource changes) -- After runtime selection (Step 2): confirm runtime choice -- Before calling deploy_model (Step 7): review and confirm deployment parameters -- On deployment failure (Step 9): present diagnostic options, wait for user decision -- **NEVER** auto-delete failed deployments or auto-select runtimes without confirmation - -## Example Usage - -See [model-deploy examples](../../docs/examples/model-deploy.md) for complete deployment walkthroughs (vLLM and NIM). diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/model-deploy/references/model-deploy-preflight-checklist.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/model-deploy/references/model-deploy-preflight-checklist.md deleted file mode 100644 index 02dbd83d..00000000 --- a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/model-deploy/references/model-deploy-preflight-checklist.md +++ /dev/null @@ -1,64 +0,0 @@ -# Model Deploy Pre-flight Checklist - -Run these checks BEFORE deploying to avoid repeated deployment failures. Each check uses MCP tools to validate the target environment. - -## 0. Validate Namespace is an RHOAI Data Science Project - -**MCP Tool**: `list_data_science_projects` (from rhoai) - -Verify the target namespace appears in the project list. If not found, warn: "Namespace `[namespace]` is not a Data Science Project. Model serving may not be configured. Create one via the OpenShift AI dashboard or proceed at your own risk." - -## 0b. Check Model Storage Access (S3 Sources) - -**MCP Tool**: `list_data_connections` (from rhoai) -- `namespace`: target namespace - -If model source is S3-based, verify a matching data connection exists. If not found, inform user: "No S3 data connection found in namespace. Create one via the OpenShift AI dashboard or provide model source as PVC or HuggingFace URI." - -## 1. Check Deployment Mode Support - -**MCP Tool**: `resources_list` (from openshift) -- `apiVersion`: `"serving.knative.dev/v1"`, `kind`: `"Service"`, `namespace`: target namespace - -If Knative Services are not available (CRD not found or error) -> auto-select **RawDeployment** mode and inform the user: "Knative Services are not available on this cluster. Switching to RawDeployment mode." - -## 2. Check Namespace Resource Constraints - -**MCP Tool**: `resources_list` (from openshift) -- `apiVersion`: `"v1"`, `kind`: `"LimitRange"`, `namespace`: target namespace - -If a LimitRange exists: -- **Action**: `resources_get` for each LimitRange to extract min/max/default values -- Validate that planned resource requests fit within max limits -- **Warning**: If LimitRange minimum CPU > 10m or minimum memory > 15Mi, KServe-injected sidecar containers (with hardcoded 10m CPU / 15Mi memory requests) will fail to schedule. Warn the user: "LimitRange minimums conflict with KServe sidecar containers. The LimitRange must be adjusted or removed before deployment can succeed." -- Adjust planned resource requests/limits to fit within constraints -- Present adjusted values to user - -## 3. Discover GPU Node Taints - -**MCP Tool**: `resources_list` (from openshift) -- `apiVersion`: `"v1"`, `kind`: `"Node"`, `labelSelector`: `"nvidia.com/gpu.present=true"` - -For each GPU node, extract taints. If custom taints exist (beyond standard Kubernetes taints like `node-role.kubernetes.io/*`): -- Auto-generate matching tolerations for the InferenceService -- Present discovered taints and proposed tolerations to user for confirmation -- Common example: `ai-app=true:NoSchedule` requires toleration `{key: "ai-app", operator: "Equal", value: "true", effect: "NoSchedule"}` - -## 4. Check Existing Deployments in Namespace - -**MCP Tool**: `list_inference_services` (from rhoai) -- `namespace`: target namespace -- `verbosity`: `"standard"` - -If similar InferenceServices exist, inspect their `storageUri`, runtime, and tolerations as a reference for proven-working configuration in this environment. - -## 5. Validate Model Source Accessibility - -If using `oci://` source: -- Check namespace service account `imagePullSecrets` can access the registry -- For `registry.redhat.io/rhelai1/*` images: these require RHEL AI subscription entitlements -- verify pull secret has access or recommend switching to `hf://` (HuggingFace) source -- **Default preference**: For public open-source models, prefer `hf://` sources (e.g., `hf://ibm-granite/granite-3.1-2b-instruct`) as they require no authentication - -## Summary - -Present pre-flight results in a summary table and note any adjustments made. **WAIT for user confirmation if significant changes were needed** (e.g., deployment mode switch, resource adjustments, tolerations added). diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/common-issues.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/with_skills/rh-ai-engineer__model-deploy/environment/skills/references/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/common-issues.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/examples/model-deploy.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/examples/model-deploy.md deleted file mode 100644 index dcd79e44..00000000 --- a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/examples/model-deploy.md +++ /dev/null @@ -1,160 +0,0 @@ -# /model-deploy Examples - -Example interactions demonstrating the model-deploy skill workflow. - -## Example 1: Deploy Llama 3.1 8B with vLLM - -``` -User: "Deploy Llama 3.1 8B on my cluster" - -Agent: [Invokes /model-deploy skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand runtime capabilities. - -Step 1 of 10: Gathering deployment information. - -| Setting | Value | Source | -|---------|-------|--------| -| Model | Llama 3.1 8B Instruct | user input | -| Runtime | TBD | auto-detect | -| Namespace | ? | needs input | -| Model Source | ? | needs input | -| Deployment Mode | Serverless | default | - -What namespace should I deploy to, and where are the model weights stored? -(e.g., S3 bucket, OCI registry, PVC) - -User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" - -Agent: Step 2 of 10: Determining runtime. - -I consulted supported-runtimes.md to understand runtime capabilities. - -Based on Llama 3.1 8B (open-source HuggingFace model): - -Recommended runtime: vLLM -Reason: Default runtime for open-source LLMs. Broad compatibility, - OpenAI-compatible API, no additional setup required. - -Proceed with vLLM? (yes / no / suggest alternative) - -User: "yes" - -Agent: Step 3 of 10: Looking up hardware profile. - -I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. - -| Requirement | Value | -|-------------|-------| -| GPUs | 1x A10G/L4/A100/H100 | -| VRAM | 16GB | -| Key Args | --max-model-len=4096 --tool-call-parser hermes | - -Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). -Ensure your cluster has at least 1 GPU with 16GB VRAM. - -Step 5 of 10: Skipped (NIM platform check not needed for vLLM). - -Step 6-7 of 10: Generating InferenceService YAML. - -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - name: llama-3-1-8b - namespace: ai-models - annotations: - serving.kserve.io/deploymentMode: Serverless -spec: - predictor: - model: - modelFormat: - name: vLLM - runtime: vllm-runtime - storageUri: s3://my-bucket/llama-3.1-8b - resources: - limits: - nvidia.com/gpu: "1" - requests: - cpu: "4" - memory: "16Gi" - args: - - --max-model-len=4096 - - --tool-call-parser=hermes - -| Setting | Value | -|---------|-------| -| Model | llama-3-1-8b | -| Runtime | vllm-runtime | -| GPUs | 1 | -| Namespace | ai-models | -| Mode | Serverless | - -Proceed with creating this InferenceService? (yes/no/modify) - -User: "yes" - -Agent: Step 8 of 10: Creating InferenceService... - [Creates via resources_create_or_update] - - InferenceService created. Monitoring rollout... - -Step 9 of 10: Monitoring rollout. - -| Stage | Status | -|-------|--------| -| Pod Scheduled | Complete | -| Image Pulled | Complete | -| Container Started | Complete | -| Model Loaded | Complete | -| Ready | Complete | - -Step 10 of 10: Deployment complete! - -Model Deployment Successful - -Model: llama-3-1-8b -Runtime: vLLM -Namespace: ai-models -GPUs: 1 - -Inference Endpoint: - URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com - API: OpenAI-compatible REST - -Quick Test: - curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' - -Next Steps: - Monitor performance: "/ai-observability llama-3-1-8b" -``` - -## Example 2: Deploy a Model with NIM (NIM Not Set Up) - -``` -User: "Deploy Llama 3.1 70B with NIM" - -Agent: [Invokes /model-deploy skill] - ... - Step 2: User requested NIM runtime. - - Step 5: Checking NIM platform... - -NIM platform is not set up in namespace ai-models. - -The NIM runtime requires a one-time platform setup. Would you like to: -1. Run /nim-setup now to configure the NIM platform -2. Switch to vLLM runtime instead (no setup required) -3. Abort deployment - -User: "1" - -Agent: [Invokes /nim-setup skill — walks through NGC credential setup] - ... - NIM Platform Setup Complete. - - Resuming model deployment... - [Continues from Step 6 with NIM runtime] -``` diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/examples/nim-setup.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/examples/nim-setup.md deleted file mode 100644 index ccfc1a5d..00000000 --- a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/examples/nim-setup.md +++ /dev/null @@ -1,115 +0,0 @@ -# /nim-setup Examples - -Example interactions demonstrating the nim-setup skill workflow. - -## Example 1: First-Time NIM Setup - -``` -User: "Set up NIM on my cluster" - -Agent: [Invokes /nim-setup skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand NIM platform requirements. - -Let me verify the prerequisites for NIM setup. - -Step 1 of 7: Checking GPU Operator and NFD Operator... - -GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) -NFD Operator: nfd.v4.19.0 (Succeeded) - -Both operators are installed and healthy. - -Step 2 of 7: I need your NGC credentials. - -To set up NIM, I need your NVIDIA NGC API key. -You can generate one at: https://ngc.nvidia.com/setup/api-key - -Please provide: -1. Your NGC API key -2. The target namespace for NIM resources - -User: "Here's my key: [user provides NGC API key], namespace: ai-models" - -Agent: Step 3 of 7: Creating NGC image pull secret... - -I'll create a docker-registry secret for pulling NIM images from nvcr.io: - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-image-pull-secret - namespace: ai-models - type: kubernetes.io/dockerconfigjson - data: - .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] - -Should I create this image pull secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - Image pull secret created successfully. - -Step 4 of 7: Creating NGC API key secret... - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-api-key - namespace: ai-models - type: Opaque - stringData: - NGC_API_KEY: "[REDACTED]" - -Should I create this API key secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - API key secret created successfully. - -Step 5 of 7: Creating NIM Account CR... - - apiVersion: nim.opendatahub.io/v1 - kind: Account - metadata: - name: nim-account - namespace: ai-models - spec: - apiKeySecret: - name: ngc-api-key - imagePullSecret: - name: ngc-image-pull-secret - -Should I create this NIM Account CR in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates Account CR via resources_create_or_update] - Account CR created successfully. - -Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) - -User: "no" - -Agent: Step 7 of 7: Validating NIM platform readiness... - -NIM Platform Setup Complete - -Account CR: `nim-account` (namespace: ai-models) - Status: Ready - -NGC Credentials: - Image Pull Secret: ngc-image-pull-secret [created] - API Key Secret: ngc-api-key [created] - -NIM ServingRuntimes Available: - - nim-llama-3-1-8b-instruct - - nim-llama-3-1-70b-instruct - - nim-mistral-7b-instruct - -Next Steps: - To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" -``` diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/references/known-model-profiles.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/references/known-model-profiles.md deleted file mode 100644 index 40148981..00000000 --- a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/references/known-model-profiles.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Known Model Hardware Profiles -category: references -tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] -semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] -use_cases: [model-deploy, debug-inference, ai-observability] -last_updated: 2026-03-10 ---- - -# Known Model Hardware Profiles - -Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). - -**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. - -## Model Source Conventions - -Each model lists a recommended `storageUri` with its authentication requirements: -- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** -- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). -- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. - -When the user does not specify a model source, use the `hf://` URI listed in the profile below. - -## Llama 3.x (Meta) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | -| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | -| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | -| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | - -- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) -- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly - -## Granite 3.x (IBM/Red Hat) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | -| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | - -- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) -- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser granite --chat-template` -- Red Hat-supported model family on RHOAI - -## Mixtral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | -| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) -- Mixture-of-Experts architecture: only ~13B/45B parameters active per token - -## Mistral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | -| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) - -## When a Model Is Not Listed - -If the requested model is not in this file, the agent MUST use the live doc lookup protocol: - -1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure -2. Fetch hardware specs from the appropriate source -3. Report findings to the user before proceeding with deployment - -Common cases requiring live lookup: -- Newly released models (after this file's last update) -- Domain-specific fine-tuned models -- Models with custom quantization -- NIM-specific optimized profiles diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/references/supported-runtimes.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/references/supported-runtimes.md deleted file mode 100644 index 9d018651..00000000 --- a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/references/supported-runtimes.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: Supported Serving Runtimes -category: references -tags: [runtimes, vllm, nim, caikit, tgis, serving] -semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] -use_cases: [model-deploy, serving-runtime-config, nim-setup] -last_updated: 2026-02-26 ---- - -# Supported Serving Runtimes - -This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. - -## Runtime Comparison - -| Runtime | API Type | Model Format | GPU Support | License | Setup Required | -|---------|----------|-------------|-------------|---------|----------------| -| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | -| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | -| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | -| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | - -## vLLM - -**Default runtime for most open-source models.** - -- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) -- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized -- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) -- **Model source**: S3-compatible storage, OCI registry, PVC, URI -- **Key features**: - - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) - - PagedAttention for efficient memory management - - Tensor parallelism for multi-GPU inference - - Tool/function calling support (`--tool-call-parser`) - - Continuous batching for high throughput -- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name - -## NVIDIA NIM - -**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** - -- **API**: OpenAI-compatible REST -- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog -- **GPU support**: NVIDIA only (requires specific GPU models per model profile) -- **Model source**: NGC model catalog (pulled automatically via NGC credentials) -- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) -- **Key features**: - - TensorRT-LLM optimization for lower latency - - Pre-compiled model engines (no compilation on first load) - - Optimized and generic profiles per GPU type - - Automatic model download from NGC -- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. -- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM -- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. -- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state - -## Caikit+TGIS - -**Red Hat's Caikit format with Text Generation Inference Server.** - -- **API**: gRPC (not REST) -- **Model formats**: Caikit format (requires conversion from HuggingFace) -- **GPU support**: NVIDIA -- **Model source**: S3-compatible storage -- **Key features**: - - Red Hat-supported runtime - - gRPC API for streaming inference - - Integrated with RHOAI model serving platform -- **When to choose**: When the model is already in Caikit format or when gRPC API is required. -- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name - -## Custom Runtimes - -**User-provided ServingRuntime CRs for unsupported frameworks.** - -- **API**: Defined by the custom runtime -- **Model formats**: Defined by the custom runtime -- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. -- **How to create**: Use `/serving-runtime-config` skill -- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility - -## Runtime Selection Decision Tree - -``` -Is the user's preferred runtime explicitly stated? -├── Yes → Use that runtime -└── No → Continue - -Is the model available in the NGC NIM catalog? -├── Yes → Suggest NIM (with vLLM as fallback) -│ Note: Requires /nim-setup and NVIDIA GPUs -└── No → Continue - -Is the model in Caikit format? -├── Yes → Caikit+TGIS -└── No → Continue - -Is the model a standard open-source LLM (HuggingFace-compatible)? -├── Yes → vLLM (default) -└── No → Custom runtime via /serving-runtime-config -``` diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/docs/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/nim-setup/SKILL.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/nim-setup/SKILL.md deleted file mode 100644 index 34df2121..00000000 --- a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/nim-setup/SKILL.md +++ /dev/null @@ -1,370 +0,0 @@ ---- -name: nim-setup -description: | - Configure NVIDIA NIM platform on OpenShift AI for optimized model inference. - - Use when: - - "Set up NIM on my cluster" - - "Configure NGC credentials for NIM" - - "I want to deploy a NIM model but haven't set up the platform" - - "Create the NIM Account CR" - - One-time prerequisite before deploying models with NVIDIA NIM runtime via /model-deploy. - - NOT for deploying models (use /model-deploy instead). - NOT for vLLM or Caikit deployments (NIM-specific only). -model: inherit -color: blue ---- - -# /nim-setup Skill - -Configure the NVIDIA NIM platform on OpenShift AI. This is a one-time setup that creates NGC credentials and the NIM Account custom resource, enabling NIM-based model deployments via `/model-deploy`. - -## Prerequisites - -**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools** (from openshift): -- `resources_get` - Check operator installations and existing resources -- `resources_list` - List resources in a namespace -- `resources_create_or_update` - Create secrets, Account CR, ConfigMap -- `events_list` - Check events for errors during setup - -**Optional MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) - -**Optional MCP Tools** (from rhoai): -- `list_data_science_projects` - Validate namespace is an RHOAI Data Science Project -- `list_serving_runtimes` - Verify NIM ServingRuntimes after setup - -**Optional MCP Server**: `ai-observability` (for `get_gpu_info` to verify GPU availability) - -**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). - -**Required User Input**: -- NGC API key (from https://ngc.nvidia.com) -- Target namespace for NIM resources - -**Additional cluster requirements**: -- OpenShift cluster >= 4.14 -- NVIDIA GPU Operator installed -- Node Feature Discovery (NFD) Operator installed -- ServiceAccount with RBAC permissions to create Secrets, Accounts, and ConfigMaps - -## When to Use This Skill - -**Use this skill when you need to:** -- Set up NVIDIA NIM platform on OpenShift AI for the first time -- Create or refresh NGC credentials (image pull secret + API key secret) -- Create the NIM Account custom resource -- Verify GPU Operator and NFD Operator are installed and healthy - -**Do NOT use this skill when:** -- You want to deploy a model (use `/model-deploy` after NIM setup is complete) -- You want to deploy with vLLM or Caikit+TGIS (NIM-specific only, use `/model-deploy` directly) -- You need to create a custom ServingRuntime (use `/serving-runtime-config`) - -## Workflow - -### Step 0: Validate Target Namespace (Optional) - -If the `rhoai` MCP server is available, validate that the target namespace is an RHOAI Data Science Project: - -**MCP Tool**: `list_data_science_projects` (from rhoai) - -If the namespace is not in the project list, warn: "Namespace `[namespace]` is not a Data Science Project. NIM setup may not work correctly. Consider creating a Data Science Project first." - -If `rhoai` MCP is not available, skip this check and proceed. - -### Step 1: Verify GPU Operator and Node Feature Discovery - -**Document Consultation** (read before verifying operators): -1. **Action**: Read [supported-runtimes.md](../../docs/references/supported-runtimes.md) using the Read tool to understand NIM platform requirements -2. **Output to user**: "I consulted [supported-runtimes.md](../../docs/references/supported-runtimes.md) to understand NIM platform requirements." - -Check that the NVIDIA GPU Operator and NFD Operator are installed and healthy. - -**MCP Tool**: `resources_get` (from openshift) - -**Parameters**: -- `apiVersion`: `"operators.coreos.com/v1alpha1"` - REQUIRED -- `kind`: `"ClusterServiceVersion"` - REQUIRED -- `namespace`: `"nvidia-gpu-operator"` - REQUIRED (namespace where GPU Operator CSV is installed) -- `name`: the CSV name matching `"gpu-operator-certified"` prefix - -**Expected Output**: ClusterServiceVersion object with `status.phase: "Succeeded"` - -Repeat for NFD Operator: -- `namespace`: `"openshift-nfd"` -- `name`: the CSV name matching `"nfd"` prefix - -**Error Handling**: -- If GPU Operator CSV not found -> Report to user: "NVIDIA GPU Operator is not installed. Install it from OperatorHub before proceeding." -- If NFD Operator CSV not found -> Report to user: "Node Feature Discovery Operator is not installed. Install it from OperatorHub before proceeding." -- If `status.phase` != `"Succeeded"` -> Report current phase and suggest troubleshooting -- Offer to skip this check if user confirms operators are installed via another method - -### Step 2: Collect NGC Credentials from User - -Ask the user for their NGC API key. This key is used for two purposes: -1. Pulling NIM container images from `nvcr.io` (image pull secret) -2. Authenticating NIM API calls at runtime (API key secret) - -**Ask the user**: -``` -To set up NIM, I need your NVIDIA NGC API key. - -You can generate one at: https://ngc.nvidia.com/setup/api-key - -Please provide: -1. Your NGC API key -2. The target namespace for NIM resources (e.g., "my-ai-project") -``` - -Proceed automatically without waiting for user input. - -**SECURITY**: Store the key in memory only for the duration of this skill. Never echo or display the actual key value in output. - -### Step 3: Create NGC Image Pull Secret - -Generate and display the docker-registry Secret YAML for pulling NIM images from `nvcr.io`. - -**Show the user the Secret manifest** (with API key value redacted): - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: ngc-image-pull-secret - namespace: [namespace] -type: kubernetes.io/dockerconfigjson -data: - .dockerconfigjson: [base64-encoded docker config for nvcr.io] -``` - -Note: The `.dockerconfigjson` contains: -- Registry: `nvcr.io` -- Username: `$oauthtoken` -- Password: `[NGC API key - REDACTED in display]` - -**Ask**: "Should I create this image pull secret in namespace `[namespace]`? (yes/no)" - -Proceed automatically without waiting for user input. - -**MCP Tool**: `resources_create_or_update` (from openshift) - -**Parameters**: -- `manifest`: full Secret manifest as JSON string - REQUIRED - - The JSON must include apiVersion, kind, metadata (name, namespace), type, and data fields -- `namespace`: user-specified namespace - REQUIRED - - Example: `"my-ai-project"` - -**Expected Output**: Created Secret object with `metadata.uid` - -**Error Handling**: -- If secret already exists -> Ask user: "Secret `ngc-image-pull-secret` already exists. Should I update it? (yes/no)" -- If namespace not found -> Report error, suggest creating namespace first -- If RBAC error -> Report insufficient permissions - -### Step 4: Create NGC API Key Secret - -Generate and display the generic Secret YAML for the NGC API key used at runtime. - -**Show the user the Secret manifest** (with API key value redacted): - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: ngc-api-key - namespace: [namespace] -type: Opaque -stringData: - NGC_API_KEY: "[REDACTED]" -``` - -**Ask**: "Should I create this API key secret in namespace `[namespace]`? (yes/no)" - -Proceed automatically without waiting for user input. - -**MCP Tool**: `resources_create_or_update` (from openshift) - -**Parameters**: -- `manifest`: full Secret manifest as JSON string - REQUIRED -- `namespace`: user-specified namespace - REQUIRED - -**Expected Output**: Created Secret object with `metadata.uid` - -**Error Handling**: -- If secret already exists -> Ask user if they want to update it -- If RBAC error -> Report insufficient permissions - -### Step 5: Create NIM Account CR - -Generate and display the NIM Account custom resource that manages the NIM platform lifecycle. - -**Show the user the Account CR manifest:** - -```yaml -apiVersion: nim.opendatahub.io/v1 -kind: Account -metadata: - name: nim-account - namespace: [namespace] -spec: - apiKeySecret: - name: ngc-api-key - imagePullSecret: - name: ngc-image-pull-secret -``` - -**Ask**: "Should I create this NIM Account CR in namespace `[namespace]`? (yes/no)" - -Proceed automatically without waiting for user input. - -**MCP Tool**: `resources_create_or_update` (from openshift) - -**Parameters**: -- `manifest`: full Account CR manifest as JSON string - REQUIRED -- `namespace`: user-specified namespace - REQUIRED - -**Expected Output**: Created Account object with `metadata.uid` - -**Error Handling**: -- If Account CR already exists -> Report current status, ask if user wants to update -- If CRD not found (`nim.opendatahub.io/v1` Account) -> Report: "NIM CRD not available. Ensure Red Hat OpenShift AI operator is installed and includes NIM support." -- If RBAC error -> Report insufficient permissions - -### Step 6: (Optional) Configure NIM Model Catalog - -**Ask**: "Would you like to customize which NIM models appear in the catalog? (yes/no, default: no)" - -If user says **no** -> Skip to Step 7 (default catalog is used). - -If user says **yes**: - -**Show the user the ConfigMap template:** - -```yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: nim-model-catalog - namespace: [namespace] -data: - model-catalog.json: | - [ - { - "name": "[model-name]", - "displayName": "[display-name]", - "shortDescription": "[description]" - } - ] -``` - -Ask user which models to include, generate the ConfigMap, and confirm before creating. - -**MCP Tool**: `resources_create_or_update` (from openshift) - -### Step 7: Validate NIM Platform Readiness - -Check that the NIM platform is ready for model deployments. - -**Step 7a: Check Account CR Status** - -**MCP Tool**: `resources_get` (from openshift) - -**Parameters**: -- `apiVersion`: `"nim.opendatahub.io/v1"` - REQUIRED -- `kind`: `"Account"` - REQUIRED -- `namespace`: user-specified namespace - REQUIRED -- `name`: `"nim-account"` - REQUIRED - -**Expected Output**: Account object with `status.conditions` showing ready state - -**Step 7b: Verify NIM ServingRuntimes** - -**MCP Tool**: `list_serving_runtimes` (from rhoai) - preferred if rhoai MCP available - -**Parameters**: -- `namespace`: user-specified namespace - REQUIRED -- `include_templates`: `false` - -**Fallback MCP Tool**: `resources_list` (from openshift) -- `apiVersion`: `"serving.kserve.io/v1alpha1"`, `kind`: `"ServingRuntime"`, `namespace`: user-specified namespace - -**Expected Output**: List of ServingRuntime objects including NIM runtimes - -**Step 7c: (Optional) GPU Inventory Check** - -If `ai-observability` MCP server is available, use `get_gpu_info` to report cluster GPU inventory. - -**Report results** showing: Account CR status, credentials status (created/existing), available NIM ServingRuntimes, GPU inventory (if available), and next steps (`/model-deploy`). - -**On failure**: Report Account CR status details and error message. Suggest troubleshooting steps: check Account CR events, verify NGC API key validity, check OpenShift AI operator logs. Ask if user wants help troubleshooting. - -## Common Issues - -For common issues (GPU scheduling, OOMKilled, image pull errors, RBAC), see [common-issues.md](../references/common-issues.md). - -### Issue 1: Account CR Stuck in "Pending" - -**Error**: Account CR `status.conditions` shows pending state indefinitely - -**Cause**: NGC credentials are invalid, expired, or the RHOAI operator cannot reach NGC services. - -**Solution:** -1. Verify NGC API key is valid by testing at https://ngc.nvidia.com -2. Check Account CR events: use `events_list` filtered by namespace to find events related to the Account resource -3. Regenerate NGC API key and re-run `/nim-setup` with new credentials - -### Issue 2: GPU Operator Not Installed - -**Error**: ClusterServiceVersion for `gpu-operator-certified` not found - -**Cause**: NVIDIA GPU Operator was not installed from OperatorHub. - -**Solution:** -1. Install NVIDIA GPU Operator from OperatorHub in the OpenShift console -2. Wait for the operator to reach `Succeeded` phase -3. Verify GPU nodes are detected: check for `nvidia.com/gpu` resources on nodes -4. Re-run `/nim-setup` - -### Issue 3: NIM ServingRuntimes Not Appearing - -**Error**: `resources_list` for ServingRuntimes returns no NIM runtimes - -**Cause**: Account CR is not yet ready, or the RHOAI operator version does not include NIM support. - -**Solution:** -1. Check Account CR status — runtimes are created asynchronously after the Account becomes ready -2. Wait 2-3 minutes and re-check -3. Verify RHOAI operator version supports NIM integration -4. Check operator logs for errors - -## Dependencies - -### MCP Tools -See [Prerequisites](#prerequisites) for the complete list of required and optional MCP tools. - -### Related Skills -- `/model-deploy` - Deploy a model using NIM runtime after setup is complete -- `/serving-runtime-config` - Configure custom serving runtimes if NIM doesn't fit - -### Reference Documentation -- [supported-runtimes.md](../../docs/references/supported-runtimes.md) - NIM runtime capabilities and requirements -- [live-doc-lookup.md](../references/live-doc-lookup.md) - Protocol for fetching current RHOAI/NIM documentation - -## Critical: Human-in-the-Loop Requirements - -See [skill-conventions.md](../references/skill-conventions.md) for general HITL and security conventions. - -**Skill-specific checkpoints:** -- Before creating each Secret: display manifest (credentials REDACTED), confirm -- Before creating Account CR: display manifest, confirm -- Before creating ConfigMap (if applicable): display manifest, confirm -- **NEVER** display actual NGC API key values in output - -## Example Usage - -See [nim-setup examples](../../docs/examples/nim-setup.md) for a complete first-time NIM setup walkthrough. diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/common-issues.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/with_skills/rh-ai-engineer__nim-setup/environment/skills/references/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/common-issues.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/model-deploy.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/model-deploy.md deleted file mode 100644 index dcd79e44..00000000 --- a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/model-deploy.md +++ /dev/null @@ -1,160 +0,0 @@ -# /model-deploy Examples - -Example interactions demonstrating the model-deploy skill workflow. - -## Example 1: Deploy Llama 3.1 8B with vLLM - -``` -User: "Deploy Llama 3.1 8B on my cluster" - -Agent: [Invokes /model-deploy skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand runtime capabilities. - -Step 1 of 10: Gathering deployment information. - -| Setting | Value | Source | -|---------|-------|--------| -| Model | Llama 3.1 8B Instruct | user input | -| Runtime | TBD | auto-detect | -| Namespace | ? | needs input | -| Model Source | ? | needs input | -| Deployment Mode | Serverless | default | - -What namespace should I deploy to, and where are the model weights stored? -(e.g., S3 bucket, OCI registry, PVC) - -User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" - -Agent: Step 2 of 10: Determining runtime. - -I consulted supported-runtimes.md to understand runtime capabilities. - -Based on Llama 3.1 8B (open-source HuggingFace model): - -Recommended runtime: vLLM -Reason: Default runtime for open-source LLMs. Broad compatibility, - OpenAI-compatible API, no additional setup required. - -Proceed with vLLM? (yes / no / suggest alternative) - -User: "yes" - -Agent: Step 3 of 10: Looking up hardware profile. - -I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. - -| Requirement | Value | -|-------------|-------| -| GPUs | 1x A10G/L4/A100/H100 | -| VRAM | 16GB | -| Key Args | --max-model-len=4096 --tool-call-parser hermes | - -Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). -Ensure your cluster has at least 1 GPU with 16GB VRAM. - -Step 5 of 10: Skipped (NIM platform check not needed for vLLM). - -Step 6-7 of 10: Generating InferenceService YAML. - -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - name: llama-3-1-8b - namespace: ai-models - annotations: - serving.kserve.io/deploymentMode: Serverless -spec: - predictor: - model: - modelFormat: - name: vLLM - runtime: vllm-runtime - storageUri: s3://my-bucket/llama-3.1-8b - resources: - limits: - nvidia.com/gpu: "1" - requests: - cpu: "4" - memory: "16Gi" - args: - - --max-model-len=4096 - - --tool-call-parser=hermes - -| Setting | Value | -|---------|-------| -| Model | llama-3-1-8b | -| Runtime | vllm-runtime | -| GPUs | 1 | -| Namespace | ai-models | -| Mode | Serverless | - -Proceed with creating this InferenceService? (yes/no/modify) - -User: "yes" - -Agent: Step 8 of 10: Creating InferenceService... - [Creates via resources_create_or_update] - - InferenceService created. Monitoring rollout... - -Step 9 of 10: Monitoring rollout. - -| Stage | Status | -|-------|--------| -| Pod Scheduled | Complete | -| Image Pulled | Complete | -| Container Started | Complete | -| Model Loaded | Complete | -| Ready | Complete | - -Step 10 of 10: Deployment complete! - -Model Deployment Successful - -Model: llama-3-1-8b -Runtime: vLLM -Namespace: ai-models -GPUs: 1 - -Inference Endpoint: - URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com - API: OpenAI-compatible REST - -Quick Test: - curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' - -Next Steps: - Monitor performance: "/ai-observability llama-3-1-8b" -``` - -## Example 2: Deploy a Model with NIM (NIM Not Set Up) - -``` -User: "Deploy Llama 3.1 70B with NIM" - -Agent: [Invokes /model-deploy skill] - ... - Step 2: User requested NIM runtime. - - Step 5: Checking NIM platform... - -NIM platform is not set up in namespace ai-models. - -The NIM runtime requires a one-time platform setup. Would you like to: -1. Run /nim-setup now to configure the NIM platform -2. Switch to vLLM runtime instead (no setup required) -3. Abort deployment - -User: "1" - -Agent: [Invokes /nim-setup skill — walks through NGC credential setup] - ... - NIM Platform Setup Complete. - - Resuming model deployment... - [Continues from Step 6 with NIM runtime] -``` diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/nim-setup.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/nim-setup.md deleted file mode 100644 index ccfc1a5d..00000000 --- a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/nim-setup.md +++ /dev/null @@ -1,115 +0,0 @@ -# /nim-setup Examples - -Example interactions demonstrating the nim-setup skill workflow. - -## Example 1: First-Time NIM Setup - -``` -User: "Set up NIM on my cluster" - -Agent: [Invokes /nim-setup skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand NIM platform requirements. - -Let me verify the prerequisites for NIM setup. - -Step 1 of 7: Checking GPU Operator and NFD Operator... - -GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) -NFD Operator: nfd.v4.19.0 (Succeeded) - -Both operators are installed and healthy. - -Step 2 of 7: I need your NGC credentials. - -To set up NIM, I need your NVIDIA NGC API key. -You can generate one at: https://ngc.nvidia.com/setup/api-key - -Please provide: -1. Your NGC API key -2. The target namespace for NIM resources - -User: "Here's my key: [user provides NGC API key], namespace: ai-models" - -Agent: Step 3 of 7: Creating NGC image pull secret... - -I'll create a docker-registry secret for pulling NIM images from nvcr.io: - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-image-pull-secret - namespace: ai-models - type: kubernetes.io/dockerconfigjson - data: - .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] - -Should I create this image pull secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - Image pull secret created successfully. - -Step 4 of 7: Creating NGC API key secret... - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-api-key - namespace: ai-models - type: Opaque - stringData: - NGC_API_KEY: "[REDACTED]" - -Should I create this API key secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - API key secret created successfully. - -Step 5 of 7: Creating NIM Account CR... - - apiVersion: nim.opendatahub.io/v1 - kind: Account - metadata: - name: nim-account - namespace: ai-models - spec: - apiKeySecret: - name: ngc-api-key - imagePullSecret: - name: ngc-image-pull-secret - -Should I create this NIM Account CR in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates Account CR via resources_create_or_update] - Account CR created successfully. - -Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) - -User: "no" - -Agent: Step 7 of 7: Validating NIM platform readiness... - -NIM Platform Setup Complete - -Account CR: `nim-account` (namespace: ai-models) - Status: Ready - -NGC Credentials: - Image Pull Secret: ngc-image-pull-secret [created] - API Key Secret: ngc-api-key [created] - -NIM ServingRuntimes Available: - - nim-llama-3-1-8b-instruct - - nim-llama-3-1-70b-instruct - - nim-mistral-7b-instruct - -Next Steps: - To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" -``` diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/known-model-profiles.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/known-model-profiles.md deleted file mode 100644 index 40148981..00000000 --- a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/known-model-profiles.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Known Model Hardware Profiles -category: references -tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] -semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] -use_cases: [model-deploy, debug-inference, ai-observability] -last_updated: 2026-03-10 ---- - -# Known Model Hardware Profiles - -Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). - -**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. - -## Model Source Conventions - -Each model lists a recommended `storageUri` with its authentication requirements: -- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** -- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). -- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. - -When the user does not specify a model source, use the `hf://` URI listed in the profile below. - -## Llama 3.x (Meta) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | -| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | -| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | -| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | - -- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) -- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly - -## Granite 3.x (IBM/Red Hat) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | -| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | - -- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) -- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser granite --chat-template` -- Red Hat-supported model family on RHOAI - -## Mixtral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | -| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) -- Mixture-of-Experts architecture: only ~13B/45B parameters active per token - -## Mistral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | -| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) - -## When a Model Is Not Listed - -If the requested model is not in this file, the agent MUST use the live doc lookup protocol: - -1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure -2. Fetch hardware specs from the appropriate source -3. Report findings to the user before proceeding with deployment - -Common cases requiring live lookup: -- Newly released models (after this file's last update) -- Domain-specific fine-tuned models -- Models with custom quantization -- NIM-specific optimized profiles diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/supported-runtimes.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/supported-runtimes.md deleted file mode 100644 index 9d018651..00000000 --- a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/supported-runtimes.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: Supported Serving Runtimes -category: references -tags: [runtimes, vllm, nim, caikit, tgis, serving] -semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] -use_cases: [model-deploy, serving-runtime-config, nim-setup] -last_updated: 2026-02-26 ---- - -# Supported Serving Runtimes - -This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. - -## Runtime Comparison - -| Runtime | API Type | Model Format | GPU Support | License | Setup Required | -|---------|----------|-------------|-------------|---------|----------------| -| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | -| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | -| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | -| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | - -## vLLM - -**Default runtime for most open-source models.** - -- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) -- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized -- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) -- **Model source**: S3-compatible storage, OCI registry, PVC, URI -- **Key features**: - - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) - - PagedAttention for efficient memory management - - Tensor parallelism for multi-GPU inference - - Tool/function calling support (`--tool-call-parser`) - - Continuous batching for high throughput -- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name - -## NVIDIA NIM - -**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** - -- **API**: OpenAI-compatible REST -- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog -- **GPU support**: NVIDIA only (requires specific GPU models per model profile) -- **Model source**: NGC model catalog (pulled automatically via NGC credentials) -- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) -- **Key features**: - - TensorRT-LLM optimization for lower latency - - Pre-compiled model engines (no compilation on first load) - - Optimized and generic profiles per GPU type - - Automatic model download from NGC -- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. -- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM -- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. -- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state - -## Caikit+TGIS - -**Red Hat's Caikit format with Text Generation Inference Server.** - -- **API**: gRPC (not REST) -- **Model formats**: Caikit format (requires conversion from HuggingFace) -- **GPU support**: NVIDIA -- **Model source**: S3-compatible storage -- **Key features**: - - Red Hat-supported runtime - - gRPC API for streaming inference - - Integrated with RHOAI model serving platform -- **When to choose**: When the model is already in Caikit format or when gRPC API is required. -- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name - -## Custom Runtimes - -**User-provided ServingRuntime CRs for unsupported frameworks.** - -- **API**: Defined by the custom runtime -- **Model formats**: Defined by the custom runtime -- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. -- **How to create**: Use `/serving-runtime-config` skill -- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility - -## Runtime Selection Decision Tree - -``` -Is the user's preferred runtime explicitly stated? -├── Yes → Use that runtime -└── No → Continue - -Is the model available in the NGC NIM catalog? -├── Yes → Suggest NIM (with vLLM as fallback) -│ Note: Requires /nim-setup and NVIDIA GPUs -└── No → Continue - -Is the model in Caikit format? -├── Yes → Caikit+TGIS -└── No → Continue - -Is the model a standard open-source LLM (HuggingFace-compatible)? -├── Yes → vLLM (default) -└── No → Custom runtime via /serving-runtime-config -``` diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/docs/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/common-issues.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/references/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/serving-runtime-config/SKILL.md b/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/serving-runtime-config/SKILL.md deleted file mode 100644 index 39ba97a1..00000000 --- a/evaluation/with_skills/rh-ai-engineer__serving-runtime-config/environment/skills/serving-runtime-config/SKILL.md +++ /dev/null @@ -1,278 +0,0 @@ ---- -name: serving-runtime-config -description: | - Configure custom ServingRuntime CRs on OpenShift AI for model serving frameworks not covered by built-in runtimes. - - Use when: - - "Create a custom serving runtime" - - "I need a runtime for ONNX / Triton / custom framework" - - "Customize vLLM runtime parameters" - - "What serving runtimes are available?" - - "Add a custom container image for model serving" - - Handles listing existing runtimes, creating new ServingRuntime CRs, and validating compatibility with target models. - - NOT for deploying models (use /model-deploy after runtime is configured). - NOT for NIM platform setup (use /nim-setup). -model: inherit -color: blue ---- - -# /serving-runtime-config Skill - -Configure custom ServingRuntime custom resources on Red Hat OpenShift AI. Use when built-in runtimes (vLLM, NIM, Caikit+TGIS) do not support the target model framework, or when customizing an existing runtime's parameters (env vars, model format, container image). - -## Prerequisites - -**Required MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) - -**Required MCP Tools** (from rhoai): -- `list_serving_runtimes` - List available runtimes and platform templates with supported model formats -- `create_serving_runtime` - Instantiate a serving runtime from a platform template (no YAML needed) -- `list_data_science_projects` - Validate namespace is an RHOAI project - -**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools** (from openshift): -- `resources_get` (from openshift) - Inspect existing ServingRuntime CRs in detail -- `resources_create_or_update` (from openshift) - Create fully custom ServingRuntime CR (when not using templates) - -**Optional MCP Server**: `ai-observability` ([AI Observability MCP](https://github.com/rh-ai-quickstart/ai-observability-summarizer)) - -**Optional MCP Tools** (from ai-observability): -- `list_models` - Verify deployed models use the new runtime - -**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, KServe, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). - -## When to Use This Skill - -**Use this skill when you need to:** -- Create a custom ServingRuntime for a framework not covered by built-in runtimes -- Customize an existing runtime's parameters (env vars, container image, model format) -- Instantiate a platform template runtime into a namespace -- List and compare available serving runtimes and templates - -**Do NOT use this skill when:** -- You want to deploy a model using an existing runtime (use `/model-deploy`) -- You need NIM platform setup (use `/nim-setup`) -- You need to troubleshoot a deployment (use `/debug-inference`) - -## Workflow - -### Step 1: Gather Requirements - -**Ask the user for:** -- **Use case**: What framework/model needs serving? (e.g., "ONNX model", "custom TensorRT engine", "vLLM with custom args") -- **Namespace**: Target namespace for the ServingRuntime -- **Intent**: New runtime from scratch, or customize an existing one? - -**Document Consultation** (read before listing runtimes): -1. **Action**: Read [supported-runtimes.md](../../docs/references/supported-runtimes.md) using the Read tool to understand available runtimes and their capabilities -2. **Output to user**: "I consulted [supported-runtimes.md](../../docs/references/supported-runtimes.md) to understand available runtimes." - -**MCP Tool**: `list_serving_runtimes` (from rhoai) - -**Parameters**: -- `namespace`: user-specified namespace - REQUIRED -- `include_templates`: `true` - REQUIRED (shows both existing runtimes and platform templates) - -**Present findings** in a table: - -| Runtime Name | Model Format | Source | Requires Instantiation | -|--------------|-------------|--------|----------------------| -| [name] | [format] | namespace / template | [true/false] | - -The response distinguishes between: -- **Existing runtimes** (`source: "namespace"`) - ready to use with `/model-deploy` -- **Platform templates** (`source: "template"`, `requires_instantiation: true`) - must be instantiated first - -If an existing runtime fits the user's need, recommend using it directly with `/model-deploy`. If a platform template fits, offer to instantiate it (Step 4 alternative). Otherwise, proceed to Step 2 for custom runtime creation. - -Proceed automatically without waiting for user input. - -### Step 2: Determine Runtime Configuration - -Based on the user's framework and model requirements, determine the ServingRuntime spec. - -**If customizing an existing runtime:** - -**MCP Tool**: `resources_get` (from openshift) - -**Parameters**: -- `apiVersion`: `"serving.kserve.io/v1alpha1"` - REQUIRED -- `kind`: `"ServingRuntime"` - REQUIRED -- `namespace`: user-specified namespace - REQUIRED -- `name`: name of the existing runtime to customize - REQUIRED - -Extract the current spec as a starting point. Present the current configuration and ask what the user wants to change. - -**If the user requests a runtime for an unfamiliar framework -> Trigger live doc lookup:** -1. **Action**: Read [live-doc-lookup.md](../references/live-doc-lookup.md) using the Read tool for the lookup protocol -2. **Output to user**: "Framework [name] is not in my cached runtimes. I'll look up its serving requirements." -3. Use **WebFetch** to retrieve specs from Red Hat OpenShift AI documentation -4. Extract: container image, model format name, supported protocols, required env vars -5. **Output to user**: "I looked up [framework] on [source] to confirm its runtime requirements: [summary]" - -**Collect runtime parameters:** - -| Parameter | Value | Source | -|-----------|-------|--------| -| Runtime name | [name] | user input | -| Container image | [image:tag] | user input / doc lookup | -| Model format name | [format] | user input / doc lookup | -| Supported protocol versions | [v1, v2, grpc-v2] | user input / default | -| Multi-model serving | [true/false] | default: false (single-model) | -| Environment variables | [list] | user input | -| GPU resource requirements | [limits] | user input | - -Proceed automatically without waiting for user input. - -### Step 3: Generate ServingRuntime YAML - -Generate the ServingRuntime manifest using values from Steps 1-2. - -```yaml -apiVersion: serving.kserve.io/v1alpha1 -kind: ServingRuntime -metadata: - name: [runtime-name] - namespace: [namespace] - labels: - opendatahub.io/dashboard: "true" - annotations: - openshift.io/display-name: "[Display Name]" -spec: - supportedModelFormats: - - name: [model-format-name] - version: "[version]" - autoSelect: true - multiModel: false - containers: - - name: kserve-container - image: [container-image:tag] - ports: - - containerPort: 8080 - protocol: TCP - env: - - name: [ENV_VAR_NON_SECRET] - value: "[non-sensitive-value]" - - name: [SECRET_ENV_VAR] - valueFrom: - secretKeyRef: - name: [k8s-secret-name] - key: [secret-key-name] - resources: - limits: - nvidia.com/gpu: "[gpu-count]" - requests: - cpu: "[cpu]" - memory: "[memory]" -``` - -Display the ServingRuntime YAML to the user, **redacting any sensitive values**. - -**Ask**: "Proceed with creating this ServingRuntime? (yes/no/modify)" - -Proceed automatically without waiting for user input. - -- If **yes** -> Proceed to Step 4 -- If **no** -> Abort -- If **modify** -> Ask what to change, regenerate YAML, return to this step - -### Step 4: Create ServingRuntime - -**If instantiating from a platform template** (user chose a template from Step 1): - -**MCP Tool**: `create_serving_runtime` (from rhoai) - -**Parameters**: -- `namespace`: target namespace - REQUIRED -- `template_name`: name of the template to instantiate (e.g., `"vllm-cuda-runtime-template"`) - REQUIRED - -The response includes the created runtime name, display name, and supported model formats. - -**If creating a fully custom runtime** (custom container image, non-template configuration): - -**MCP Tool**: `resources_create_or_update` (from openshift) - -**Parameters**: -- `manifest`: full ServingRuntime manifest as JSON string - REQUIRED -- `namespace`: user-specified namespace - REQUIRED - -**Error Handling**: -- If namespace not found -> Report error, suggest creating namespace or using `/ds-project-setup` -- If runtime name already exists -> Ask user: "ServingRuntime `[name]` already exists. Update it? (yes/no)" -- If CRD not found -> Report: "ServingRuntime CRD not available. Ensure Red Hat OpenShift AI operator is installed." -- If RBAC error -> Report insufficient permissions - -### Step 5: Validate Runtime - -**MCP Tool**: `list_serving_runtimes` (from rhoai) - -**Parameters**: -- `namespace`: user-specified namespace - REQUIRED -- `include_templates`: `false` - -Verify the runtime appears in the namespace runtime list. - -For detailed inspection: - -**MCP Tool**: `resources_get` (from openshift) - -**Parameters**: -- `apiVersion`: `"serving.kserve.io/v1alpha1"` - REQUIRED -- `kind`: `"ServingRuntime"` - REQUIRED -- `namespace`: user-specified namespace - REQUIRED -- `name`: the created runtime name - REQUIRED - -**Report results** showing: runtime name, namespace, model format, container image, and next steps (`/model-deploy` to deploy a model using this runtime). - -## Common Issues - -For common issues (GPU scheduling, OOMKilled, image pull errors, RBAC), see [common-issues.md](../references/common-issues.md). - -### Issue 1: InferenceService Cannot Find Runtime - -**Error**: InferenceService status shows "Unknown" or runtime not matched - -**Cause**: The `modelFormat.name` in the InferenceService does not match any `supportedModelFormats[].name` in available ServingRuntimes. - -**Solution:** -1. Verify the model format name matches exactly (case-sensitive) -2. Check the runtime is in the same namespace as the InferenceService -3. Ensure the runtime has `opendatahub.io/dashboard: "true"` label - -### Issue 2: Runtime Port Mismatch - -**Error**: InferenceService created but health checks fail, endpoint returns connection refused - -**Cause**: The `containerPort` in the ServingRuntime does not match the port the serving framework actually listens on. - -**Solution:** -1. Check the framework's documentation for its default serving port -2. Update the `containerPort` in the ServingRuntime spec -3. Or set an environment variable to configure the framework's listen port to match - -## Dependencies - -### MCP Tools -See [Prerequisites](#prerequisites) for the complete list of required and optional MCP tools. - -### Related Skills -- `/model-deploy` - Deploy a model using the configured runtime -- `/nim-setup` - NIM platform setup (if NIM runtime is needed instead) -- `/debug-inference` - Troubleshoot InferenceService failures after deployment - -### Reference Documentation -- [supported-runtimes.md](../../docs/references/supported-runtimes.md) - Runtime capabilities and model format names -- [live-doc-lookup.md](../references/live-doc-lookup.md) - Protocol for fetching specs for unknown frameworks - -## Critical: Human-in-the-Loop Requirements - -See [skill-conventions.md](../references/skill-conventions.md) for general HITL and security conventions. - -**Skill-specific checkpoints:** -- After listing existing runtimes (Step 1): confirm whether to create new or customize existing -- After collecting parameters (Step 2): confirm runtime configuration -- Before creating ServingRuntime (Step 3): display full YAML, confirm -- **NEVER** overwrite an existing ServingRuntime without user confirmation diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/common-issues.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/model-deploy.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/model-deploy.md deleted file mode 100644 index dcd79e44..00000000 --- a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/model-deploy.md +++ /dev/null @@ -1,160 +0,0 @@ -# /model-deploy Examples - -Example interactions demonstrating the model-deploy skill workflow. - -## Example 1: Deploy Llama 3.1 8B with vLLM - -``` -User: "Deploy Llama 3.1 8B on my cluster" - -Agent: [Invokes /model-deploy skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand runtime capabilities. - -Step 1 of 10: Gathering deployment information. - -| Setting | Value | Source | -|---------|-------|--------| -| Model | Llama 3.1 8B Instruct | user input | -| Runtime | TBD | auto-detect | -| Namespace | ? | needs input | -| Model Source | ? | needs input | -| Deployment Mode | Serverless | default | - -What namespace should I deploy to, and where are the model weights stored? -(e.g., S3 bucket, OCI registry, PVC) - -User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" - -Agent: Step 2 of 10: Determining runtime. - -I consulted supported-runtimes.md to understand runtime capabilities. - -Based on Llama 3.1 8B (open-source HuggingFace model): - -Recommended runtime: vLLM -Reason: Default runtime for open-source LLMs. Broad compatibility, - OpenAI-compatible API, no additional setup required. - -Proceed with vLLM? (yes / no / suggest alternative) - -User: "yes" - -Agent: Step 3 of 10: Looking up hardware profile. - -I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. - -| Requirement | Value | -|-------------|-------| -| GPUs | 1x A10G/L4/A100/H100 | -| VRAM | 16GB | -| Key Args | --max-model-len=4096 --tool-call-parser hermes | - -Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). -Ensure your cluster has at least 1 GPU with 16GB VRAM. - -Step 5 of 10: Skipped (NIM platform check not needed for vLLM). - -Step 6-7 of 10: Generating InferenceService YAML. - -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - name: llama-3-1-8b - namespace: ai-models - annotations: - serving.kserve.io/deploymentMode: Serverless -spec: - predictor: - model: - modelFormat: - name: vLLM - runtime: vllm-runtime - storageUri: s3://my-bucket/llama-3.1-8b - resources: - limits: - nvidia.com/gpu: "1" - requests: - cpu: "4" - memory: "16Gi" - args: - - --max-model-len=4096 - - --tool-call-parser=hermes - -| Setting | Value | -|---------|-------| -| Model | llama-3-1-8b | -| Runtime | vllm-runtime | -| GPUs | 1 | -| Namespace | ai-models | -| Mode | Serverless | - -Proceed with creating this InferenceService? (yes/no/modify) - -User: "yes" - -Agent: Step 8 of 10: Creating InferenceService... - [Creates via resources_create_or_update] - - InferenceService created. Monitoring rollout... - -Step 9 of 10: Monitoring rollout. - -| Stage | Status | -|-------|--------| -| Pod Scheduled | Complete | -| Image Pulled | Complete | -| Container Started | Complete | -| Model Loaded | Complete | -| Ready | Complete | - -Step 10 of 10: Deployment complete! - -Model Deployment Successful - -Model: llama-3-1-8b -Runtime: vLLM -Namespace: ai-models -GPUs: 1 - -Inference Endpoint: - URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com - API: OpenAI-compatible REST - -Quick Test: - curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' - -Next Steps: - Monitor performance: "/ai-observability llama-3-1-8b" -``` - -## Example 2: Deploy a Model with NIM (NIM Not Set Up) - -``` -User: "Deploy Llama 3.1 70B with NIM" - -Agent: [Invokes /model-deploy skill] - ... - Step 2: User requested NIM runtime. - - Step 5: Checking NIM platform... - -NIM platform is not set up in namespace ai-models. - -The NIM runtime requires a one-time platform setup. Would you like to: -1. Run /nim-setup now to configure the NIM platform -2. Switch to vLLM runtime instead (no setup required) -3. Abort deployment - -User: "1" - -Agent: [Invokes /nim-setup skill — walks through NGC credential setup] - ... - NIM Platform Setup Complete. - - Resuming model deployment... - [Continues from Step 6 with NIM runtime] -``` diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/nim-setup.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/nim-setup.md deleted file mode 100644 index ccfc1a5d..00000000 --- a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/nim-setup.md +++ /dev/null @@ -1,115 +0,0 @@ -# /nim-setup Examples - -Example interactions demonstrating the nim-setup skill workflow. - -## Example 1: First-Time NIM Setup - -``` -User: "Set up NIM on my cluster" - -Agent: [Invokes /nim-setup skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand NIM platform requirements. - -Let me verify the prerequisites for NIM setup. - -Step 1 of 7: Checking GPU Operator and NFD Operator... - -GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) -NFD Operator: nfd.v4.19.0 (Succeeded) - -Both operators are installed and healthy. - -Step 2 of 7: I need your NGC credentials. - -To set up NIM, I need your NVIDIA NGC API key. -You can generate one at: https://ngc.nvidia.com/setup/api-key - -Please provide: -1. Your NGC API key -2. The target namespace for NIM resources - -User: "Here's my key: [user provides NGC API key], namespace: ai-models" - -Agent: Step 3 of 7: Creating NGC image pull secret... - -I'll create a docker-registry secret for pulling NIM images from nvcr.io: - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-image-pull-secret - namespace: ai-models - type: kubernetes.io/dockerconfigjson - data: - .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] - -Should I create this image pull secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - Image pull secret created successfully. - -Step 4 of 7: Creating NGC API key secret... - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-api-key - namespace: ai-models - type: Opaque - stringData: - NGC_API_KEY: "[REDACTED]" - -Should I create this API key secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - API key secret created successfully. - -Step 5 of 7: Creating NIM Account CR... - - apiVersion: nim.opendatahub.io/v1 - kind: Account - metadata: - name: nim-account - namespace: ai-models - spec: - apiKeySecret: - name: ngc-api-key - imagePullSecret: - name: ngc-image-pull-secret - -Should I create this NIM Account CR in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates Account CR via resources_create_or_update] - Account CR created successfully. - -Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) - -User: "no" - -Agent: Step 7 of 7: Validating NIM platform readiness... - -NIM Platform Setup Complete - -Account CR: `nim-account` (namespace: ai-models) - Status: Ready - -NGC Credentials: - Image Pull Secret: ngc-image-pull-secret [created] - API Key Secret: ngc-api-key [created] - -NIM ServingRuntimes Available: - - nim-llama-3-1-8b-instruct - - nim-llama-3-1-70b-instruct - - nim-mistral-7b-instruct - -Next Steps: - To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" -``` diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/references/known-model-profiles.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/references/known-model-profiles.md deleted file mode 100644 index 40148981..00000000 --- a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/references/known-model-profiles.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Known Model Hardware Profiles -category: references -tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] -semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] -use_cases: [model-deploy, debug-inference, ai-observability] -last_updated: 2026-03-10 ---- - -# Known Model Hardware Profiles - -Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). - -**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. - -## Model Source Conventions - -Each model lists a recommended `storageUri` with its authentication requirements: -- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** -- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). -- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. - -When the user does not specify a model source, use the `hf://` URI listed in the profile below. - -## Llama 3.x (Meta) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | -| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | -| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | -| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | - -- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) -- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly - -## Granite 3.x (IBM/Red Hat) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | -| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | - -- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) -- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser granite --chat-template` -- Red Hat-supported model family on RHOAI - -## Mixtral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | -| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) -- Mixture-of-Experts architecture: only ~13B/45B parameters active per token - -## Mistral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | -| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) - -## When a Model Is Not Listed - -If the requested model is not in this file, the agent MUST use the live doc lookup protocol: - -1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure -2. Fetch hardware specs from the appropriate source -3. Report findings to the user before proceeding with deployment - -Common cases requiring live lookup: -- Newly released models (after this file's last update) -- Domain-specific fine-tuned models -- Models with custom quantization -- NIM-specific optimized profiles diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/references/supported-runtimes.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/references/supported-runtimes.md deleted file mode 100644 index 9d018651..00000000 --- a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/references/supported-runtimes.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: Supported Serving Runtimes -category: references -tags: [runtimes, vllm, nim, caikit, tgis, serving] -semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] -use_cases: [model-deploy, serving-runtime-config, nim-setup] -last_updated: 2026-02-26 ---- - -# Supported Serving Runtimes - -This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. - -## Runtime Comparison - -| Runtime | API Type | Model Format | GPU Support | License | Setup Required | -|---------|----------|-------------|-------------|---------|----------------| -| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | -| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | -| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | -| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | - -## vLLM - -**Default runtime for most open-source models.** - -- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) -- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized -- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) -- **Model source**: S3-compatible storage, OCI registry, PVC, URI -- **Key features**: - - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) - - PagedAttention for efficient memory management - - Tensor parallelism for multi-GPU inference - - Tool/function calling support (`--tool-call-parser`) - - Continuous batching for high throughput -- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name - -## NVIDIA NIM - -**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** - -- **API**: OpenAI-compatible REST -- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog -- **GPU support**: NVIDIA only (requires specific GPU models per model profile) -- **Model source**: NGC model catalog (pulled automatically via NGC credentials) -- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) -- **Key features**: - - TensorRT-LLM optimization for lower latency - - Pre-compiled model engines (no compilation on first load) - - Optimized and generic profiles per GPU type - - Automatic model download from NGC -- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. -- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM -- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. -- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state - -## Caikit+TGIS - -**Red Hat's Caikit format with Text Generation Inference Server.** - -- **API**: gRPC (not REST) -- **Model formats**: Caikit format (requires conversion from HuggingFace) -- **GPU support**: NVIDIA -- **Model source**: S3-compatible storage -- **Key features**: - - Red Hat-supported runtime - - gRPC API for streaming inference - - Integrated with RHOAI model serving platform -- **When to choose**: When the model is already in Caikit format or when gRPC API is required. -- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name - -## Custom Runtimes - -**User-provided ServingRuntime CRs for unsupported frameworks.** - -- **API**: Defined by the custom runtime -- **Model formats**: Defined by the custom runtime -- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. -- **How to create**: Use `/serving-runtime-config` skill -- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility - -## Runtime Selection Decision Tree - -``` -Is the user's preferred runtime explicitly stated? -├── Yes → Use that runtime -└── No → Continue - -Is the model available in the NGC NIM catalog? -├── Yes → Suggest NIM (with vLLM as fallback) -│ Note: Requires /nim-setup and NVIDIA GPUs -└── No → Continue - -Is the model in Caikit format? -├── Yes → Caikit+TGIS -└── No → Continue - -Is the model a standard open-source LLM (HuggingFace-compatible)? -├── Yes → vLLM (default) -└── No → Custom runtime via /serving-runtime-config -``` diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/docs/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/common-issues.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/live-doc-lookup.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/skill-conventions.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/references/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/workbench-manage/SKILL.md b/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/workbench-manage/SKILL.md deleted file mode 100644 index 5ad0fa58..00000000 --- a/evaluation/with_skills/rh-ai-engineer__workbench-manage/environment/skills/workbench-manage/SKILL.md +++ /dev/null @@ -1,396 +0,0 @@ ---- -name: workbench-manage -description: | - Create and manage Jupyter notebook workbenches on OpenShift AI with image selection, resource configuration, PVC storage, and lifecycle management. - - Use when: - - "Create a notebook workbench" - - "Spin up a Jupyter environment for data science" - - "Start / stop my workbench" - - "What notebook images are available?" - - "Delete a workbench I no longer need" - - Handles Notebook CR lifecycle: create with configurable images and resources, start/stop, attach storage, and delete with data loss warnings. - - NOT for deploying models (use /model-deploy). - NOT for creating projects (use /ds-project-setup). - NOT for managing pipelines (use /pipeline-manage). -color: blue -model: inherit -metadata: - author: "Red Hat Ecosystem Engineering" - version: "1.0" ---- - -# /workbench-manage Skill - -Create and manage Jupyter notebook workbenches on Red Hat OpenShift AI. Handles the full workbench lifecycle: listing available notebook images, creating Notebook CRs with configurable CPU/memory/GPU resources, provisioning PVC storage, starting and stopping workbenches, and deleting them with proper data loss warnings. - -## Prerequisites - -**Required MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) - -**Required MCP Tools** (from rhoai): -- `list_data_science_projects` - Validate namespace is an RHOAI Data Science Project -- `list_notebook_images` - List available notebook container images (PyTorch, TensorFlow, Standard DS, etc.) -- `list_workbenches` - List existing workbenches in a project -- `get_workbench` - Get workbench details (status, image, resources, storage) -- `create_workbench` - Create a new Notebook CR with image, resources, and storage -- `start_workbench` - Start a stopped workbench -- `stop_workbench` - Stop a running workbench -- `delete_workbench` - Delete a workbench -- `get_workbench_url` - Get the OAuth-protected notebook URL -- `list_storage` - List PVCs in the project -- `create_storage` - Create a PVC for workbench storage -- `delete_storage` - Delete a PVC -- `list_data_connections` - List data connections available to attach - -**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools** (from openshift): -- `resources_get` (from openshift) - Inspect Notebook CR details, check node GPU availability -- `events_list` (from openshift) - Check pod events when workbench is stuck - -**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). - -**Additional cluster requirements**: -- Target namespace is an RHOAI Data Science Project (label: `opendatahub.io/dashboard: "true"`) - -## When to Use This Skill - -**Use this skill when you need to:** -- Create a new Jupyter notebook workbench for a data scientist -- List available notebook images (PyTorch, TensorFlow, Standard Data Science, etc.) -- Start or stop an existing workbench -- List workbenches in a project and check their status -- Delete a workbench and its associated storage -- Provision persistent storage for a workbench - -**Do NOT use this skill when:** -- You need to create a Data Science Project first (use `/ds-project-setup`) -- You want to deploy a model for inference (use `/model-deploy`) -- You need to manage data science pipelines (use `/pipeline-manage`) -- You need to troubleshoot a model deployment (use `/debug-inference`) - -## Workflow - -### Step 1: Determine Intent - -**Ask the user what they want to do:** -- **Create** a new workbench -- **Start / Stop** an existing workbench -- **List** workbenches in a project -- **Delete** a workbench - -**Ask for the target namespace** (required for all operations). - -**Validate namespace** is a Data Science Project: - -**MCP Tool**: `list_data_science_projects` (from rhoai) - -**Parameters**: none - -Verify the user-specified namespace appears in the project list. If not, report: "Namespace `[name]` is not an RHOAI Data Science Project. Use `/ds-project-setup` to create one." - -**Route to the appropriate sub-workflow:** -- Create -> Step 2 -- Start/Stop -> Step 5 -- List -> Use `list_workbenches`, display results, done -- Delete -> Step 6 - -### Step 2: Gather Configuration (Create) - -**List available notebook images:** - -**MCP Tool**: `list_notebook_images` (from rhoai) - -**Parameters**: none - -**Present available images** in a table: - -| Image Name | Description | -|------------|-------------| -| [name] | [description] | - -**Ask the user for workbench configuration:** -- **Workbench name**: DNS-compatible name (lowercase, hyphens, max 63 chars) -- **Image**: Selection from the available images list -- **CPU**: Number of CPU cores (default: 2) -- **Memory**: Memory allocation (default: 8Gi) -- **Storage size**: PVC size for persistent storage (default: 20Gi) -- **GPU** (optional): Number of GPUs to attach (e.g., 1) - -**Display configuration table:** - -| Setting | Value | -|---------|-------| -| Workbench name | [name] | -| Namespace | [namespace] | -| Image | [image_name] | -| CPU | [cpu] cores | -| Memory | [memory] | -| Storage | [storage_size] | -| GPU | [gpu_count or none] | - -Proceed automatically without waiting for user input. - -### Step 3: Provision Storage (Create) - -**Check existing storage:** - -**MCP Tool**: `list_storage` (from rhoai) - -**Parameters**: -- `namespace`: target namespace - REQUIRED - -If a suitable PVC already exists, ask user if they want to reuse it or create a new one. - -**Create PVC for workbench storage:** - -**MCP Tool**: `create_storage` (from rhoai) - -**Parameters**: -- `namespace`: target namespace - REQUIRED -- `name`: PVC name (default: `[workbench-name]-storage`) - REQUIRED -- `size`: storage size from Step 2 (e.g., `"20Gi"`) - REQUIRED -- `access_mode`: `"ReadWriteOnce"` - REQUIRED (default, single-pod access) - -**Verify creation:** - -**MCP Tool**: `list_storage` (from rhoai) - -**Parameters**: -- `namespace`: target namespace - REQUIRED - -Confirm the PVC appears and is in `Bound` or `Pending` state. - -**Error Handling**: -- If PVC name already exists -> Ask: "PVC `[name]` already exists. Reuse it or create with a different name?" -- If StorageClass not available -> Report: "Default StorageClass not configured. Contact your cluster administrator." -- If quota exceeded -> Report namespace storage quota limits - -### Step 4: Create Workbench (Create) - -**MCP Tool**: `create_workbench` (from rhoai) - -**Parameters**: -- `namespace`: target namespace - REQUIRED -- `name`: workbench name from Step 2 - REQUIRED -- `image`: selected notebook image name from Step 2 - REQUIRED -- `cpu`: CPU cores (e.g., `"2"`) - REQUIRED -- `memory`: memory allocation (e.g., `"8Gi"`) - REQUIRED -- `storage_size`: PVC storage size (e.g., `"20Gi"`) - REQUIRED - -**Monitor workbench startup** by polling status: - -**MCP Tool**: `get_workbench` (from rhoai) - -**Parameters**: -- `namespace`: target namespace - REQUIRED -- `name`: workbench name - REQUIRED - -Check until status shows the workbench is running. If status does not become ready within a reasonable polling window (3-4 checks), proceed to report current status and advise user to check back. - -**Get notebook URL:** - -**MCP Tool**: `get_workbench_url` (from rhoai) - -**Parameters**: -- `namespace`: target namespace - REQUIRED -- `name`: workbench name - REQUIRED - -**Error Handling**: -- If workbench name already exists -> Report: "Workbench `[name]` already exists. Choose a different name or manage the existing one." -- If image not found -> Re-run `list_notebook_images` and suggest available alternatives -- If RBAC error -> Report insufficient permissions to create Notebook CRs -- If GPU unavailable -> Report: "Requested GPU resources not available on cluster nodes. Reduce GPU count or wait for resources." - -**Report to user:** - -| Detail | Value | -|--------|-------| -| Workbench | [name] | -| Status | [Running / Starting] | -| Image | [image] | -| Resources | [cpu] CPU, [memory] RAM, [gpu] GPU | -| Storage | [storage_size] | -| URL | [notebook_url] | - -**Suggest next steps:** -- Access the notebook at the provided URL (OpenShift authentication required) -- Use `/ds-project-setup` to add data connections to the project -- Use `/model-deploy` when ready to deploy a trained model - -### Step 5: Manage Lifecycle (Start/Stop) - -**List workbenches to identify the target:** - -**MCP Tool**: `list_workbenches` (from rhoai) - -**Parameters**: -- `namespace`: target namespace - REQUIRED - -If user did not specify a workbench name, present the list and ask which one to manage. - -**For Start:** - -Confirm the workbench is currently stopped. If already running, report its URL and current status. - -**MCP Tool**: `start_workbench` (from rhoai) - -**Parameters**: -- `namespace`: target namespace - REQUIRED -- `name`: workbench name - REQUIRED - -**MCP Tool**: `get_workbench_url` (from rhoai) - -**Parameters**: -- `namespace`: target namespace - REQUIRED -- `name`: workbench name - REQUIRED - -**Output to user**: "Workbench `[name]` started. Access it at: [url]" - -**For Stop:** - -Proceed automatically without waiting for user input. - -**MCP Tool**: `stop_workbench` (from rhoai) - -**Parameters**: -- `namespace`: target namespace - REQUIRED -- `name`: workbench name - REQUIRED - -**Verify state change:** - -**MCP Tool**: `get_workbench` (from rhoai) - -**Parameters**: -- `namespace`: target namespace - REQUIRED -- `name`: workbench name - REQUIRED - -**Output to user**: "Workbench `[name]` stopped. Persistent storage is preserved. Use `/workbench-manage` to start it again." - -**Error Handling**: -- If workbench not found -> List available workbenches and ask user to select -- If already in target state -> Report current state (e.g., "Workbench is already running") - -### Step 6: Delete Workbench - -**Get workbench details:** - -**MCP Tool**: `get_workbench` (from rhoai) - -**Parameters**: -- `namespace`: target namespace - REQUIRED -- `name`: workbench name - REQUIRED - -**Display workbench details and data loss warning:** - -| Detail | Value | -|--------|-------| -| Workbench | [name] | -| Status | [Running / Stopped] | -| Image | [image] | -| Storage | [pvc_name] ([size]) | - -**WARNING**: Deleting this workbench will remove the Notebook CR. If the workbench is running, it will be stopped first. Any unsaved notebook work will be lost. - -**Ask**: "Delete workbench `[name]`? This action cannot be undone. (yes/no)" - -Proceed automatically without waiting for user input. - -**MCP Tool**: `delete_workbench` (from rhoai) - -**Parameters**: -- `namespace`: target namespace - REQUIRED -- `name`: workbench name - REQUIRED - -**Associated storage cleanup** (separate confirmation): - -**Ask**: "The PVC `[pvc_name]` ([size]) associated with this workbench still exists. Delete it too? WARNING: All data in this volume will be permanently lost. (yes/no)" - -Proceed automatically without waiting for user input. - -If user confirms PVC deletion: - -**MCP Tool**: `delete_storage` (from rhoai) - -**Parameters**: -- `namespace`: target namespace - REQUIRED -- `name`: PVC name - REQUIRED - -If user declines, report: "PVC `[pvc_name]` preserved. It can be reattached to a new workbench." - -**Output to user**: "Workbench `[name]` deleted. [PVC deleted / PVC preserved]." - -## Common Issues - -For common issues (GPU scheduling, OOMKilled, image pull errors, RBAC), see [common-issues.md](../references/common-issues.md). - -### Issue 1: Notebook Image Not Found - -**Error**: `create_workbench` fails with image not found or image reference is invalid - -**Cause**: The selected image name does not match any available notebook image, or the image registry is unreachable. - -**Solution:** -1. Run `list_notebook_images` to see current available images -2. Verify the exact image name (case-sensitive) -3. If no images are listed, the RHOAI operator may not have imported notebook images -- contact cluster administrator - -### Issue 2: PVC Binding Failure - -**Error**: PVC remains in `Pending` state, workbench cannot start - -**Cause**: The default StorageClass does not support the requested access mode, or no StorageClass is configured. - -**Solution:** -1. Check available StorageClasses via `resources_get` (from openshift) on `storageclasses.storage.k8s.io` -2. Use `ReadWriteOnce` access mode (most widely supported) -3. If `ReadWriteMany` is required, verify the StorageClass supports it (e.g., NFS, CephFS) -4. Contact cluster administrator if no StorageClass is available - -### Issue 3: Workbench Stuck in Starting - -**Error**: Workbench status remains in a starting/initializing state for an extended period - -**Cause**: Pod scheduling issues, image pull errors, or resource constraints. - -**Solution:** -1. Use `events_list` (from openshift) filtered by namespace to check for pod events -2. Common causes: - - `ImagePullBackOff`: Image registry unreachable or credentials missing - - `Insufficient cpu/memory`: Reduce resource requests or free up cluster resources - - `FailedScheduling`: Node taints or affinity rules preventing scheduling -3. If GPU is requested, verify GPU nodes have available capacity - -## Dependencies - -### MCP Tools -See [Prerequisites](#prerequisites) for the complete list of required and optional MCP tools. - -### Related Skills -- `/ds-project-setup` - Create a Data Science Project (prerequisite: namespace must exist) -- `/model-deploy` - Deploy a trained model from the workbench -- `/ai-observability` - Check GPU inventory before requesting GPU workbenches - -### Reference Documentation -- [skill-conventions.md](../references/skill-conventions.md) - Shared prerequisite, HITL, and security conventions - -## Example Usage - -**User**: "Create a PyTorch notebook workbench in my ml-team project with 4 CPUs and a GPU" - -**Skill response**: Validates `ml-team` is an RHOAI project, lists available notebook images, presents configuration table (PyTorch image, 4 CPU, 8Gi memory, 1 GPU, 20Gi storage), provisions PVC storage, creates workbench, monitors startup, and returns the notebook URL. - -## Critical: Human-in-the-Loop Requirements - -See [skill-conventions.md](../references/skill-conventions.md) for general HITL and security conventions. - -**Skill-specific checkpoints:** -- Before creating workbench (Step 4): display full configuration table, confirm -- Before stopping a workbench (Step 5): warn about unsaved work, confirm -- Before deleting a workbench (Step 6): display details, warn about data loss, confirm -- Before deleting associated PVC (Step 6): separate confirmation with permanent data loss warning -- **NEVER** auto-delete workbenches or storage -- **NEVER** stop a running workbench without confirmation (user may have unsaved notebook work) diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/with_skills/rh-developer__containerize-deploy/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__containerize-deploy/environment/skills/containerize-deploy/SKILL.md b/evaluation/with_skills/rh-developer__containerize-deploy/environment/skills/containerize-deploy/SKILL.md deleted file mode 100644 index 67487f6d..00000000 --- a/evaluation/with_skills/rh-developer__containerize-deploy/environment/skills/containerize-deploy/SKILL.md +++ /dev/null @@ -1,477 +0,0 @@ ---- -name: containerize-deploy -description: | - Complete end-to-end workflow for containerizing and deploying applications to OpenShift or standalone RHEL systems. Orchestrates /detect-project, /s2i-build, /deploy, /helm-deploy, and /rhel-deploy skills with user confirmation checkpoints at each phase. Supports S2I, Podman, Helm deployment strategies for OpenShift, and Podman/native deployments for RHEL hosts. Use this skill when user wants to go from source code to running application in one guided workflow. Supports resume after interruption and rollback on failure. Triggers on /containerize-deploy command. -model: inherit -color: green -metadata: - user_invocable: "true" ---- - -# /containerize-deploy Skill - -Provide a complete, guided workflow from local source code to running application on OpenShift or standalone RHEL systems. This skill orchestrates `/detect-project`, `/s2i-build`, `/deploy`, `/helm-deploy`, and `/rhel-deploy` with clear user checkpoints at each phase. - -## Overview - -``` -[Intro] → [Detect] → [Target] → [Strategy] ──┬─→ [OpenShift: S2I/Podman/Helm] ──┬─→ [Complete] - └─→ [RHEL: /rhel-deploy] ──────────┘ -``` - -## When to Use This Skill - -Use `/containerize-deploy` when a user wants a complete guided workflow from source code to running application on OpenShift or standalone RHEL systems. This skill orchestrates project detection, build strategy selection, and deployment with user confirmation at each phase. - -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -## Workflow - -### Phase 0: Introduction - -Present the workflow overview and available deployment targets/strategies. Ask: **Ready to begin?** (yes/no) - -Proceed automatically without waiting for user input. - -### Phase 1: Project Detection - -Execute the `/detect-project` workflow. - -**If Remote URL provided:** -Follow the "Remote Repository Strategy" path in `/detect-project`. -- Ask user to choose: Remote S2I, Remote Podman, or Clone. - -**If Local Files:** -Proceed with standard detection. - -```markdown -## Phase 1: Analyzing Your Project - -[If Local] -Scanning project directory for language indicators... - -[If Remote] -Analyzing remote repository options... - -... -``` - -Store confirmed values in session state, including `BUILD_STRATEGY` and `HELM_CHART_DETECTED`. - -### Phase 1.4: Deployment Target Selection - -```markdown -## Deployment Target - -Where would you like to deploy this application? - -| Target | Description | Requirements | -|--------|-------------|--------------| -| **OpenShift** | Deploy to OpenShift/Kubernetes cluster | `oc login` access | -| **RHEL Host** | Deploy directly to a standalone RHEL system | SSH access to RHEL 8+ | - -**Which target would you like to use?** -1. OpenShift - Deploy to current cluster -2. RHEL - Deploy to a RHEL host via SSH -``` - -Store `DEPLOYMENT_TARGET` in session state. - -Proceed automatically without waiting for user input. - -**If user selects "RHEL":** -- Store `DEPLOYMENT_TARGET = "rhel"` in session state -- Delegate to `/rhel-deploy` skill with detected project info -- Pass: `APP_NAME`, `LANGUAGE`, `FRAMEWORK`, `VERSION`, `BUILDER_IMAGE`, `CONTAINER_PORT` -- The `/rhel-deploy` skill handles SSH connection, deployment strategy, and service creation -- After `/rhel-deploy` completes → Go to **Phase 8 (Completion)** - -**If user selects "OpenShift":** -- Store `DEPLOYMENT_TARGET = "openshift"` in session state -- Continue to Phase 1.5 (Strategy Selection) - -### Phase 1.5: Strategy Selection - -If multiple deployment options are available (Helm chart detected, Dockerfile present, or standard project): - -```markdown -## Deployment Strategy - -Based on my analysis, you have these options: - -| Strategy | Use When | Detected | -|----------|----------|----------| -| **S2I** | Standard apps, no Dockerfile needed | [Yes/No] | -| **Podman** | Custom Containerfile/Dockerfile exists | [Yes/No] | -| **Helm** | Helm chart exists or complex deployments | [Yes/No] | - -**Detected in your project:** -[List what was found: language indicators, Dockerfile, Helm chart at ./chart] - -**Which deployment strategy would you like to use?** -1. S2I - Build with Source-to-Image -2. Podman - Build from Containerfile/Dockerfile -3. Helm - Use existing Helm chart -4. Create Helm chart - Generate a new Helm chart for your project (if no chart exists) -``` - -Store `DEPLOYMENT_STRATEGY` in session state. - -Proceed automatically without waiting for user input. - -### Phase 1.6: Image Selection (S2I/Podman only) - -If user selected S2I or Podman deployment strategy, offer image selection options: - -```markdown -## Image Selection - -**Current recommendation:** `[builder-image]` -(Based on: [language] [version]) - -**Image Selection Options:** -- **quick** - Use the recommended image (good for most cases) -- **smart** - Run `/recommend-image` for tailored selection (production vs dev, security, performance) - -Which option would you prefer? -``` - -**If user selects "smart":** -- Invoke `/recommend-image` skill with detected `LANGUAGE`, `FRAMEWORK`, `VERSION` -- Store the result in `BUILDER_IMAGE` and `IMAGE_VARIANT` session state -- Continue to Phase 2 - -**If user selects "quick":** -- Use the already-detected `BUILDER_IMAGE` -- Continue to Phase 2 - -**BRANCHING LOGIC:** -- If `DEPLOYMENT_STRATEGY` is **"S2I"** or **"Podman"** → After Phase 2, continue to **Phase 3 (S2I/Podman Path)** -- If `DEPLOYMENT_STRATEGY` is **"Helm"** → After Phase 2, go to **Phase 2-H (Helm Path)** - -### Phase 1.7: Configuration Review (MANDATORY) - -**This phase MUST NOT be skipped regardless of how the user responded to previous phases.** - -```markdown -## Configuration Review - -Before I proceed, let me confirm the deployment configuration: - -**Environment Type:** -| Type | Characteristics | -|------|-----------------| -| **Development** | `latest` tags, lower resources, quick iteration | -| **Staging** | Version tags, moderate resources, testing | -| **Production** | Pinned versions, higher resources, HA-ready | - -**Which environment is this deployment for?** -1. Development -2. Staging -3. Production - ---- - -**Configuration Approach:** -| Approach | When to Use | -|----------|-------------| -| **Runtime config** | Need to change settings without rebuilding (Recommended for prod) | -| **Build-time config** | Simpler, settings baked into image (OK for dev) | - -**How should environment variables be handled?** -1. Runtime (ConfigMap mount) -2. Build-time (baked into image) - ---- - -**Resource Settings:** -| Setting | Dev Default | Prod Default | -|---------|-------------|--------------| -| Replicas | 1 | 2+ | -| CPU limit | 200m | 400m+ | -| Memory limit | 256Mi | 512Mi+ | - -**Use defaults for your environment, or customize?** -1. Use defaults -2. Customize resources -``` - -Proceed automatically without waiting for user input. - -Store: `ENVIRONMENT_TYPE`, `CONFIG_APPROACH`, `RESOURCE_PROFILE` in session state. - -### Phase 2: OpenShift Connection - -```markdown -## Phase 2: Connecting to OpenShift - -Checking cluster connection... - -**Current Context:** -| Setting | Value | -|---|---| -| Cluster | [cluster-api-url] | -| User | [username] | -| Namespace | [current-namespace] | - -**Is this the correct cluster and namespace?** -- yes - Continue to build -- no - I need to change this - -[If no] -**To change context:** -1. Run `oc login ` in your terminal -2. Or run `oc project ` to switch namespace -3. Then tell me to continue - -**Available namespaces you have access to:** -[List first 10 namespaces/projects] - -Which namespace should I deploy to? -``` - -Store confirmed `NAMESPACE` in session state. - ---- - -## S2I/PODMAN PATH (If DEPLOYMENT_STRATEGY is "S2I" or "Podman") - -### Phase 3: Git Repository Check - -```markdown -## Git Repository - -I need a Git URL for the S2I build. - -**Detected from .git/config:** -- Remote: `[git-url]` -- Branch: `[current-branch]` - -**Is this correct?** (yes/no) - -[If no git config found] -**Please provide:** -1. Git repository URL (e.g., https://github.com/user/repo.git) -2. Branch name (default: main) -``` - -Store `GIT_URL` and `GIT_BRANCH` in session state. - -### Phase 4: Pre-Build Summary - -```markdown -## Phase 3: Build Configuration - -Here's what I'll create on OpenShift: - -**Target:** -- Cluster: [cluster] -- Namespace: [namespace] - -**Resources to Create:** - -1. **ImageStream** `[app-name]` - - Stores built container images - -2. **BuildConfig** `[app-name]` - - Source: [git-url] (branch: [branch]) - - Builder: [builder-image] - - Output: [app-name]:latest - ---- - -**Would you like to see the full YAML?** (yes/no) - -[If yes, show both YAML manifests] - ---- - -**Proceed with creating these resources and starting the build?** -- yes - Create resources and start build -- modify - I need to change something -- cancel - Stop here -``` - -### Phase 5: Execute Build - -```markdown -## Creating Build Resources... - -[x] ImageStream created: [app-name] -[x] BuildConfig created: [app-name] - -## Starting Build... - -**Build:** [app-name]-1 -**Status:** Running - ---- -**Build Logs:** -``` -[Stream S2I build output] -``` ---- - -[When complete] - -## Build Successful! - -**Build:** [app-name]-1 -**Duration:** [X]m [Y]s -**Image:** [image-reference] - -**CRITICAL: Wait for the build to reach 'Complete' status before proceeding.** - -Continue to deployment? (yes/no) -``` - -### Phase 6: Pre-Deploy Summary - -```markdown -## Phase 4: Deployment Configuration - -**Image ready!** Now let's deploy it. - -**Resources to Create:** - -1. **Deployment** `[app-name]` - - Image: [app-name]:latest - - Replicas: 1 - - Port: [detected-port] - -2. **Service** `[app-name]` - - Internal load balancer - - Port: [port] - -3. **Route** `[app-name]` - - External HTTPS access - - URL: https://[app-name]-[namespace].[domain] - ---- - -**Would you like to see the full YAML?** (yes/no) - -[If yes, show all three YAML manifests] - ---- - -**Proceed with deployment?** -- yes - Deploy the application -- modify - I need to change something -- cancel - Stop here (build artifacts preserved) -``` - -### Phase 7: Execute Deployment - -```markdown -## Deploying Application... - -[x] Deployment created: [app-name] -[x] Service created: [app-name] -[x] Route created: [app-name] - -## Waiting for Rollout... - -**Pod Status:** -| Pod | Status | Ready | -|-----|-----|---| -| [app-name]-xxx-yyy | Running | 1/1 | - -Rollout complete! -``` - -**If rollout fails** (pods not ready, CrashLoopBackOff, ImagePullBackOff, etc.): - -```markdown -## Deployment Failed - -The deployment did not complete successfully. - -**Pod Status:** -| Pod | Status | Ready | Restarts | -|-----|--------|-------|----------| -| [app-name]-xxx-yyy | [status] | 0/1 | [count] | - ---- - -**Would you like me to diagnose the issue?** - -1. **Debug Pod** (`/debug-pod`) - Investigate pod failures -2. **Debug Network** (`/debug-network`) - Check service/route connectivity -3. **Debug Build** (`/debug-build`) - Re-check build if image issues -4. **View logs manually** -5. **Rollback and stop** - -Select an option: -``` - -- If user selects a debug option → Invoke the corresponding skill -- After debugging → Offer to retry deployment - ---- - -## HELM PATH (If DEPLOYMENT_STRATEGY is "Helm") - -### Phase 2-H: Helm Deployment - -If user selected Helm in Phase 1.5, execute this path instead of Phases 3-7. - -```markdown -## Helm Deployment - -Switching to Helm deployment workflow... - -The `/helm-deploy` skill will handle: -1. Validate the Helm chart -2. Review and customize values -3. Install/upgrade the release -4. Monitor deployment -5. Present results - -Proceeding with Helm deployment... -``` - -**Delegate to `/helm-deploy` skill:** -- Pass `APP_NAME`, `NAMESPACE`, `HELM_CHART_PATH` from session state -- The helm-deploy skill handles chart detection, values review, and installation -- After helm-deploy completes → Go to **Phase 8 (Completion)** - -**If user chose "Create Helm chart":** -- Generate chart using templates from templates/helm/ -- Replace `${APP_NAME}` placeholders with detected app name -- Set `${CONTAINER_PORT}` based on detected port -- Then proceed with helm-deploy workflow - ---- - -## COMPLETION (Both paths converge here) - -### Phase 8: Completion - -Present a summary including: -- Application name, namespace, language, framework -- Access URLs (external route, internal service DNS) -- Resources created with status (ImageStream, BuildConfig, Deployment, Service, Route) -- Quick commands: view logs, scale, rebuild, delete -- Next steps: open app URL, set up webhooks, add env vars, configure autoscaling - -## Dependencies - -### Required MCP Servers -- `openshift` - cluster resource management for OpenShift deployments - -### Related Skills -- `/debug-pod` - Pod failures (CrashLoopBackOff, OOMKilled, ImagePullBackOff) -- `/debug-build` - S2I or Podman build failures -- `/debug-network` - Service connectivity issues (no endpoints, 503 errors) -- `/debug-rhel` - RHEL deployment failures (systemd, SELinux, firewall) - -### Reference Documentation -- [docs/builder-images.md](../../docs/builder-images.md) - Language detection, S2I builder images -- [docs/image-selection-criteria.md](../../docs/image-selection-criteria.md) - Image variant selection, LTS timelines -- [docs/python-s2i-entrypoints.md](../../docs/python-s2i-entrypoints.md) - Python S2I configuration -- [docs/rhel-deployment.md](../../docs/rhel-deployment.md) - RHEL host deployment -- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common error patterns and troubleshooting -- [docs/prerequisites.md](../../docs/prerequisites.md) - All required tools by skill diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__debug-build/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/with_skills/rh-developer__debug-build/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__debug-build/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/with_skills/rh-developer__debug-build/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__debug-build/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/with_skills/rh-developer__debug-build/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__debug-build/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/with_skills/rh-developer__debug-build/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__debug-build/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/with_skills/rh-developer__debug-build/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__debug-build/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/with_skills/rh-developer__debug-build/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__debug-build/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/with_skills/rh-developer__debug-build/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__debug-build/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/with_skills/rh-developer__debug-build/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__debug-build/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/with_skills/rh-developer__debug-build/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__debug-build/environment/skills/debug-build/SKILL.md b/evaluation/with_skills/rh-developer__debug-build/environment/skills/debug-build/SKILL.md deleted file mode 100644 index 0a5eebf2..00000000 --- a/evaluation/with_skills/rh-developer__debug-build/environment/skills/debug-build/SKILL.md +++ /dev/null @@ -1,315 +0,0 @@ ---- -name: debug-build -description: | - Diagnose OpenShift build failures including S2I builds, Docker/Podman builds, and BuildConfig issues. Automates multi-step diagnosis: BuildConfig validation, build pod logs, registry authentication, and source repository access. Use this skill when builds fail, hang, or produce unexpected results. Triggers on /debug-build command or phrases like "build failed", "S2I error", "can't pull builder image", "can't push to registry", "build timeout". -model: inherit -color: cyan -metadata: - user_invocable: "true" ---- - -# /debug-build Skill - -Diagnose OpenShift build failures by automatically gathering BuildConfig, Build status, build pod logs, and related resources. - -## Prerequisites - -Before running this skill: -1. User is logged into OpenShift cluster -2. User has access to the target namespace -3. Build or BuildConfig name is known (or can be identified from recent builds) - -## When to Use This Skill - -Use this skill when OpenShift builds fail, hang, or produce unexpected results. It diagnoses S2I builds, Docker/Podman builds, and BuildConfig issues by analyzing build pod logs, registry authentication, and source repository access. - -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -## Workflow - -### Step 1: Identify Target Build - -```markdown -## Build Debugging - -**Current OpenShift Context:** -- Cluster: [cluster] -- Namespace: [namespace] - -Which build would you like me to debug? - -1. **Specify build name** - Enter the build name directly (e.g., myapp-1) -2. **List failed builds** - Show recent failed builds in current namespace -3. **From BuildConfig** - Debug latest build from a specific BuildConfig - -Select an option or enter a build name: -``` - -Proceed automatically without waiting for user input. - -If user selects "List failed builds": -Use kubernetes MCP `resources_list` for builds, filter by Failed phase: - -```markdown -## Recent Failed Builds in [namespace] - -| Build | BuildConfig | Status | Started | Duration | -|-------|-------------|--------|---------|----------| -| [app-1] | [app] | Failed | [timestamp] | [duration] | -| [app-2] | [app] | Cancelled | [timestamp] | [duration] | -| [other-1] | [other] | Failed | [timestamp] | [duration] | - -Which build would you like me to debug? -``` - -Proceed automatically without waiting for user input. - -### Step 2: Get Build Status Overview - -Use kubernetes MCP `resources_get` to get Build details: - -```markdown -## Build Status: [build-name] - -**Build Info:** -| Field | Value | -|-------|-------| -| BuildConfig | [buildconfig-name] | -| Strategy | [Source/Docker/JenkinsPipeline] | -| Phase | [New/Pending/Running/Complete/Failed/Cancelled] | -| Started | [timestamp] | -| Completed | [timestamp or "Still running"] | -| Duration | [duration] | - -**Build Configuration:** -| Setting | Value | -|---------|-------| -| Source Type | [Git/Binary/Dockerfile] | -| Git URL | [url] | -| Git Ref | [branch/tag] | -| Builder Image | [image:tag] | -| Output Image | [imagestream:tag] | - -**Build Status:** -- Phase: [phase] -- Reason: [reason if failed] -- Message: [message if available] - -**Quick Assessment:** -[Based on status, provide initial assessment - e.g., "Build failed during assemble phase - likely dependency installation issue"] - -Continue with detailed analysis? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Step 3: Analyze BuildConfig - -Use kubernetes MCP `resources_get` to get BuildConfig: - -```markdown -## BuildConfig Analysis: [buildconfig-name] - -**Source Configuration:** -| Setting | Value | Status | -|---------|-------|--------| -| Git URL | [url] | [OK/WARN: check access] | -| Git Ref | [ref] | [OK/WARN: branch not found] | -| Context Dir | [dir or "/"] | [OK] | -| Source Secret | [secret-name or "None"] | [OK/MISSING] | - -**Builder Image:** -| Setting | Value | Status | -|---------|-------|--------| -| Image | [image:tag] | [OK/WARN: check exists] | -| Pull Secret | [secret-name or "None"] | [OK/MISSING] | - -**Output Configuration:** -| Setting | Value | Status | -|---------|-------|--------| -| Output To | [ImageStreamTag] | [OK] | -| Push Secret | [secret-name or "None"] | [OK/MISSING] | - -**Environment Variables:** -| Name | Value | Source | -|------|-------|--------| -| [VAR] | [value or "***"] | [Direct/ConfigMap/Secret] | - -**Issues Found:** -- [Issue 1 - e.g., "Source secret 'github-creds' referenced but not found"] -- [Issue 2 - e.g., "Builder image uses older tag, may have compatibility issues"] - -Continue to view build logs? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Step 4: Get Build Pod Logs - -Use kubernetes MCP `pod_logs` for the builder pod: - -```markdown -## Build Logs: [build-name] - -**Build Phases:** -| Phase | Status | Duration | -|-------|--------|----------| -| Fetching source | [Complete/Failed] | [duration] | -| Pulling builder image | [Complete/Failed] | [duration] | -| Assemble | [Complete/Failed] | [duration] | -| Commit | [Complete/Failed] | [duration] | -| Push | [Complete/Failed] | [duration] | - -**Failed Phase: [phase-name]** - -``` -[Last 100 lines of build logs, focused on the failing phase] -``` - -**Log Analysis:** - -[Analyze logs and identify errors:] - -**Errors Found:** -- Line [X]: [error description - e.g., "npm ERR! 404 Not Found - package 'nonexistent@1.0.0'"] -- Line [Y]: [error description - e.g., "error: unable to resolve 'github.com/private/repo'"] - -**S2I Phase Explanation:** - -[For S2I builds, explain what the failed phase does:] -- **assemble**: Installs dependencies and builds application -- **commit**: Creates the final container image layer -- **push**: Pushes image to internal registry - -Continue to check related resources? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Step 5: Check Related Resources - -Check secrets, imagestreams, and source access: - -```markdown -## Related Resources Analysis - -**ImageStreams:** -| ImageStream | Tags | Last Updated | Status | -|-------------|------|--------------|--------| -| [app] | [latest, v1.0] | [timestamp] | [OK] | -| [builder] | [imported] | [timestamp] | [OK/MISSING] | - -**Secrets:** -| Secret | Type | Used By | Status | -|--------|------|---------|--------| -| [source-secret] | kubernetes.io/basic-auth | Source | [OK/MISSING] | -| [push-secret] | kubernetes.io/dockerconfigjson | Output | [OK/MISSING] | - -**Source Repository Access:** -[If GitHub MCP available, check if source URL is accessible] -- URL: [git-url] -- Status: [Accessible/401 Unauthorized/404 Not Found/Timeout] - -**Registry Access:** -[Check if internal registry is accessible] -- Registry: image-registry.openshift-image-registry.svc:5000 -- Status: [OK/Unreachable] - -**Issues Found:** -- [Issue 1 - e.g., "Secret 'github-token' missing - cannot authenticate to private repo"] -- [Issue 2 - e.g., "Builder ImageStreamTag 'nodejs:18' not imported"] - -Continue to full diagnosis summary? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Step 6: Present Diagnosis Summary - -```markdown -## Diagnosis Summary: [build-name] - -### Root Cause - -**Primary Issue:** [Categorized root cause] - -| Category | Status | Details | -|----------|--------|---------| -| Source Access | [OK/FAIL] | [details] | -| Builder Image | [OK/FAIL] | [details] | -| Dependencies | [OK/FAIL] | [details] | -| Build Script | [OK/FAIL] | [details] | -| Registry Push | [OK/FAIL] | [details] | - -### Detailed Findings - -**[Category 1: e.g., Dependency Installation]** -- Problem: [specific problem - e.g., "npm package 'lodash@99.0.0' does not exist"] -- Evidence: [from build logs] -- Impact: [build fails during assemble phase] - -**[Category 2: e.g., Source Authentication]** -- Problem: [specific problem] -- Evidence: [from events/logs] -- Impact: [cannot clone repository] - -### Recommended Actions - -1. **[Action 1]** - [description] - ```bash - [command to fix - e.g., oc create secret generic github-token --from-literal=...] - ``` - -2. **[Action 2]** - [description] - ```bash - [command to fix - e.g., oc import-image nodejs:18 --from=registry.access.redhat.com/ubi9/nodejs-18] - ``` - -3. **[Action 3]** - [description] - -### Retry Build - -After fixing the issue: -```bash -# Start a new build -oc start-build [buildconfig-name] -n [namespace] - -# Or start build with follow -oc start-build [buildconfig-name] -n [namespace] --follow -``` - ---- - -Would you like me to: -1. Execute one of the recommended fixes -2. Retry the build -3. Compare with last successful build -4. Debug the build pod (/debug-pod) -5. Exit debugging - -Select an option: -``` - -Proceed automatically without waiting for user input. - -## Build Failure Categories - -For S2I build phase failures, common error patterns (Node.js, Python, Java), and troubleshooting decision trees, see [docs/debugging-patterns.md](../../docs/debugging-patterns.md). - -## Dependencies - -### Required MCP Servers -- `openshift` - Kubernetes/OpenShift resource access for builds, BuildConfigs, and build pod logs - -### Related Skills -- `/s2i-build` - To retry build after fixing issues -- `/debug-pod` - To debug the builder pod directly -- `/deploy` - To deploy after a successful build - -### Reference Documentation -- [docs/builder-images.md](../../docs/builder-images.md) - S2I builder image selection, version mapping -- [docs/python-s2i-entrypoints.md](../../docs/python-s2i-entrypoints.md) - Python APP_MODULE configuration -- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common error patterns -- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (oc), cluster access verification diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__debug-container/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/with_skills/rh-developer__debug-container/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__debug-container/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/with_skills/rh-developer__debug-container/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__debug-container/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/with_skills/rh-developer__debug-container/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__debug-container/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/with_skills/rh-developer__debug-container/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__debug-container/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/with_skills/rh-developer__debug-container/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__debug-container/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/with_skills/rh-developer__debug-container/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__debug-container/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/with_skills/rh-developer__debug-container/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__debug-container/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/with_skills/rh-developer__debug-container/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__debug-container/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/with_skills/rh-developer__debug-container/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__debug-container/environment/skills/debug-container/SKILL.md b/evaluation/with_skills/rh-developer__debug-container/environment/skills/debug-container/SKILL.md deleted file mode 100644 index cdbc2469..00000000 --- a/evaluation/with_skills/rh-developer__debug-container/environment/skills/debug-container/SKILL.md +++ /dev/null @@ -1,344 +0,0 @@ ---- -name: debug-container -description: | - Diagnose local container issues with Podman/Docker including image pull errors, container startup failures, OOM kills, and networking problems. Automates multi-step diagnosis: container inspect, logs retrieval, image analysis, and resource constraint checking. Use this skill when containers fail to run locally before deployment. Triggers on /debug-container command or phrases like "container won't start", "podman run fails", "local container crashing", "container exits immediately". -model: inherit -color: cyan -metadata: - user_invocable: "true" ---- - -# /debug-container Skill - -Diagnose local Podman/Docker container issues by automatically gathering container status, logs, and configuration. - -## Overview - -``` -[Identify Container] → [Inspect] → [Logs] → [Image Analysis] → [Resource Check] → [Summary] -``` - -**This skill diagnoses:** -- Container startup failures -- Immediate exit (exit codes) -- OOM kills -- Image pull errors -- Entrypoint/CMD issues -- Volume mount problems - -## Prerequisites - -1. Podman or Docker installed locally -2. Container or image name is known - -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -## When to Use This Skill - -Use `/debug-container` when local Podman or Docker containers fail to run, crash on startup, get OOM killed, or have image pull errors. This skill automates container inspection, log retrieval, image analysis, and resource constraint checking. - -## Workflow - -### Step 1: Identify Target Container - -```markdown -## Container Debugging - -What would you like me to debug? - -1. **Running/stopped container** - Debug an existing container -2. **Failed container run** - Debug a recent failed `podman run` -3. **Image issue** - Debug image pull or build problems -4. **List containers** - Show all containers (including stopped) - -Select an option or enter a container name/ID: -``` - -Proceed automatically without waiting for user input. - -If user selects "List containers": -Use Podman MCP `container_list`: - -```markdown -## Containers - -| Container ID | Image | Status | Created | Names | -|--------------|-------|--------|---------|-------| -| [abc123] | [myapp:latest] | Exited (1) 5 minutes ago | [time] | [myapp] | -| [def456] | [nginx:latest] | Up 2 hours | [time] | [webserver] | -| [ghi789] | [postgres:15] | Exited (137) 1 hour ago | [time] | [db] | - -Which container would you like me to debug? -``` - -Proceed automatically without waiting for user input. - -### Step 2: Inspect Container - -Use Podman MCP `container_inspect`: - -```markdown -## Container Inspection: [container-name] - -**Basic Info:** -| Field | Value | -|-------|-------| -| ID | [full-id] | -| Image | [image:tag] | -| Created | [timestamp] | -| Status | [running/exited/created] | - -**State:** -| Field | Value | -|-------|-------| -| Running | [true/false] | -| Paused | [true/false] | -| Restarting | [true/false] | -| OOMKilled | [true/false] | -| Exit Code | [code] | -| Error | [error message or empty] | -| Started At | [timestamp] | -| Finished At | [timestamp] | - -**Configuration:** -| Setting | Value | -|---------|-------| -| Entrypoint | [entrypoint] | -| Cmd | [command] | -| Working Dir | [workdir] | -| User | [user or root] | - -**Port Mappings:** -| Container Port | Host Binding | -|----------------|--------------| -| [8080/tcp] | [0.0.0.0:8080] | - -**Volume Mounts:** -| Source | Destination | Mode | -|--------|-------------|------| -| [/host/path] | [/container/path] | [rw/ro] | - -**Environment Variables:** -| Name | Value | -|------|-------| -| [VAR1] | [value] | -| [VAR2] | [value] | - -**Quick Assessment:** -[Based on state, provide initial assessment - e.g., "Container exited with code 1 - application error. OOMKilled=false, so not a memory issue."] - -Continue with container logs? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Step 3: Get Container Logs - -Use Podman MCP `container_logs`: - -```markdown -## Container Logs: [container-name] - -**Last 100 lines:** -``` -[container log output] -``` - -**Log Analysis:** - -[Analyze logs and identify errors:] - -**Errors Found:** -- [error 1 - e.g., "Error: Cannot find module 'express'"] -- [error 2 - e.g., "FATAL: password authentication failed for user 'app'"] -- [error 3 - e.g., "bind: address already in use"] - -**Error Categories:** -| Category | Count | First Occurrence | -|----------|-------|------------------| -| Module/Import | [X] | [line] | -| Connection | [Y] | [line] | -| Permission | [Z] | [line] | - -Continue to check image? (yes/no/skip) -``` - -Proceed automatically without waiting for user input. - -### Step 4: Analyze Image - -Use Podman MCP `image_list` to check the image: - -```markdown -## Image Analysis: [image:tag] - -**Image Info:** -| Field | Value | -|-------|-------| -| Repository | [repo] | -| Tag | [tag] | -| ID | [image-id] | -| Created | [timestamp] | -| Size | [size] | - -**Image Layers:** -[If available, show layer info] - -**Image Issues:** -- [Issue 1 - e.g., "Image is 2 years old - may have outdated dependencies"] -- [Issue 2 - e.g., "Using 'latest' tag - version not pinned"] - -**Entrypoint/CMD Check:** - -[Compare image defaults with container override] - -| Setting | Image Default | Container Override | -|---------|---------------|-------------------| -| Entrypoint | [image-entrypoint] | [container-entrypoint or "none"] | -| Cmd | [image-cmd] | [container-cmd or "none"] | - -**Potential Issues:** -- [Issue - e.g., "CMD is empty and no command provided at runtime"] -- [Issue - e.g., "Entrypoint is shell script but container run overrides it"] - -Continue to resource analysis? (yes/no/skip) -``` - -Proceed automatically without waiting for user input. - -### Step 5: Resource Analysis - -```markdown -## Resource Analysis - -**Container Resource Limits:** -| Resource | Limit | Status | -|----------|-------|--------| -| Memory | [512m or unlimited] | [OK/WARNING: OOMKilled] | -| CPU | [1.0 or unlimited] | [OK] | -| PIDs | [unlimited] | [OK] | - -**OOM Analysis:** - -[If OOMKilled=true:] -**Container was killed due to Out of Memory!** - -- Memory limit: [limit] -- Recommendation: Increase memory limit or optimize application - -```bash -# Run with more memory -podman run --memory=1g [image] -``` - -**Port Binding Analysis:** - -[Check if ports conflict:] - -| Port | Requested | Status | -|------|-----------|--------| -| [8080] | 0.0.0.0:8080 | [OK/ERROR: already in use] | - -[If port conflict:] -```bash -# Find process using port -lsof -i :[port] -# Or use different port -podman run -p 8081:8080 [image] -``` - -Continue to diagnosis summary? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Step 6: Present Diagnosis Summary - -```markdown -## Diagnosis Summary: [container-name] - -### Root Cause - -**Primary Issue:** [Categorized root cause] - -| Category | Status | Details | -|----------|--------|---------| -| Container State | [OK/FAIL] | [exit code, status] | -| Entrypoint/CMD | [OK/FAIL] | [details] | -| Dependencies | [OK/FAIL] | [missing modules] | -| Environment | [OK/FAIL] | [missing vars] | -| Volumes | [OK/FAIL] | [mount issues] | -| Ports | [OK/FAIL] | [binding issues] | -| Memory | [OK/FAIL] | [OOM status] | - -### Detailed Findings - -**[Category 1: e.g., Exit Code 1 - Application Error]** -- Problem: [specific problem - e.g., "Cannot find module 'express'"] -- Evidence: [from logs] -- Impact: [container exits immediately] - -**[Category 2: e.g., Volume Mount Issue]** -- Problem: [specific problem - e.g., "Permission denied on /data"] -- Evidence: [from logs] -- Impact: [application cannot access data] - -See [debugging-patterns.md](../../docs/debugging-patterns.md) for exit code reference. - -### Recommended Actions - -1. **[Action 1]** - [description] - ```bash - podman run [fixed-command] - ``` - -2. **[Action 2]** - [description] - ```bash - [command to fix - e.g., podman run --memory=1g ...] - ``` - -3. **[Action 3]** - [description] - -### Test Fix - -```bash -# Remove failed container -podman rm [container-name] - -# Run with fixes applied -podman run [corrected-options] [image] - -# Or run interactively to debug -podman run -it --entrypoint /bin/sh [image] -``` - ---- - -Would you like me to: -1. Execute one of the recommended fixes -2. Run container interactively for debugging -3. Inspect the image layers -4. Remove and recreate the container -5. Exit debugging - -Select an option: -``` - -Proceed automatically without waiting for user input. - -For exit codes, common container issues, and SELinux volume guidance, see [debugging-patterns.md](../../docs/debugging-patterns.md). - -## Dependencies - -### Required MCP Servers -- `podman` - container inspection, logs, and image analysis - -### Related Skills -- `/debug-rhel` - systemd service issues on RHEL hosts -- `/recommend-image` - select a better base image - -### Reference Documentation -- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common error patterns, exit codes -- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (podman) diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__debug-network/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/with_skills/rh-developer__debug-network/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__debug-network/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/with_skills/rh-developer__debug-network/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__debug-network/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/with_skills/rh-developer__debug-network/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__debug-network/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/with_skills/rh-developer__debug-network/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__debug-network/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/with_skills/rh-developer__debug-network/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__debug-network/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/with_skills/rh-developer__debug-network/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__debug-network/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/with_skills/rh-developer__debug-network/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__debug-network/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/with_skills/rh-developer__debug-network/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__debug-network/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/with_skills/rh-developer__debug-network/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__debug-network/environment/skills/debug-network/SKILL.md b/evaluation/with_skills/rh-developer__debug-network/environment/skills/debug-network/SKILL.md deleted file mode 100644 index c178ec74..00000000 --- a/evaluation/with_skills/rh-developer__debug-network/environment/skills/debug-network/SKILL.md +++ /dev/null @@ -1,331 +0,0 @@ ---- -name: debug-network -description: | - Diagnose OpenShift service connectivity issues including DNS resolution, service endpoints, route ingress, and network policies. Automates multi-step diagnosis: service endpoint verification, pod selector matching, route status, and network policy analysis. Use this skill when services can't communicate, routes return 503/502 errors, or external access fails. Triggers on /debug-network command or phrases like "can't reach service", "route returning 503", "pods can't communicate", "no endpoints". -model: inherit -color: cyan -metadata: - user_invocable: "true" ---- - -# /debug-network Skill - -Diagnose OpenShift service connectivity issues by automatically checking endpoints, routes, network policies, and pod readiness. - -## Prerequisites - -Before running this skill: -1. User is logged into OpenShift cluster -2. User has access to the target namespace -3. Service, Route, or application name is known - -## When to Use This Skill - -Use this skill when services cannot communicate, routes return 503/502 errors, or external access fails. It automates checking service endpoints, pod selector matching, route status, and network policy analysis to pinpoint connectivity issues. - -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -## Workflow - -### Step 1: Identify Target Service - -```markdown -## Network Debugging - -**Current OpenShift Context:** -- Cluster: [cluster] -- Namespace: [namespace] - -What connectivity issue would you like me to debug? - -1. **Service connectivity** - Internal service-to-service communication -2. **Route/Ingress** - External access to application -3. **Specify service name** - Debug a specific service -4. **List services** - Show services in current namespace - -Select an option or enter a service name: -``` - -Proceed automatically without waiting for user input. - -If user selects "List services": -Use kubernetes MCP `resources_list` for services: - -```markdown -## Services in [namespace] - -| Service | Type | Cluster IP | Ports | Endpoints | -|---------|------|------------|-------|-----------| -| [app-service] | ClusterIP | [ip] | [8080/TCP] | [2 ready] | -| [db-service] | ClusterIP | [ip] | [5432/TCP] | [0 - no endpoints!] | -| [api-service] | ClusterIP | [ip] | [3000/TCP] | [1 ready] | - -Which service would you like me to debug? -``` - -Proceed automatically without waiting for user input. - -### Step 2: Check Service and Endpoints - -Use kubernetes MCP `resources_get` for Service and Endpoints: - -```markdown -## Service Analysis: [service-name] - -**Service Configuration:** -| Field | Value | -|-------|-------| -| Type | [ClusterIP/NodePort/LoadBalancer] | -| Cluster IP | [ip] | -| Ports | [port-mappings] | -| Selector | [label-selector] | - -**Endpoints:** -| Subset | Addresses | Ports | Status | -|--------|-----------|-------|--------| -| [subset] | [pod-ip-1, pod-ip-2] | [port] | [Ready] | - -[If no endpoints:] -**WARNING: Service has NO endpoints!** - -This means no pods match the service selector, or matching pods are not ready. - -**Service Selector:** `app=[value], tier=[value]` - -**Quick Assessment:** -[Based on endpoints status, provide initial assessment] - -Continue with pod analysis? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Step 3: Verify Backend Pods - -Use kubernetes MCP `pod_list` with label selector matching service: - -```markdown -## Backend Pods for Service: [service-name] - -**Service Selector:** `[selector-labels]` - -**Matching Pods:** -| Pod | Status | Ready | IP | Node | -|-----|--------|-------|-----|------| -| [pod-1] | Running | 1/1 | [ip] | [node] | -| [pod-2] | Running | 0/1 | [ip] | [node] | -| [pod-3] | CrashLoopBackOff | 0/1 | [ip] | [node] | - -**Readiness Analysis:** -| Pod | Readiness Probe | Last Check | Status | -|-----|-----------------|------------|--------| -| [pod-1] | HTTP GET :8080/ | [time] | Passing | -| [pod-2] | HTTP GET :8080/ | [time] | Failing - Connection refused | -| [pod-3] | HTTP GET :8080/ | [time] | Failing - Container not running | - -[If selector mismatch:] -**WARNING: Label Mismatch Detected!** - -Service selector: `app=myapp` -Pod labels: `app=my-app` (hyphen difference!) - -**Issues Found:** -- [Issue 1 - e.g., "Pod [pod-2] failing readiness probe - application not listening on port 8080"] -- [Issue 2 - e.g., "Pod [pod-3] is in CrashLoopBackOff - run /debug-pod for details"] - -Continue to check Route? (yes/no/skip) -``` - -Proceed automatically without waiting for user input. - -### Step 4: Check Route Status - -Use kubernetes MCP `resources_get` for Route: - -```markdown -## Route Analysis: [route-name] - -**Route Configuration:** -| Field | Value | -|-------|-------| -| Host | [hostname] | -| Path | [path or "/"] | -| TLS Termination | [edge/passthrough/reencrypt/none] | -| Insecure Policy | [Redirect/Allow/None] | -| Target Service | [service-name] | -| Target Port | [port-name or port-number] | -| Weight | [100] | - -**Route Status:** -| Condition | Status | Reason | Message | -|-----------|--------|--------|---------| -| Admitted | [True/False] | [reason] | [message] | - -[If not admitted:] -**WARNING: Route NOT admitted by router!** - -**Ingress Status:** -| Router | Admitted | Host | Conditions | -|--------|----------|------|------------| -| [default] | [True/False] | [host] | [conditions] | - -**TLS Configuration:** -| Setting | Value | -|---------|-------| -| Certificate | [Provided/Default/None] | -| Key | [Provided/None] | -| CA Certificate | [Provided/None] | -| Destination CA | [Provided/None] (for reencrypt) | - -**Issues Found:** -- [Issue 1 - e.g., "Route not admitted - hostname conflicts with existing route"] -- [Issue 2 - e.g., "TLS termination is 'passthrough' but backend is HTTP only"] - -Continue to check Network Policies? (yes/no/skip) -``` - -Proceed automatically without waiting for user input. - -### Step 5: Analyze Network Policies - -Use kubernetes MCP `resources_list` for NetworkPolicy: - -```markdown -## Network Policy Analysis - -**NetworkPolicies in [namespace]:** -| Policy | Pod Selector | Ingress Rules | Egress Rules | -|--------|--------------|---------------|--------------| -| [policy-1] | app=myapp | [2 rules] | [Allow all] | -| [policy-2] | tier=backend | [1 rule] | [1 rule] | -| [default-deny] | {} (all pods) | [Deny all] | [Allow all] | - -**Policies Affecting [service-name] Pods:** - -**Policy: [policy-name]** -```yaml -ingress: -- from: - - podSelector: - matchLabels: - app: frontend - ports: - - port: 8080 - protocol: TCP -``` - -**Analysis:** -- Pods with `app=myapp` only accept traffic from pods with `app=frontend` -- Traffic from other namespaces is BLOCKED -- Traffic on ports other than 8080 is BLOCKED - -**Potential Blocking:** -- [Issue 1 - e.g., "Source pods have label 'app=web' but policy requires 'app=frontend'"] -- [Issue 2 - e.g., "Cross-namespace traffic blocked - no namespaceSelector in policy"] - -Continue to diagnosis summary? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Step 6: Present Diagnosis Summary - -```markdown -## Network Diagnosis Summary: [service-name] - -### Connectivity Path - -``` -[Source] → [Service] → [Endpoints] → [Pod] - OK → OK → [STATUS] → [STATUS] - -[External] → [Route] → [Router] → [Service] → [Pod] - OK → OK → OK → OK → [STATUS] -``` - -### Root Cause - -**Primary Issue:** [Categorized root cause] - -| Component | Status | Details | -|-----------|--------|---------| -| Service | [OK/FAIL] | [details] | -| Endpoints | [OK/FAIL] | [count] ready | -| Pod Readiness | [OK/FAIL] | [X/Y] pods ready | -| Route | [OK/FAIL] | [details] | -| Network Policy | [OK/WARN] | [details] | -| DNS | [OK/FAIL] | [details] | - -### Detailed Findings - -**[Category 1: e.g., No Endpoints]** -- Problem: [specific problem - e.g., "Service selector doesn't match any pods"] -- Evidence: [selector vs pod labels] -- Impact: [all traffic to service fails] - -**[Category 2: e.g., Readiness Probe Failing]** -- Problem: [specific problem] -- Evidence: [probe configuration and failure reason] -- Impact: [pod removed from endpoints] - -### Recommended Actions - -1. **[Action 1]** - [description] - ```bash - [command to fix - e.g., oc label pod myapp-xxx app=myapp --overwrite] - ``` - -2. **[Action 2]** - [description] - ```bash - [command to fix - e.g., oc patch svc myapp -p '{"spec":{"selector":{"app":"my-app"}}}'] - ``` - -3. **[Action 3]** - [description] - -### Test Connectivity - -After fixing, verify with: -```bash -# Test internal connectivity from another pod -oc run test-curl --rm -i --tty --image=curlimages/curl -- \ - curl -v http://[service-name].[namespace].svc.cluster.local:[port] - -# Test route externally -curl -v https://[route-host] - -# Check endpoints -oc get endpoints [service-name] -n [namespace] -``` - ---- - -Would you like me to: -1. Execute one of the recommended fixes -2. Test connectivity from a debug pod -3. Debug specific pods (/debug-pod) -4. Check DNS resolution -5. Exit debugging - -Select an option: -``` - -Proceed automatically without waiting for user input. - -## Common Connectivity Issues - -For detailed diagnosis and fix tables covering service, route, and network policy issues, see [docs/debugging-patterns.md](../../docs/debugging-patterns.md). - -## Dependencies - -### Required MCP Servers -- `openshift` - Kubernetes/OpenShift resource access for services, routes, endpoints, and network policies - -### Related Skills -- `/debug-pod` - To debug specific backend pods -- `/deploy` - To fix and redeploy the service - -### Reference Documentation -- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common error patterns -- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (oc), cluster access verification diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/with_skills/rh-developer__debug-pipeline/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__debug-pipeline/environment/skills/debug-pipeline/SKILL.md b/evaluation/with_skills/rh-developer__debug-pipeline/environment/skills/debug-pipeline/SKILL.md deleted file mode 100644 index 507936a7..00000000 --- a/evaluation/with_skills/rh-developer__debug-pipeline/environment/skills/debug-pipeline/SKILL.md +++ /dev/null @@ -1,306 +0,0 @@ ---- -name: debug-pipeline -description: | - Diagnose OpenShift Pipelines (Tekton) CI/CD failures including PipelineRun failures, TaskRun step errors, workspace/PVC binding issues, and authentication problems. Automates multi-step diagnosis: PipelineRun status, failed TaskRun analysis, step container logs, and related resource checks. Use this skill when pipelines fail, hang, or produce unexpected results. Triggers on /debug-pipeline command or phrases like "pipeline failed", "PipelineRun error", "TaskRun failed", "tekton error", "pipeline stuck", "pipeline timeout". -model: inherit -color: cyan -metadata: - user_invocable: "true" ---- - -# /debug-pipeline Skill - -Diagnose OpenShift Pipelines (Tekton) CI/CD failures by automatically gathering PipelineRun status, failed TaskRun details, step container logs, and related resources. - -## Prerequisites - -Before running this skill: -1. User is logged into OpenShift cluster -2. User has access to the target namespace -3. OpenShift Pipelines operator is installed on the cluster -4. PipelineRun name is known (or can be identified from recent runs) - -### Tekton CRD Access via MCP - -Tekton resources are standard Kubernetes CRDs. Use the generic MCP tools with these parameters: - -| Resource | kind | apiVersion | -|----------|------|------------| -| PipelineRun | `PipelineRun` | `tekton.dev/v1` | -| TaskRun | `TaskRun` | `tekton.dev/v1` | -| Pipeline | `Pipeline` | `tekton.dev/v1` | -| Task | `Task` | `tekton.dev/v1` | -| ClusterTask | `ClusterTask` | `tekton.dev/v1beta1` | -| EventListener | `EventListener` | `triggers.tekton.dev/v1beta1` | -| TriggerTemplate | `TriggerTemplate` | `triggers.tekton.dev/v1beta1` | -| TriggerBinding | `TriggerBinding` | `triggers.tekton.dev/v1beta1` | - -## When to Use This Skill - -Use this skill when OpenShift Pipelines (Tekton) fail, hang, or produce unexpected results. It diagnoses PipelineRun failures, TaskRun step errors, workspace/PVC binding issues, and authentication problems by analyzing run status, step container logs, and related resources. - -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -## Workflow - -### Step 1: Identify Target PipelineRun - -```markdown -## Pipeline Debugging - -**Current OpenShift Context:** -- Cluster: [cluster] -- Namespace: [namespace] - -Which PipelineRun would you like me to debug? - -1. **Specify PipelineRun name** - Enter the PipelineRun name directly -2. **List failed PipelineRuns** - Show recent failed PipelineRuns in current namespace -3. **From Pipeline** - Debug latest run of a specific Pipeline - -Select an option or enter a PipelineRun name: -``` - -Proceed automatically without waiting for user input. - -If user selects "List failed PipelineRuns": -Use kubernetes MCP `resources_list` with kind `PipelineRun`, filter by Failed status: - -```markdown -## Recent Failed PipelineRuns in [namespace] - -| PipelineRun | Pipeline | Status | Started | Duration | -|-------------|----------|--------|---------|----------| -| [run-name] | [pipeline] | Failed | [timestamp] | [duration] | - -Which PipelineRun would you like me to debug? -``` - -Proceed automatically without waiting for user input. - -### Step 2: Get PipelineRun Status Overview - -Use kubernetes MCP `resources_get` for the PipelineRun: - -```markdown -## PipelineRun Status: [pipelinerun-name] - -**PipelineRun Info:** -| Field | Value | -|-------|-------| -| Pipeline | [pipeline-name] | -| Status | [Succeeded/Failed/Running/Cancelled] | -| Started | [timestamp] | -| Completed | [timestamp or "Still running"] | -| Duration | [duration] | - -**Parameters:** -| Name | Value | -|------|-------| -| [param-name] | [param-value] | - -**TaskRun Status:** -| Task | TaskRun | Status | Duration | -|------|---------|--------|----------| -| [task-1] | [taskrun-1] | Succeeded | [duration] | -| [task-2] | [taskrun-2] | **Failed** | [duration] | -| [task-3] | [taskrun-3] | Skipped | - | - -**Quick Assessment:** -[Based on status conditions - e.g., "PipelineRun failed because TaskRun 'build' failed at step 'build-push'"] - -Continue with failed TaskRun analysis? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Step 3: Analyze Failed TaskRun(s) - -Use kubernetes MCP `resources_get` for each failed TaskRun: - -```markdown -## Failed TaskRun: [taskrun-name] - -**TaskRun Info:** -| Field | Value | -|-------|-------| -| Task | [task-name] | -| Pod | [taskrun-name]-pod | -| Status | [Failed] | -| Reason | [reason from conditions] | - -**Step Status:** -| Step | Container | Status | Exit Code | Reason | -|------|-----------|--------|-----------|--------| -| [step-1] | step-[step-1] | Completed | 0 | - | -| [step-2] | step-[step-2] | **Terminated** | [code] | [reason] | -| [step-3] | step-[step-3] | - | - | Skipped | - -**Workspace Bindings:** -| Workspace | Type | Resource | Status | -|-----------|------|----------|--------| -| [shared-workspace] | PVC | [pvc-name] | [Bound/Pending] | -| [output] | EmptyDir | - | OK | - -**Issues Found:** -- [Issue 1 - e.g., "Step 'build-push' failed with exit code 1"] - -Continue to view step logs? (yes/no) -``` - -**Note:** Tekton names step containers as `step-` in the TaskRun pod. Use this convention with `pod_logs`. - -Proceed automatically without waiting for user input. - -### Step 4: Get TaskRun Pod Logs - -Use kubernetes MCP `pod_logs` for the TaskRun pod, targeting the failed step container (`step-`): - -```markdown -## Step Logs: [step-name] (Pod: [taskrun-name]-pod) - -**Failed Step Container:** `step-[step-name]` - -``` -[log output from the failed step container] -``` - -**Log Analysis:** - -**Errors Found:** -- Line [X]: [error description] - -Continue to check related resources? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Step 5: Check Related Resources - -Check resources that could cause pipeline failures: - -```markdown -## Related Resources Analysis - -**ServiceAccount:** -| Field | Value | Status | -|-------|-------|--------| -| Name | [sa-name] | [OK] | -| Image Pull Secrets | [secrets] | [OK/MISSING] | -| Linked Secrets | [secrets] | [OK/MISSING] | - -**Workspaces/PVCs:** -| PVC | Status | Access Mode | Storage | -|-----|--------|-------------|---------| -| [pvc-name] | [Bound/Pending] | [RWO/RWX] | [size] | - -**Secrets:** -| Secret | Type | Referenced By | Status | -|--------|------|---------------|--------| -| [git-creds] | kubernetes.io/basic-auth | git-clone task | [OK/MISSING] | -| [registry-creds] | kubernetes.io/dockerconfigjson | push task | [OK/MISSING] | - -**Pipeline/Task Definitions:** -| Resource | Exists | Issues | -|----------|--------|--------| -| Pipeline [name] | [Yes/No] | [none / param mismatch] | -| Task [name] | [Yes/No] | [none / not found] | - -[If triggered by EventListener:] -**EventListener:** -| Field | Value | Status | -|-------|-------|--------| -| Name | [el-name] | [Running/NotRunning] | -| TriggerTemplate | [tt-name] | [OK/MISSING] | -| TriggerBinding | [tb-name] | [OK/MISSING] | - -**Issues Found:** -- [Issue 1] - -Continue to full diagnosis summary? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Step 6: Present Diagnosis Summary - -```markdown -## Diagnosis Summary: [pipelinerun-name] - -### Root Cause - -**Primary Issue:** [Categorized root cause] - -| Category | Status | Details | -|----------|--------|---------| -| Pipeline Definition | [OK/FAIL] | [details] | -| TaskRun Execution | [OK/FAIL] | [details] | -| Step Container | [OK/FAIL] | [details] | -| Workspace/PVC | [OK/FAIL] | [details] | -| Authentication | [OK/FAIL] | [details] | -| Resources/Quota | [OK/FAIL] | [details] | - -### Detailed Findings - -**[Category: e.g., Authentication]** -- Problem: [specific problem] -- Evidence: [from logs/events] -- Impact: [effect on pipeline] - -### Recommended Actions - -1. **[Action 1]** - [description] - ```bash - [command to fix] - ``` - -2. **[Action 2]** - [description] - ```bash - [command to fix] - ``` - -### Retry PipelineRun - -After fixing the issue: -```bash -# Rerun using the same PipelineRun spec -oc create -f <(oc get pipelinerun [name] -n [namespace] -o json | jq 'del(.metadata.resourceVersion, .metadata.uid, .metadata.creationTimestamp, .status) | .metadata.name = .metadata.name + "-retry"') -n [namespace] - -# Or using tkn CLI (if available) -tkn pipeline start [pipeline-name] --use-pipelinerun [pipelinerun-name] -n [namespace] -``` - ---- - -Would you like me to: -1. Execute one of the recommended fixes -2. Retry the PipelineRun -3. Debug the TaskRun pod directly (/debug-pod) -4. View Pipeline or Task definition -5. Exit debugging - -Select an option: -``` - -Proceed automatically without waiting for user input. - -## Pipeline Failure Reference - -For failure categories, error patterns, and troubleshooting decision trees, see [docs/debugging-patterns.md](../../docs/debugging-patterns.md) (sections: Pipeline/Tekton Failure Patterns, Common Tekton Error Messages). - -## Dependencies - -### Required MCP Servers -- `openshift` - Kubernetes/OpenShift resource access for PipelineRuns, TaskRuns, and Tekton CRDs - -### Related Skills -- `/debug-pod` - To debug TaskRun pods directly -- `/debug-build` - If the pipeline uses OpenShift Build tasks -- `/debug-network` - If pipeline tasks fail due to network issues -- `/validate-environment` - To verify OpenShift and pipeline operator setup - -### Reference Documentation -- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common error patterns and pipeline troubleshooting trees -- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (oc), cluster access verification diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__debug-pod/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/with_skills/rh-developer__debug-pod/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__debug-pod/environment/skills/debug-pod/SKILL.md b/evaluation/with_skills/rh-developer__debug-pod/environment/skills/debug-pod/SKILL.md deleted file mode 100644 index 1953df94..00000000 --- a/evaluation/with_skills/rh-developer__debug-pod/environment/skills/debug-pod/SKILL.md +++ /dev/null @@ -1,260 +0,0 @@ ---- -name: debug-pod -description: | - Diagnose pod failures on OpenShift including CrashLoopBackOff, ImagePullBackOff, OOMKilled, and pending pods. Automates multi-step diagnosis: pod status, events, logs (current + previous), and resource constraint analysis. Use this skill when pods are not running, restarting frequently, or stuck in non-ready states. Triggers on /debug-pod command or phrases like "my pod is crashing", "pod won't start", "CrashLoopBackOff", "ImagePullBackOff", "OOMKilled". -model: inherit -color: cyan -metadata: - user_invocable: "true" ---- - -# /debug-pod Skill - -Diagnose pod failures on OpenShift by automatically gathering status, events, logs, and resource information. - -## Prerequisites - -Before running this skill: -1. User is logged into OpenShift cluster -2. User has access to the target namespace -3. Pod or deployment name is known (or can be identified from recent deployments) - -## When to Use This Skill - -Use this skill when pods are not running, restarting frequently, or stuck in non-ready states such as CrashLoopBackOff, ImagePullBackOff, OOMKilled, or Pending. It automates gathering pod status, events, logs, and resource constraints to identify the root cause. - -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -## Workflow - -### Step 1: Identify Target Pod - -```markdown -## Pod Debugging - -**Current OpenShift Context:** -- Cluster: [cluster] -- Namespace: [namespace] - -Which pod would you like me to debug? - -1. **Specify pod name** - Enter the pod name directly -2. **List failing pods** - Show pods with issues in current namespace -3. **From deployment** - Debug pods from a specific deployment - -Select an option or enter a pod name: -``` - -Proceed automatically without waiting for user input. - -If user selects "List failing pods": -Use kubernetes MCP `pod_list` with namespace, then filter to show pods NOT in Running/Succeeded state: - -```markdown -## Pods with Issues in [namespace] - -| Pod | Status | Restarts | Age | Reason | -|-----|--------|----------|-----|--------| -| [pod-name] | CrashLoopBackOff | 5 | 10m | [waiting reason] | -| [pod-name-2] | ImagePullBackOff | 0 | 3m | [waiting reason] | -| [pod-name-3] | Pending | 0 | 15m | [conditions] | - -Which pod would you like me to debug? -``` - -Proceed automatically without waiting for user input. - -### Step 2: Get Pod Status Overview - -Use kubernetes MCP `resources_get` to get pod details: - -```markdown -## Pod Status: [pod-name] - -**Basic Info:** -| Field | Value | -|-------|-------| -| Namespace | [namespace] | -| Node | [node-name or "Not scheduled"] | -| Status | [phase: Pending/Running/Failed/Succeeded] | -| IP | [pod-ip or "Not assigned"] | -| Created | [timestamp] | - -**Container Status:** -| Container | State | Ready | Restarts | Exit Code | Reason | -|-----------|-------|-------|----------|-----------|--------| -| [container-name] | [Waiting/Running/Terminated] | [true/false] | [count] | [code or N/A] | [reason] | - -**Quick Assessment:** -[Based on status, provide initial assessment - e.g., "Pod is in CrashLoopBackOff - container keeps crashing after startup"] - -Continue with detailed analysis? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Step 3: Analyze Events - -Use kubernetes MCP `events_list` filtered by pod: - -```markdown -## Recent Events for [pod-name] - -| Time | Type | Reason | Message | -|------|------|--------|---------| -| [timestamp] | [Normal/Warning] | [reason] | [message] | -| [timestamp] | [Normal/Warning] | [reason] | [message] | -| ... | - -**Event Analysis:** - -[Analyze events and identify key issues:] - -**Issues Found:** -- [Issue 1 - e.g., "FailedScheduling: 0/3 nodes available - insufficient memory"] -- [Issue 2 - e.g., "ImagePullBackOff: unauthorized - check image pull secrets"] - -Continue to view container logs? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Step 4: Get Container Logs - -Use kubernetes MCP `pod_logs` for current and previous container: - -```markdown -## Container Logs: [container-name] - -**Current Container Logs** (last 50 lines): -``` -[log output] -``` - -[If container has restarted, also show previous logs:] - -**Previous Container Logs** (before last restart): -``` -[log output from --previous] -``` - -**Log Analysis:** - -[Analyze logs and identify errors:] - -**Errors Found:** -- Line [X]: [error description - e.g., "Connection refused to database on port 5432"] -- Line [Y]: [error description - e.g., "Out of memory - heap allocation failed"] - -Continue to analyze resource constraints? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Step 5: Analyze Resource Constraints - -Check resource requests, limits, and actual usage: - -```markdown -## Resource Analysis: [pod-name] - -**Container: [container-name]** - -| Resource | Request | Limit | Status | -|----------|---------|-------|--------| -| Memory | [128Mi] | [512Mi] | [OK / WARNING: OOMKilled] | -| CPU | [100m] | [500m] | [OK / WARNING: throttled] | - -**Node Resources (if scheduled):** -| Resource | Allocatable | Allocated | Available | -|----------|-------------|-----------|-----------| -| Memory | [8Gi] | [7.5Gi] | [512Mi] | -| CPU | [4000m] | [3800m] | [200m] | - -**Resource Issues:** -- [Issue 1 - e.g., "Container was OOMKilled - memory limit too low for application"] -- [Issue 2 - e.g., "Pod cannot be scheduled - no nodes have 2Gi available memory"] - -Continue to full diagnosis summary? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Step 6: Present Diagnosis Summary - -```markdown -## Diagnosis Summary: [pod-name] - -### Root Cause - -**Primary Issue:** [Categorized root cause] - -| Category | Status | Details | -|----------|--------|---------| -| Container Start | [OK/FAIL] | [details] | -| Image Pull | [OK/FAIL] | [details] | -| Resource Scheduling | [OK/FAIL] | [details] | -| Application Health | [OK/FAIL] | [details] | -| Volume Mounts | [OK/FAIL] | [details] | - -### Detailed Findings - -**[Category 1: e.g., Image Pull Issues]** -- Problem: [specific problem] -- Evidence: [from events/logs] -- Impact: [how this affects the pod] - -**[Category 2: e.g., Application Crash]** -- Problem: [specific problem] -- Evidence: [from logs] -- Impact: [how this affects the pod] - -### Recommended Actions - -1. **[Action 1]** - [description] - ```bash - [command to fix - e.g., oc create secret docker-registry...] - ``` - -2. **[Action 2]** - [description] - ```bash - [command to fix - e.g., oc set resources deployment/app --limits=memory=1Gi] - ``` - -3. **[Action 3]** - [description] - -### Related Documentation - -- [Link to relevant Red Hat KB article if applicable] -- [Link to OpenShift docs for the specific issue] - ---- - -Would you like me to: -1. Execute one of the recommended fixes -2. Dig deeper into a specific area -3. Debug a related resource (Service, Route, ConfigMap) -4. Exit debugging - -Select an option: -``` - -Proceed automatically without waiting for user input. - -For pod failure categories and exit code reference, see [debugging-patterns.md](../../docs/debugging-patterns.md). - -## Dependencies - -### Required MCP Servers -- `openshift` - Kubernetes/OpenShift resource access for pod status, events, and logs - -### Related Skills -- `/debug-build` - If pod failure is due to bad image from build -- `/debug-network` - If pod is running but service connectivity fails -- `/deploy` - To redeploy after fixing issues - -### Reference Documentation -- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common error patterns and troubleshooting trees -- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (oc), cluster access verification diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/with_skills/rh-developer__debug-rhel/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__debug-rhel/environment/skills/debug-rhel/SKILL.md b/evaluation/with_skills/rh-developer__debug-rhel/environment/skills/debug-rhel/SKILL.md deleted file mode 100644 index 4c3601ad..00000000 --- a/evaluation/with_skills/rh-developer__debug-rhel/environment/skills/debug-rhel/SKILL.md +++ /dev/null @@ -1,455 +0,0 @@ ---- -name: debug-rhel -description: | - Diagnose RHEL system issues including systemd service failures, SELinux denials, firewall blocking, and system resource problems. Automates multi-step diagnosis: journalctl log analysis, SELinux denial detection (ausearch), firewall rule inspection, and systemd unit status. Use this skill when applications fail on standalone RHEL/Fedora/CentOS hosts deployed via /rhel-deploy. Triggers on /debug-rhel command or phrases like "service won't start on RHEL", "SELinux blocking", "systemd failed", "firewall blocking". -model: inherit -color: cyan -metadata: - user_invocable: "true" ---- - -# /debug-rhel Skill - -Diagnose RHEL system issues by automatically gathering systemd status, journal logs, SELinux denials, and firewall configuration. - -## Overview - -``` -[Connect] → [Identify Service] → [systemd Status] → [Journal Logs] → [SELinux] → [Firewall] → [Summary] -``` - -**This skill diagnoses:** -- systemd service failures -- SELinux access denials (AVC) -- Firewall port blocking -- Permission issues -- Resource constraints - -## Prerequisites - -1. SSH access to target RHEL host -2. sudo privileges on the target host -3. RHEL 8+, CentOS Stream, Rocky Linux, or Fedora - -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -## Note: SSH/Bash Required - -This skill operates on **remote RHEL hosts** via SSH, not local MCP servers. Unlike OpenShift/Podman skills, direct Bash commands with SSH are the correct approach here since MCP servers run locally and cannot access remote systems. - -## When to Use This Skill - -Use `/debug-rhel` when applications fail on standalone RHEL, Fedora, or CentOS hosts. This skill automates multi-step diagnosis of systemd service failures, SELinux denials, firewall blocking, and system resource problems via SSH. - -## Workflow - -### Phase 1: SSH Connection - -```markdown -## RHEL System Debugging - -I'll help you diagnose issues on your RHEL system. - -**SSH Target:** -[If RHEL_HOST in session state from /rhel-deploy:] -- Using previous connection: [user]@[host] - -Is this correct? (yes/no/different host) - -[If no RHEL_HOST:] -Please provide your RHEL host details: - -| Setting | Value | Default | -|---------|-------|---------| -| Host | [required] | - | -| User | [current user] | $USER | -| Port | 22 | 22 | - -**Enter your SSH target:** -``` - -Proceed automatically without waiting for user input. - -**Connection verification:** - -```bash -# Test SSH connection -ssh -o BatchMode=yes -o ConnectTimeout=10 [user]@[host] "echo 'Connection successful'" -``` - -If connection fails: -```markdown -**SSH Connection Failed** - -Unable to connect to [host]. - -**Troubleshooting:** -1. Check host is reachable: `ping [host]` -2. Verify SSH key is configured: `ssh-add -l` -3. Check firewall allows SSH: port 22 -4. Verify username is correct - -Would you like to: -1. Try a different host -2. Get help with SSH setup -3. Exit -``` - -### Phase 2: Identify Target Service - -```markdown -## Phase 2: Identify Service - -Which service would you like me to debug? - -1. **Specify service name** - Enter the systemd unit name -2. **List failed services** - Show failed services on the host -3. **From /rhel-deploy** - Debug the last deployed service - -Select an option or enter a service name: -``` - -Proceed automatically without waiting for user input. - -If user selects "List failed services": - -```bash -# Get failed services -ssh [user]@[host] "systemctl --failed --no-pager" -``` - -```markdown -## Failed Services on [host] - -| Unit | Load | Active | Sub | Description | -|------|------|--------|-----|-------------| -| [myapp.service] | loaded | failed | failed | My Application | -| [other.service] | loaded | failed | failed | Other Service | - -Which service would you like me to debug? -``` - -Proceed automatically without waiting for user input. - -### Phase 3: Get Service Status - -```bash -# Get detailed service status -ssh [user]@[host] "systemctl status [service] --no-pager -l" -``` - -```markdown -## Service Status: [service-name] - -**Status Overview:** -| Field | Value | -|-------|-------| -| Loaded | [loaded/not-found/masked] | -| Active | [active (running)/inactive (dead)/failed] | -| Main PID | [pid or N/A] | -| Status | [status text] | -| Since | [timestamp] | - -**Recent Activity:** -``` -[systemctl status output - last 10 lines] -``` - -**Quick Assessment:** -[Based on status, provide initial assessment - e.g., "Service failed to start - exit code 1 suggests application error"] - -Continue with journal logs? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Phase 4: Analyze Journal Logs - -```bash -# Get service logs -ssh [user]@[host] "journalctl -u [service] -n 100 --no-pager" -``` - -```markdown -## Journal Logs: [service-name] - -**Last 100 log entries:** -``` -[journalctl output] -``` - -**Log Analysis:** - -[Analyze logs and identify errors:] - -**Errors Found:** -- [timestamp]: [error - e.g., "Permission denied: /var/data/config.yaml"] -- [timestamp]: [error - e.g., "Connection refused: localhost:5432"] -- [timestamp]: [error - e.g., "Port 8080 already in use"] - -**Error Categories:** -| Category | Count | Example | -|----------|-------|---------| -| Permission | [X] | [first occurrence] | -| Connection | [Y] | [first occurrence] | -| Resource | [Z] | [first occurrence] | - -Continue to check SELinux? (yes/no/skip) -``` - -Proceed automatically without waiting for user input. - -### Phase 5: Check SELinux Denials - -```bash -# Check SELinux status -ssh [user]@[host] "getenforce" - -# Get recent AVC denials -ssh [user]@[host] "sudo ausearch -m AVC -ts recent 2>/dev/null || echo 'No recent denials or ausearch not available'" -``` - -```markdown -## SELinux Analysis - -**SELinux Status:** [Enforcing/Permissive/Disabled] - -**Recent AVC Denials:** - -[If denials found:] -| Time | Source | Target | Permission | Denied | -|------|--------|--------|------------|--------| -| [time] | [source_context] | [target_context] | [permission] | [target_file] | -| [time] | [source_context] | [target_context] | [permission] | [target_port] | - -**Denial Analysis:** - -**Denial 1: [description]** -- **What happened:** Process `[name]` tried to [action] on `[target]` -- **Why denied:** SELinux type `[source_type]` cannot [action] `[target_type]` -- **Impact:** [how this affects the application] - -**Recommended Fixes:** - -1. **Set SELinux boolean** (if applicable): - ```bash - sudo setsebool -P [boolean_name] on - ``` - -2. **Change file context** (if file access): - ```bash - sudo semanage fcontext -a -t [correct_type] "[path](/.*)?" - sudo restorecon -Rv [path] - ``` - -3. **Allow port** (if port binding): - ```bash - sudo semanage port -a -t [port_type] -p tcp [port] - ``` - -[If no denials:] -No recent SELinux denials found. SELinux is likely not the issue. - -Continue to check firewall? (yes/no/skip) -``` - -Proceed automatically without waiting for user input. - -### Phase 6: Check Firewall - -```bash -# Get firewall status -ssh [user]@[host] "sudo firewall-cmd --state 2>/dev/null || echo 'firewalld not running'" - -# List firewall rules -ssh [user]@[host] "sudo firewall-cmd --list-all 2>/dev/null" -``` - -```markdown -## Firewall Analysis - -**Firewall Status:** [running/not running] - -**Active Zone:** [zone-name] - -**Current Rules:** -| Type | Value | -|------|-------| -| Services | [ssh, http, https, ...] | -| Ports | [8080/tcp, 3000/tcp, ...] | -| Rich Rules | [count] | - -**Application Port:** [detected-port from logs/config] - -**Port Status:** -| Port | Protocol | Status | -|------|----------|--------| -| [8080] | TCP | [OPEN/BLOCKED] | -| [443] | TCP | [OPEN/BLOCKED] | - -[If port blocked:] -**WARNING: Application port [port] is NOT open in firewall!** - -**To open port:** -```bash -sudo firewall-cmd --permanent --add-port=[port]/tcp -sudo firewall-cmd --reload -``` - -**Or add service:** -```bash -sudo firewall-cmd --permanent --add-service=[service] -sudo firewall-cmd --reload -``` - -Continue to diagnosis summary? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Phase 7: Red Hat Insights Check (Optional) - -**This phase runs only if the `lightspeed-mcp` server is available.** Use `ToolSearch` to check for Lightspeed MCP tools. If not available, skip this phase silently and proceed to Phase 8. - -**Step 1:** Use `find_host_by_name` with the hostname from `RHEL_HOST` to look up the system in Red Hat Insights. - -**Step 2:** If system found, use `get_system_cves` with the system ID to check for known CVEs affecting this system. - -**Step 3:** Use `get_active_rules` to get advisor configuration recommendations. Optionally use `get_rule_by_text_search` with error text found in Phase 4 logs to find relevant advisor recommendations. - -```markdown -## Red Hat Insights Check - -**System in Insights:** [Found / Not registered] - -[If found:] -**System Details:** -| Field | Value | -|-------|-------| -| Display Name | [hostname] | -| RHEL Version | [version] | -| Last Check-in | [timestamp] | -| Stale | [yes/no] | - -**Known Vulnerabilities:** -| CVE | CVSS | Severity | Remediation | -|-----|------|----------|-------------| -| [CVE-ID] | [score] | [severity] | [Available/None] | - -**Advisor Recommendations:** -| Rule | Category | Risk | Description | -|------|----------|------|-------------| -| [rule-id] | [Security/Performance/Availability/Stability] | [Critical/Important/Moderate/Low] | [description] | - -[If any CVE or advisor rule matches the symptoms from earlier phases:] -**Potentially Related to Current Issue:** -- [CVE or advisor rule that matches the symptoms] - -Continue to diagnosis summary? (yes/no) -``` - -Proceed automatically without waiting for user input. - -[If system not registered in Insights, just note it:] -```markdown -## Red Hat Insights Check - -System [hostname] is not registered in Red Hat Insights. Skipping vulnerability and advisor checks. - -Continue to diagnosis summary? (yes/no) -``` - -### Phase 8: Present Diagnosis Summary - -```markdown -## Diagnosis Summary: [service-name] on [host] - -### Root Cause - -**Primary Issue:** [Categorized root cause] - -| Category | Status | Details | -|----------|--------|---------| -| Service Unit | [OK/FAIL] | [loaded/enabled status] | -| Application | [OK/FAIL] | [exit code, error] | -| SELinux | [OK/BLOCKED] | [denial count] | -| Firewall | [OK/BLOCKED] | [port status] | -| Permissions | [OK/FAIL] | [file/dir issues] | -| Resources | [OK/FAIL] | [memory/cpu/disk] | -| Insights/CVE | [OK/WARN/N/A] | [CVE count or "Not registered"] | - -### Detailed Findings - -**[Category 1: e.g., SELinux Denial]** -- Problem: [specific problem - e.g., "httpd_t cannot bind to port 8080"] -- Evidence: [AVC denial message] -- Impact: [application cannot start] - -**[Category 2: e.g., Missing Dependency]** -- Problem: [specific problem - e.g., "libpq.so.5 not found"] -- Evidence: [error from logs] -- Impact: [application crashes on startup] - -### Recommended Actions - -1. **[Action 1 - Highest Priority]** - [description] - ```bash - ssh [user]@[host] "[command]" - ``` - -2. **[Action 2]** - [description] - ```bash - ssh [user]@[host] "[command]" - ``` - -3. **[Action 3]** - [description] - ```bash - ssh [user]@[host] "[command]" - ``` - -### Verify Fix - -After applying fixes: -```bash -# Restart service -ssh [user]@[host] "sudo systemctl restart [service]" - -# Check status -ssh [user]@[host] "systemctl status [service]" - -# View logs -ssh [user]@[host] "journalctl -u [service] -f" -``` - ---- - -Would you like me to: -1. Execute one of the recommended fixes -2. Dig deeper into a specific area -3. Restart the service -4. View live logs -5. Exit debugging - -Select an option: -``` - -Proceed automatically without waiting for user input. - -For common RHEL issues (systemd exit codes, SELinux denials, firewall), see [debugging-patterns.md](../../docs/debugging-patterns.md) and [selinux-troubleshooting.md](../../docs/selinux-troubleshooting.md). - -## Dependencies - -### Required MCP Servers -- `lightspeed-mcp` (optional) - Red Hat Insights CVE and advisor checks in Phase 7 - -### Related Skills -- `/rhel-deploy` - redeploy after fixing issues -- `/debug-container` - debug Podman containers on the host - -### Reference Documentation -- [docs/selinux-troubleshooting.md](../../docs/selinux-troubleshooting.md) - SELinux denial analysis -- [docs/rhel-deployment.md](../../docs/rhel-deployment.md) - RHEL deployment patterns -- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common error patterns -- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools and setup diff --git a/evaluation/with_skills/rh-developer__deploy/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__deploy/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/with_skills/rh-developer__deploy/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__deploy/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__deploy/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/with_skills/rh-developer__deploy/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/with_skills/rh-developer__deploy/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__deploy/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/with_skills/rh-developer__deploy/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/with_skills/rh-developer__deploy/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__deploy/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/with_skills/rh-developer__deploy/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__deploy/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__deploy/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/with_skills/rh-developer__deploy/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/with_skills/rh-developer__deploy/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__deploy/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/with_skills/rh-developer__deploy/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__deploy/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__deploy/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/with_skills/rh-developer__deploy/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__deploy/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__deploy/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/with_skills/rh-developer__deploy/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/with_skills/rh-developer__deploy/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__deploy/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/with_skills/rh-developer__deploy/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__deploy/environment/skills/deploy/SKILL.md b/evaluation/with_skills/rh-developer__deploy/environment/skills/deploy/SKILL.md deleted file mode 100644 index b24e7d12..00000000 --- a/evaluation/with_skills/rh-developer__deploy/environment/skills/deploy/SKILL.md +++ /dev/null @@ -1,277 +0,0 @@ ---- -name: deploy -description: | - Create Kubernetes Deployment, Service, and Route resources on OpenShift to deploy and expose an application. Use this skill after /s2i-build to make the built image accessible. Handles port detection, replica configuration, HTTPS route creation, rollout monitoring, and rollback on failure. Triggers on /deploy command when user wants to deploy a container image to OpenShift. -model: inherit -color: green -metadata: - user_invocable: "true" ---- - -# /deploy Skill - -Create Kubernetes/OpenShift resources (Deployment, Service, Route) to deploy and expose an application from a container image. - -## Prerequisites - -Before running this skill: -1. User is logged into OpenShift cluster -2. Container image exists (from ImageStream or external registry) -3. Target namespace exists - -## When to Use This Skill - -Use `/deploy` after building a container image (via `/s2i-build` or external registry) to create Deployment, Service, and Route resources on OpenShift. This skill handles port detection, replica configuration, rollout monitoring, and rollback on failure. - -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -## Workflow - -### Step 1: Gather Deployment Information - -```markdown -## Deployment Configuration - -**Current OpenShift Context:** -- Cluster: [cluster] -- Namespace: [namespace] - -**Please confirm deployment settings:** - -| Setting | Value | Source | -|---------|-------|--------| -| App Name | `[name]` | [from s2i-build / input] | -| Image | `[image-ref]` | [from ImageStream / input] | -| Container Port | `[port]` | [detected / needs input] | -| Replicas | `1` | [default] | -| Expose Route | `yes` | [default] | - -Confirm these settings or tell me what to change. -``` - -Proceed automatically without waiting for user input. - -### Step 2: Detect Container Port - -Try to detect port from project files: - -1. **Dockerfile:** Look for `EXPOSE ` (Most accurate for container builds) -2. **Web Server Config:** Look for `listen ` in `nginx.conf`, `httpd.conf`, etc. -3. **Framework Defaults:** - - **Node.js:** Look for `PORT` env var usage, common: 3000 (dev), 8080 (prod/S2I) - - **Python:** Flask default 5000, FastAPI/Uvicorn 8000 - - **Java:** Spring Boot 8080, Quarkus 8080 - - **Go:** Common 8080 - - **Ruby Rails:** 3000 - -```markdown -## Port Detection - -I detected port **[port]** based on: -- [reason - e.g., "PORT environment variable in package.json scripts"] - -Is this correct? -- yes - Use port [port] -- no - Specify the correct port -``` - -Proceed automatically without waiting for user input. - -If unable to detect: -```markdown -## Port Required - -I couldn't automatically detect the container port. - -Common ports by framework: -- Node.js/Express: 3000 or 8080 -- Python Flask: 5000 -- Python FastAPI: 8000 -- Java Spring Boot: 8080 -- Go: 8080 - -**What port does your application listen on?** -``` - -### Step 3: Create Deployment - -Show the Deployment manifest: - -```markdown -## Step 1 of 3: Create Deployment - -Read `templates/deployment.yaml.template` and substitute `${APP_NAME}`, `${NAMESPACE}`, `${PORT}`, `${REPLICAS}` with session state values. - -Show the rendered YAML to user and confirm. - -**Proceed with creating this Deployment?** (yes/no) -``` - -Proceed automatically without waiting for user input. - -- If user says "yes" → Use kubernetes MCP `resources_create_or_update` to apply -- If user says "no" → Ask what they would like to change -- If user wants modifications → Update the YAML and show again for confirmation - -### Step 4: Create Service - -```markdown -## Step 2 of 3: Create Service - -Read `templates/service.yaml.template` and substitute `${APP_NAME}`, `${NAMESPACE}`, `${PORT}`. - -Show the rendered YAML to user and confirm. - -**Proceed with creating this Service?** (yes/no) -``` - -Proceed automatically without waiting for user input. - -- If user says "yes" → Use kubernetes MCP `resources_create_or_update` to apply -- If user says "no" → Ask what they would like to change -- If user wants modifications → Update the YAML and show again for confirmation - -### Step 5: Create Route (Optional) - -If user wants external exposure: - -```markdown -## Step 3 of 3: Create Route - -Read `templates/route.yaml.template` and substitute `${APP_NAME}`, `${NAMESPACE}`. - -Show the rendered YAML to user and confirm. - -**Proceed with creating this Route?** (yes/no/skip) -``` - -Proceed automatically without waiting for user input. - -- If user says "yes" → Use kubernetes MCP `resources_create_or_update` to apply -- If user says "skip" → Skip Route creation and proceed to rollout monitoring -- If user says "no" → Ask what they would like to change -- If user wants modifications → Update the YAML and show again for confirmation - -### Step 6: Wait for Rollout - -Monitor deployment status: - -```markdown -## Deployment Rollout - -Waiting for pods to be ready... - -**Deployment:** [app-name] -**Desired:** [replicas] -**Ready:** [current]/[replicas] - -**Pod Status:** -| Pod | Status | Ready | Restarts | -|-----|--------|-------|----------| -| [app-name]-xxx-yyy | Running | 1/1 | 0 | - -[Poll until ready or timeout after 5 minutes] -``` - -### Step 6a: Handle Deployment Failure - -If pods do not become ready within the timeout period, or pods are in error states (CrashLoopBackOff, ImagePullBackOff, Pending): - -```markdown -## Deployment Failed - -**Status:** Rollout did not complete successfully - -**Pod Status:** -| Pod | Status | Ready | Restarts | Reason | -|-----|--------|-------|----------|--------| -| [app-name]-xxx-yyy | [CrashLoopBackOff/ImagePullBackOff/Pending] | 0/1 | [count] | [reason] | - -**Events:** -| Time | Type | Message | -|------|------|---------| -| [time] | Warning | [event message] | - ---- - -**Would you like me to diagnose the issue?** - -1. **Debug Pod** - Investigate pod failures (runs `/debug-pod`) - - Analyzes pod status, events, logs, and resource constraints - - Identifies root cause (OOM, image pull issues, crashes, etc.) - -2. **Debug Network** - Investigate connectivity issues (runs `/debug-network`) - - Checks service endpoints, route status, network policies - - Useful if pods are running but service is unreachable - -3. **View logs manually** - Show pod logs without full diagnosis - -4. **Rollback deployment** - Delete created resources and stop - -5. **Continue waiting** - Wait another 5 minutes for rollout - -Select an option: -``` - -Proceed automatically without waiting for user input. - -- If user selects "Debug Pod" → Invoke `/debug-pod` skill with pod name -- If user selects "Debug Network" → Invoke `/debug-network` skill with service name -- If user selects "View logs" → Show pod logs using `pod_logs` -- If user selects "Rollback" → Delete Deployment, Service, Route -- If user selects "Continue" → Wait another polling cycle - -### Step 7: Deployment Complete - -```markdown -## Deployment Complete! - -**Application:** [app-name] -**Namespace:** [namespace] - -**Access URLs:** -| Type | URL | -|------|-----| -| External | https://[route-host] | -| Internal | http://[app-name].[namespace].svc.cluster.local:[port] | - -**Resources Created:** -| Resource | Name | Status | -|----------|------|--------| -| Deployment | [app-name] | [replicas]/[replicas] Ready | -| Service | [app-name] | Active | -| Route | [app-name] | Admitted | - -**Quick Commands:** -```bash -# View logs -oc logs -f deployment/[app-name] -n [namespace] - -# Scale replicas -oc scale deployment/[app-name] --replicas=3 -n [namespace] - -# Restart pods -oc rollout restart deployment/[app-name] -n [namespace] - -# Delete all -oc delete all -l app=[app-name] -n [namespace] -``` - -Your application is now live! -``` - -## Dependencies - -### Required MCP Servers -- `openshift` - cluster resource creation and management - -### Related Skills -- `/debug-pod` - Pod failures (CrashLoopBackOff, OOMKilled, ImagePullBackOff) -- `/debug-network` - Service connectivity issues (no endpoints, 503 errors) -- `/debug-build` - Build failures before deployment - -### Reference Documentation -- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (oc), cluster access verification -- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common error patterns and troubleshooting diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__detect-project/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/with_skills/rh-developer__detect-project/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__detect-project/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/with_skills/rh-developer__detect-project/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__detect-project/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/with_skills/rh-developer__detect-project/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__detect-project/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/with_skills/rh-developer__detect-project/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__detect-project/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/with_skills/rh-developer__detect-project/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__detect-project/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/with_skills/rh-developer__detect-project/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__detect-project/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/with_skills/rh-developer__detect-project/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__detect-project/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/with_skills/rh-developer__detect-project/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__detect-project/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/with_skills/rh-developer__detect-project/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__detect-project/environment/skills/detect-project/SKILL.md b/evaluation/with_skills/rh-developer__detect-project/environment/skills/detect-project/SKILL.md deleted file mode 100644 index 2f6d126e..00000000 --- a/evaluation/with_skills/rh-developer__detect-project/environment/skills/detect-project/SKILL.md +++ /dev/null @@ -1,277 +0,0 @@ ---- -name: detect-project -description: | - Analyze a project folder or GitHub repository to detect programming language, framework, and version requirements. Use this skill when containerizing an application, selecting an S2I builder image, deploying to OpenShift or RHEL, or determining a project's tech stack. Supports Node.js, Python, Java, Go, Ruby, .NET, PHP, and Perl. Triggers on /detect-project command or when user needs build strategy recommendations. Run before /s2i-build or /rhel-deploy. -model: inherit -color: cyan -metadata: - user_invocable: "true" ---- - -# /detect-project Skill - -## Critical Restrictions -- **DO NOT CLONE** remote repositories unless the user explicitly selects the "Clone & Inspect" option. -- **ALWAYS** use `github-mcp-server` tools (`list_directory`, `get_file_contents`) for initial analysis of remote URLs. -- **NEVER** assume you have permission to write to the local filesystem for analysis purposes. - -Analyze the project to detect language/framework and recommend a build strategy. This skill handles both local project directories and remote Git repositories. - -## When to Use This Skill - -- User wants to containerize or deploy an application and needs language/framework detection -- User asks what tech stack a project uses or needs a build strategy recommendation -- Run before `/s2i-build`, `/recommend-image`, or `/rhel-deploy` to identify project type - -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -## Workflow - -### Step 1: Context Analysis - -**Scenario A: Local Files Available** -If you are in a project directory with source code: -1. Proceed to **Step 2: Scan Project Files**. - -**Scenario B: Remote Git URL Provided** -If the user provided a Git URL (e.g., `https://github.com/...`): - -Use the **github-mcp-server** to analyze the repository directly without cloning: - -1. Use `mcp_github_get_file_contents(owner, repo, path="/")` to list repository contents -2. Read key files using `fetch_mcp_resource` with URI format: `repo://{owner}/{repo}/contents/{file-path}` - - Example: `repo://myorg/myrepo/contents/package.json` -3. Proceed with analysis as if local files - -```markdown -## Analyzing Remote Repository - -I'm inspecting the repository: `[git-url]` - -Using GitHub API to analyze the project structure... - -[Use github MCP to get_file_contents for indicator files] - -**Files Found:** -- [list files from repo root] - -[Continue with Step 2: Scan Project Files using the remote file contents] -``` - -If GitHub MCP is unavailable or repo is private without access: - -```markdown -## Remote Repository Access - -I see you want to deploy from: `[git-url]` - -I couldn't access the repository directly. Options: - -1. **Remote S2I Build** (Recommended for standard apps) - - OpenShift will clone and build the code directly. - - I need you to confirm the language/framework. - -2. **Remote Podman Build** (Recommended if Containerfile/Dockerfile exists) - - OpenShift will use the Containerfile/Dockerfile in the repo. - - Best if you already have a custom build process. - -3. **Clone & Inspect** - - I will clone the repo locally to analyze it first. - - This helps if you're unsure about the project details. - -**Which approach do you prefer?** -``` - -Proceed automatically without waiting for user input. - -**Scenario C: No Context** -If no files and no URL: -1. Ask the user for the Git URL or to navigate to a project folder. - -### Step 2: Scan Project Files (Local Only) - -Look for these indicator files in the project root: - -| File | Language | Framework Hint | -|---|----|----| -| `Chart.yaml` | Helm Chart | Existing Helm deployment available | -| `package.json` | Node.js | Check for next, angular, vue, react | -| `pom.xml` | Java | Check for spring-boot, quarkus deps | -| `build.gradle` / `build.gradle.kts` | Java | Check for spring, quarkus plugins | -| `requirements.txt` | Python | - | -| `Pipfile` | Python | Pipenv | -| `pyproject.toml` | Python | Poetry or modern Python | -| `go.mod` | Go | - | -| `Gemfile` | Ruby | Check for rails | -| `composer.json` | PHP | Check for laravel, symfony | -| `*.csproj` / `*.sln` | .NET | - | -| `Cargo.toml` | Rust | No official S2I | -| `Dockerfile` / `Containerfile` | Pre-containerized | May not need S2I | - -### Helm Chart Detection - -Also check for Helm charts in these locations (in order): - -| Priority | Path | Description | -|----------|------|-------------| -| 1 | `./Chart.yaml` | Root directory | -| 2 | `./chart/Chart.yaml` | Chart subdirectory | -| 3 | `./charts/*/Chart.yaml` | Charts directory | -| 4 | `./helm/Chart.yaml` | Helm subdirectory | -| 5 | `./deploy/helm/Chart.yaml` | Deploy directory | - -If Chart.yaml is found, parse it to extract: -- `name`: Chart name -- `version`: Chart version (SemVer) -- `appVersion`: Application version -- `description`: Chart description - -Also check for: -- `values.yaml`: Default configuration -- `templates/`: Template files - -### Step 3: Detect Version Requirements - -For each detected language, extract version info: - -**Node.js:** -- Check `engines.node` in package.json -- Example: `"engines": { "node": ">=18" }` - -**Python:** -- Check `python_requires` in pyproject.toml -- Check `runtime.txt` for version -- Check `.python-version` file - -**Java:** -- Check `` or `` in pom.xml -- Check `sourceCompatibility` in build.gradle - -**Go:** -- Check `go` directive in go.mod -- Example: `go 1.21` - -### Step 4: Detect Framework - -Look for framework-specific indicators: - -**Node.js frameworks:** -- `next.config.js` or `next.config.mjs` → Next.js -- `angular.json` → Angular -- `vue.config.js` or `vite.config.ts` with vue → Vue.js -- `remix.config.js` → Remix - -**Java frameworks:** -- `quarkus` in dependencies → Quarkus -- `spring-boot` in dependencies → Spring Boot -- `micronaut` in dependencies → Micronaut - -**Python frameworks:** -- `django` in requirements → Django -- `flask` in requirements → Flask -- `fastapi` in requirements → FastAPI - -### Step 4.5: Detect Python Entry Point (Python projects only) - -For Python projects, detect the application entry point to ensure proper S2I configuration: - -**Check for entry point files (in order of S2I preference):** -1. `app.py` - Default S2I Python entry point (no config needed) -2. `application.py` - Alternative default -3. `wsgi.py` - WSGI module -4. `main.py` - Common alternative (requires APP_MODULE config) -5. Any file with `if __name__ == "__main__"` and Flask/FastAPI app - -**Check requirements.txt/Pipfile/pyproject.toml for WSGI server:** -- `gunicorn` - Required for APP_MODULE to work with S2I Python -- `uwsgi` - Alternative WSGI server - -### Step 5: Present Findings - -Format your response: - -```markdown -## Project Analysis Results - -**Detected Language:** [Language] -**Framework:** [Framework or "None detected"] -**Version:** [Version or "Not specified"] - -**Detection Confidence:** [High/Medium/Low] -- High: Clear indicator file with version info -- Medium: Indicator file found but no version specified -- Low: Multiple conflicting indicators or unusual setup - -**Indicator Files Found:** -- [list of files] - ---- - -**Recommended S2I Builder Image:** -`registry.access.redhat.com/ubi9/[image-name]` - -**Why this image:** -- [Brief explanation] - -**Alternative Options:** -1. `[alternative-1]` - [when to choose] -2. `[alternative-2]` - [when to choose] - ---- - -**Suggested App Name:** `[derived-name]` -(based on [folder name / package.json name / pom artifactId]) - ---- - -**Image Selection Options:** -- **quick** - Use the recommended image above (good for most cases) -- **smart** - Run `/recommend-image` for use-case aware selection (production vs dev, security, performance) - -Please confirm: -1. Is the detected language/framework correct? -2. Image selection: quick or smart? -3. Is the app name acceptable? - -Type 'yes' to confirm all with quick image selection, 'smart' for tailored recommendation, or tell me what to change. -``` - -Proceed automatically without waiting for user input. - -- If user says "yes" → Save configuration with quick image selection -- If user says "smart" → Invoke `/recommend-image` skill -- If user provides corrections → Update values and show again for confirmation - -**Note:** If the user selects "smart", invoke the `/recommend-image` skill with the detected `LANGUAGE`, `FRAMEWORK`, and `VERSION` values. - -## Output Variables - -After successful detection, these values should be available for other skills: - -| Variable | Description | Example | -|----|----|---| -| `APP_NAME` | Application name | `my-nodejs-app` | -| `LANGUAGE` | Detected language | `nodejs` | -| `FRAMEWORK` | Detected framework | `express` | -| `VERSION` | Language version | `20` | -| `BUILDER_IMAGE` | Full S2I image reference | `registry.access.redhat.com/ubi9/nodejs-20` | -| `BUILD_STRATEGY` | Build strategy | `Source` (S2I) or `Podman` | -| `CONTAINER_PORT` | Application listen port | `8080` | -| `HELM_CHART_PATH` | Path to Helm chart | `./chart` | - -## Dependencies - -### Required MCP Servers -- `github` - Remote repository analysis via GitHub API (for URL-based detection) - -### Related Skills -- `/s2i-build` - Build with the detected S2I builder image -- `/recommend-image` - Advanced image selection based on detection results -- `/rhel-deploy` - Deploy to RHEL using detected project info - -### Reference Documentation -- [docs/builder-images.md](../../docs/builder-images.md) - Language detection matrix, version-to-image mapping, S2I builder selection -- [docs/python-s2i-entrypoints.md](../../docs/python-s2i-entrypoints.md) - Python entry point detection, APP_MODULE configuration -- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (git) diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/with_skills/rh-developer__helm-deploy/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__helm-deploy/environment/skills/helm-deploy/SKILL.md b/evaluation/with_skills/rh-developer__helm-deploy/environment/skills/helm-deploy/SKILL.md deleted file mode 100644 index a44ce170..00000000 --- a/evaluation/with_skills/rh-developer__helm-deploy/environment/skills/helm-deploy/SKILL.md +++ /dev/null @@ -1,356 +0,0 @@ ---- -name: helm-deploy -description: | - Deploy applications to OpenShift using Helm charts. Use this skill when user wants to deploy with Helm, when a Helm chart is detected in the project, or when /helm-deploy command is invoked. Supports both existing charts and chart creation. Handles chart detection, values customization, install/upgrade operations, and rollback. Requires kubernetes MCP Helm tools. -model: inherit -color: green -metadata: - user_invocable: "true" ---- - -# /helm-deploy Skill - -Deploy applications to OpenShift using Helm charts. Supports existing charts or creates new ones. - -## Prerequisites - -1. User logged into OpenShift cluster -2. Helm chart exists OR user wants to create one -3. Container image available (from registry or will be built) - -## When to Use This Skill - -- User wants to deploy an application using Helm charts on OpenShift -- A Helm chart is detected in the project (Chart.yaml found) -- User invokes `/helm-deploy` or asks about Helm-based deployment - -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -## Workflow - -### Step 1: Check OpenShift Connection - -Use kubernetes MCP to verify cluster connection: - -```markdown -## Checking OpenShift Connection... - -**Cluster:** [cluster-url] -**User:** [username] -**Namespace:** [namespace] - -Is this the correct cluster and namespace? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Step 2: Detect Helm Chart - -Search for Helm charts using the same priority order as `/detect-project`: -- `./Chart.yaml`, `./chart/Chart.yaml`, `./charts/*/Chart.yaml`, `./helm/Chart.yaml`, `./deploy/helm/Chart.yaml` - -> **Note:** If `/detect-project` was already run, use the `HELM_CHART_PATH` and `HELM_CHART_DETECTED` values from session state. - -**If chart found:** - -```markdown -## Helm Chart Detected - -**Location:** [chart-path] - -| Field | Value | -|-------|-------| -| Name | [chart-name] | -| Version | [chart-version] | -| App Version | [app-version] | -| Description | [description] | - -**Templates found:** -- [list of template files] - -**Values file:** [values.yaml path] - -Would you like to: -1. Deploy using this chart (recommended) -2. Customize values before deploying -3. Use a different chart location -``` - -Proceed automatically without waiting for user input. - -**If no chart found:** - -```markdown -## No Helm Chart Found - -I searched these locations but found no Helm chart: -- ./Chart.yaml -- ./chart/Chart.yaml -- ./charts/*/Chart.yaml -- ./helm/Chart.yaml -- ./deploy/helm/Chart.yaml - -**Options:** -1. **Create a new Helm chart** - I'll generate one based on your project -2. **Specify chart path** - Point me to your chart location -3. **Use a different deployment method** - Try /deploy or /containerize-deploy - -Which would you prefer? -``` - -Proceed automatically without waiting for user input. - -### Step 3: Create Helm Chart (if needed) - -If user chooses to create a chart: - -```markdown -## Creating Helm Chart - -I'll create a Helm chart based on your project. - -**Detected Project Info:** -| Setting | Value | -|---------|-------| -| App Name | [app-name] | -| Language | [language] | -| Framework | [framework] | -| Port | [port] | - -**Chart will include:** -- Chart.yaml with project metadata -- values.yaml with configurable options -- Deployment template -- Service template -- Route template (OpenShift) -- Helper templates - -**Target directory:** ./chart/ - -Proceed with creating the Helm chart? (yes/no) -``` - -Proceed automatically without waiting for user input. - -Use templates from templates/helm/ to generate: -1. Chart.yaml -2. values.yaml -3. templates/deployment.yaml -4. templates/service.yaml -5. templates/route.yaml -6. templates/_helpers.tpl -7. templates/NOTES.txt - -Replace `${APP_NAME}` placeholders with actual app name in all template files. - -### Step 4: Check for Existing Release - -Before installing, check if a release with the same name exists: - -```markdown -## Checking for Existing Release... - -[Use helm_list to check] -``` - -**If release exists:** - -```markdown -## Existing Release Found - -A release named '[name]' already exists. - -| Field | Value | -|-------|-------| -| Status | [status] | -| Revision | [revision] | -| Chart | [chart-name] v[version] | -| Updated | [timestamp] | - -**Options:** -1. Upgrade the release with new configuration -2. Rollback to a previous revision -3. Uninstall and reinstall -4. Cancel - -Which would you like to do? -``` - -Proceed automatically without waiting for user input. - -### Step 5: Review Values - -```markdown -## Chart Values Configuration - -**Current values.yaml:** - -```yaml -replicaCount: [value] -image: - repository: [value] - tag: [value] -service: - port: [value] -route: - enabled: [value] -resources: - limits: - memory: [value] -``` - -**Common customizations:** - -| Value | Current | Description | -|-------|---------|-------------| -| `replicaCount` | 1 | Number of pods | -| `image.repository` | [repo] | Container image | -| `image.tag` | [tag] | Image version | -| `service.port` | [port] | Service port | -| `resources.limits.memory` | 512Mi | Memory limit | - -**Options:** -1. Deploy with current values -2. Modify values interactively -3. Use a custom values file - -Which would you prefer? -``` - -Proceed automatically without waiting for user input. - -### Step 6: Pre-Deploy Summary - -```markdown -## Helm Deployment Summary - -**Release Configuration:** - -| Setting | Value | -|---------|-------| -| Release Name | [release-name] | -| Namespace | [namespace] | -| Chart | [chart-path] | -| Chart Version | [version] | - -**Resources to be created:** - -| Resource | Name | -|----------|------| -| Deployment | [name] | -| Service | [name] | -| Route | [name] (if enabled) | - -**Values to apply:** -```yaml -[show customized values or "Using defaults"] -``` - -**Helm command equivalent:** -```bash -helm install [release-name] [chart-path] -n [namespace] [--set options] -``` - -**Proceed with Helm deployment?** (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Step 7: Execute Deployment - -Use kubernetes MCP `helm_install` or `helm_upgrade`: - -```markdown -## Deploying with Helm... - -**Release:** [release-name] -**Chart:** [chart-name] v[version] - -[x] Chart validated -[x] Templates rendered -[ ] Installing release... - ---- - -**Installation Progress:** - -Waiting for resources to be ready... - -| Resource | Status | -|----------|--------| -| Deployment/[name] | [status] | -| Service/[name] | [status] | -| Route/[name] | [status] | - ---- -``` - -Monitor pod status using `pods_list_in_namespace` until pods are ready or timeout. - -### Step 8: Deployment Complete - -```markdown -## Helm Deployment Complete! - -**Release:** [release-name] -**Status:** deployed -**Revision:** 1 -**Namespace:** [namespace] - ---- - -**Resources Created:** - -| Resource | Name | Status | -|----------|------|--------| -| Deployment | [name] | [replicas] Ready | -| Service | [name] | Active | -| Route | [name] | Admitted | - -**Access URL:** https://[route-host] - ---- - -**Quick Commands:** - -```bash -# Check release status -helm status [release-name] -n [namespace] - -# View release history -helm history [release-name] -n [namespace] - -# Upgrade with new values -helm upgrade [release-name] [chart-path] -n [namespace] -f new-values.yaml - -# Rollback to previous version -helm rollback [release-name] 1 -n [namespace] - -# Uninstall release -helm uninstall [release-name] -n [namespace] - -# View logs -oc logs -l app.kubernetes.io/instance=[release-name] -n [namespace] -f -``` - ---- - -Your application is live! -``` - -## Dependencies - -### Required MCP Servers -- `openshift` - Helm install, upgrade, list, and uninstall operations - -### Related Skills -- `/deploy` - Alternative deployment without Helm charts -- `/debug-pod` - Troubleshoot pods after Helm deployment -- `/debug-network` - Diagnose networking issues with deployed services - -### Reference Documentation -- [docs/builder-images.md](../../docs/builder-images.md) - Container image references for chart values -- [docs/image-selection-criteria.md](../../docs/image-selection-criteria.md) - Image variant selection for production deployments -- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (oc, helm) diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__recommend-image/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/with_skills/rh-developer__recommend-image/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__recommend-image/environment/skills/recommend-image/SKILL.md b/evaluation/with_skills/rh-developer__recommend-image/environment/skills/recommend-image/SKILL.md deleted file mode 100644 index d5c81d44..00000000 --- a/evaluation/with_skills/rh-developer__recommend-image/environment/skills/recommend-image/SKILL.md +++ /dev/null @@ -1,282 +0,0 @@ ---- -name: recommend-image -description: | - Intelligently recommend the optimal S2I builder image or container base image for a project based on detected language/framework, use-case requirements, security posture, and deployment target. Supports GitHub URLs for remote project analysis (delegates to /detect-project). Use this skill when the user needs a container image recommendation, wants to compare image options, or asks about production vs development images. Triggers on /recommend-image command, or when advanced image selection beyond basic version matching is needed. Supports Node.js, Python, Java, Go, Ruby, .NET, PHP, and Perl on Red Hat UBI. -model: inherit -color: cyan -metadata: - user_invocable: "true" ---- - -# /recommend-image Skill - -Provide intelligent, use-case-aware container image recommendations that go beyond simple language-to-image mapping. - -## When to Use This Skill - -- User asks for the "best" image for their use case -- User needs to choose between production vs development images -- User wants to compare image options (minimal vs full-featured) -- `/detect-project` completed and user wants a tailored recommendation -- User asks about image size, security, or performance trade-offs - -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -## Workflow - -### Step 1: Gather Context - -**If invoked after `/detect-project`:** -Use the already-detected values: -- `LANGUAGE` - Programming language -- `FRAMEWORK` - Framework (if detected) -- `VERSION` - Language version - -**If invoked with a GitHub URL:** - -Example: `/recommend-image for https://github.com/RHEcosystemAppEng/sast-ai-frontend` - -When a GitHub URL is provided: - -```markdown -## Analyzing Remote Repository - -I'll analyze the repository to detect the project type first. - -Invoking `/detect-project` for: `[github-url]` -``` - -**Delegate to `/detect-project`:** -- Pass the GitHub URL to `/detect-project` -- `/detect-project` will use GitHub MCP to analyze the repository -- Receive back: `LANGUAGE`, `FRAMEWORK`, `VERSION`, `APP_NAME` -- Continue to Step 2 (Use-Case Assessment) - -**If invoked standalone (no URL, no prior detection):** -Ask the user: - -```markdown -## Image Recommendation - -To recommend the best image, I need some information: - -**Option 1:** Provide a GitHub URL -- Example: `/recommend-image for https://github.com/user/repo` - -**Option 2:** Tell me about your project -1. **What language/framework is your project?** - (e.g., Python 3.11, Node.js 20, Java 17 with Spring Boot) - -2. **What version do you need?** - (or say "latest LTS" if unsure) -``` - -### Step 2: Use-Case Assessment - -Present use-case questions: - -```markdown -## Use-Case Assessment - -To recommend the optimal image, please tell me about your requirements: - -**1. Deployment Environment:** -- **Production** - Stability, security, long-term support critical -- **Development** - Tooling, debugging features preferred -- **Edge/IoT** - Minimal footprint essential - -**2. Security Priority:** -- **Standard** - Red Hat UBI with regular updates -- **Hardened** - Minimal attack surface, fewer packages -- **Compliance** - FIPS or specific compliance requirements - -**3. Performance Priority:** -- **Fast startup** - Serverless, scale-to-zero workloads -- **Low memory** - High-density deployments -- **Balanced** - General purpose applications - -**4. Build Requirements:** -- **Need build tools** - Native extensions, compilation during build -- **Runtime only** - Pre-compiled, no build tools needed - -Please describe your use case or select from the options above. -``` - -Proceed automatically without waiting for user input. - -### Step 3: Evaluate Image Options - -For each language, evaluate available variants against user requirements. - -**Image Variants:** Full (build tools), Minimal (smaller, secure), Runtime (smallest, pre-compiled only) - -**Key Scoring Factors:** Image size, security posture, build tools availability, startup time, LTS status - -> **See [docs/image-selection-criteria.md](../../docs/image-selection-criteria.md)** for comprehensive scoring matrices with weighted criteria by environment (production/development/edge/serverless). - -### Step 3.5: Dynamic Image Validation - -Before presenting recommendations, validate with dynamic sources to provide accurate, real-time data. - -#### Check if Skopeo is Available - -First, verify skopeo is installed: - -```bash -which skopeo -``` - -**If skopeo is NOT installed**, present: - -```markdown -## Skopeo Required for Image Validation - -To provide accurate image recommendations, I need `skopeo` to inspect container images. - -**Skopeo is not installed.** This tool allows me to: -- Verify the image exists before recommending it -- Get exact image size (not estimates) -- Check architecture support (amd64, arm64) -- Show when the image was last built - -**Install skopeo:** See [docs/prerequisites.md](../../docs/prerequisites.md) for installation commands by OS. - -After installing, run `/recommend-image` again for enhanced recommendations. - -**Continue without skopeo?** -- **yes** - Use static reference data only (less accurate) -- **install** - I'll install skopeo first -``` - -Proceed automatically without waiting for user input. - -If user continues without skopeo, proceed with static data and note: "Image metadata from static reference (not verified)". - -#### Skopeo Verification - -For each candidate image, verify availability and get metadata: - -```bash -# Verify image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/[candidate-image] -``` - -**Note:** The `docker://` transport is OCI-standard and works with Podman registries - it's not Docker-specific. - -### Step 4: Present Recommendation - -Format your recommendation: - -```markdown -## Image Recommendation - -Based on your requirements: - -| Factor | Your Input | -|--------|------------| -| Language | [language] [version] | -| Framework | [framework or "None"] | -| Environment | [Production/Development/Edge] | -| Security | [Standard/Hardened/Compliance] | -| Priority | [startup/memory/balanced] | -| Build Tools | [needed/not needed] | - ---- - -### Recommended Image - -`registry.access.redhat.com/ubi9/[image-name]` - -**Why this image:** -- [Reason 1 - matches primary requirement] -- [Reason 2 - matches secondary requirement] -- [Reason 3 - version/LTS consideration] - -**Image Details:** -| Property | Value | Source | -|----------|-------|--------| -| Base | UBI 9 | Static | -| Variant | [Full/Minimal/Runtime] | Static | -| Size | [exact-size]MB | Skopeo | -| Built | [build-date] | Skopeo | -| Architecture | amd64, arm64 | Skopeo | -| LTS | [Yes/No - EOL date] | Static | - -**Security Status:** [status-icon] [status-message] -- Last checked: [timestamp] -- Source: Red Hat Security Data API - -*(If skopeo unavailable: "Image metadata from static reference - install skopeo for verified data")* - -**Trade-offs:** -- [What you give up with this choice] -- [When you might choose differently] - ---- - -### Alternative Options - -| Image | Best For | Trade-off | -|-------|----------|-----------| -| `[alternative-1]` | [use case] | [trade-off] | -| `[alternative-2]` | [use case] | [trade-off] | - ---- - -**Confirm this recommendation?** -- Type **yes** to use `[recommended-image]` -- Type **alternative N** to use an alternative -- Tell me if you have different requirements -``` - -Proceed automatically without waiting for user input. - -### Step 5: Handle Confirmation - -**If user confirms:** - -```markdown -## Image Selected - -| Setting | Value | -|---------|-------| -| Builder Image | `[full-image-reference]` | -| Variant | [variant] | -| Rationale | [brief reason] | - -Configuration saved. You can now: -- Run `/s2i-build` to build with this image -- Run `/containerize-deploy` for the full workflow -``` - -**If user selects alternative:** -Update the selection and confirm. - -**If user has different requirements:** -Return to Step 2 with new inputs. - -## Image Reference - -**Quick variant selection:** -- **Production** → Minimal or Runtime variant -- **Development** → Full variant -- **Serverless** → Smallest available (minimal or native binary) - -> **See [docs/image-selection-criteria.md](../../docs/image-selection-criteria.md)** for comprehensive image size references, LTS timelines, decision trees, and framework-specific recommendations (Quarkus, Spring Boot, Next.js, Django/Flask). - -## Dependencies - -### Required MCP Servers -- None required (uses Bash for skopeo image inspection) - -### Related Skills -- `/detect-project` - Provides language/framework detection input for recommendations -- `/s2i-build` - Build with the recommended image - -### Reference Documentation -- [docs/image-selection-criteria.md](../../docs/image-selection-criteria.md) - Comprehensive scoring matrices, image size reference, LTS timelines, decision trees -- [docs/builder-images.md](../../docs/builder-images.md) - UBI image registry, framework-specific recommendations, variant availability -- [docs/dynamic-validation.md](../../docs/dynamic-validation.md) - Skopeo commands, Red Hat Security Data API, image verification patterns -- [docs/prerequisites.md](../../docs/prerequisites.md) - Skopeo installation instructions diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/with_skills/rh-developer__rhel-deploy/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__rhel-deploy/environment/skills/rhel-deploy/SKILL.md b/evaluation/with_skills/rh-developer__rhel-deploy/environment/skills/rhel-deploy/SKILL.md deleted file mode 100644 index 66efbdee..00000000 --- a/evaluation/with_skills/rh-developer__rhel-deploy/environment/skills/rhel-deploy/SKILL.md +++ /dev/null @@ -1,482 +0,0 @@ ---- -name: rhel-deploy -description: | - CRITICAL: When user types /rhel-deploy, use THIS skill immediately. This skill deploys applications to standalone RHEL/Fedora/CentOS systems (NOT OpenShift) using Podman containers with systemd, or native dnf builds. Handles SSH connectivity, SELinux, firewall-cmd, and systemd unit creation. Triggers: /rhel-deploy command, 'deploy to RHEL', 'deploy to Fedora', 'deploy to my server via SSH'. -model: inherit -color: yellow -metadata: - user_invocable: "true" ---- - -# /rhel-deploy Skill - -**IMPORTANT:** This skill is for deploying to standalone RHEL/Fedora/CentOS systems via SSH. If user invoked `/rhel-deploy`, skip any OpenShift-related steps and proceed directly with SSH-based deployment. - -Deploy applications to standalone RHEL systems using Podman containers or native builds with systemd service management. - -## Overview - -``` -[Intro] → [SSH Connect] → [Analyze] → [Strategy] ──┬─→ [Container Path] ──→ [Complete] - │ (Podman + systemd) - │ - └─→ [Native Path] ─────→ [Complete] - (dnf + systemd) -``` - -**Deployment Strategies (user chooses one):** -- **Container** - Build/pull container image, run with Podman, manage with systemd -- **Native** - Install dependencies with dnf, run application directly, manage with systemd - -## Prerequisites - -1. SSH access to target RHEL host with sudo privileges -2. RHEL 8+, CentOS Stream, Rocky Linux, or Fedora -3. For container deployments: Podman installed on target -4. For native deployments: Required development tools available via dnf - -## When to Use This Skill - -Use `/rhel-deploy` when deploying applications to standalone RHEL, Fedora, or CentOS systems via SSH. This skill handles Podman container or native dnf deployments with systemd service management, SELinux, and firewall configuration. - -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -## Workflow - -### Phase 0: Introduction - -Present the workflow overview: Connect → Analyze → Strategy → Build/Deploy → Verify. Describe Container (Podman + systemd) vs Native (dnf + systemd) strategies. Ask: **Ready to begin?** (yes/no) - -Proceed automatically without waiting for user input. - -### Phase 1: SSH Connection - -```markdown -## Phase 1: Connecting to RHEL Host - -**SSH Target Configuration:** - -Please provide your RHEL host details: - -| Setting | Value | Default | -|---------|-------|---------| -| Host | [required] | - | -| User | [current user] | $USER | -| Port | 22 | 22 | - -Example: `user@192.168.1.100` or `deploy@myserver.example.com` - -**Enter your SSH target:** -``` - -**Connection verification:** - -```bash -ssh -o BatchMode=yes -o ConnectTimeout=10 [user]@[host] "echo 'Connection successful'" -``` - -If connection fails, troubleshoot: host reachability, SSH key configuration, firewall port 22. - -Store `RHEL_HOST`, `RHEL_USER`, `RHEL_PORT` in session state. - -### Phase 2: Target Host Analysis - -```markdown -## Phase 2: Analyzing Target Host - -Checking capabilities of [host]... - -| Setting | Value | -|---------|-------| -| OS | [cat /etc/redhat-release] | -| Kernel | [uname -r] | -| Architecture | [uname -m] | -| Podman | [Installed v4.x / Not installed] | -| SELinux | [Enforcing / Permissive / Disabled] | -| Firewall | [Active / Inactive] | - -Is this the correct target host? (yes/no) -``` - -Proceed automatically without waiting for user input. - -**Commands to gather information:** - -```bash -ssh [target] "cat /etc/redhat-release" -ssh [target] "podman --version 2>/dev/null || echo 'Not installed'" -ssh [target] "getenforce" -ssh [target] "firewall-cmd --state 2>/dev/null || echo 'Not running'" -``` - -Store `RHEL_VERSION`, `PODMAN_AVAILABLE`, `SELINUX_STATUS`, `FIREWALL_STATUS` in session state. - -### Phase 2b: Red Hat Insights Pre-Deploy Check (Optional) - -**This phase runs only if the `lightspeed-mcp` server is available.** Use `ToolSearch` to check for Lightspeed MCP tools. If not available, skip silently and proceed to Phase 3. - -**Step 1:** Use `find_host_by_name` with the target hostname to look up the system in Red Hat Insights. - -**Step 2:** If found, use `get_system_cves` to check for critical/important CVEs on the target. - -**Step 3:** Use `get_rhel_lifecycle` to verify the target RHEL version is still supported. - -Append to Phase 2 output: - -```markdown -**Red Hat Insights (Optional):** -| Check | Status | Details | -|-------|--------|---------| -| Registered in Insights | [Yes/No] | [system-id or "Not found"] | -| RHEL Lifecycle | [Active/Maintenance/EOL] | [end date] | -| Critical/Important CVEs | [count] | [top 3 CVE IDs] | - -[If critical CVEs found:] -**WARNING:** Target system has [N] critical/important CVEs. Consider remediating before deploying. - -[If RHEL version is EOL:] -**WARNING:** RHEL [version] has reached End of Life ([date]). Consider upgrading before deploying. -``` - -These are informational warnings only — they do not block deployment. - -### Phase 3: Strategy Selection - -```markdown -## Deployment Strategy - -Based on your project ([language]/[framework]) and target capabilities: - -| Strategy | Description | Requirements | -|----------|-------------|--------------| -| **Container** | Build image, run with Podman + systemd | Podman installed | -| **Native** | Install with dnf, run directly + systemd | Runtime packages available | - -**Recommendation:** [Container/Native] because [reason] - -**Which deployment strategy would you like to use?** -1. Container - Deploy using Podman -2. Native - Deploy directly on host -``` - -Proceed automatically without waiting for user input. - -**If Podman not installed and user selects Container:** -```markdown -Podman is not installed on the target. Would you like me to install it? - -```bash -sudo dnf install -y podman -``` - -Proceed with Podman installation? (yes/no) -``` - -Proceed automatically without waiting for user input. - -Store `DEPLOYMENT_STRATEGY` in session state. - ---- - -## CONTAINER PATH (If DEPLOYMENT_STRATEGY is "Container") - -### Phase 4a-1: Image Selection - -```markdown -## Container Image - -**Options:** - -1. **Build on target** - Transfer source, build with Podman on RHEL host -2. **Build locally and transfer** - Build here, push to registry or transfer -3. **Use existing image** - Pull from registry (e.g., quay.io, docker.io) - -Which approach would you prefer? -``` - -Proceed automatically without waiting for user input. - -**For options 1 and 2 (building an image):** - -If no Containerfile/Dockerfile exists in the project, delegate to `/recommend-image`: - -```markdown -## Selecting Base Image - -To build your container, I need to select an appropriate base image. - -Invoking `/recommend-image` to get the optimal UBI image for your [language]/[framework] project... -``` - -Use the `BUILDER_IMAGE` output from `/recommend-image` as the base image in the Containerfile. - -**For build on target:** -```bash -# Transfer source and build -rsync -avz --exclude node_modules --exclude .git ./ [target]:/tmp/[app-name]-build/ -# If no Containerfile exists, generate one using BUILDER_IMAGE from /recommend-image -ssh [target] "cd /tmp/[app-name]-build && podman build -t [app-name]:latest ." -``` - -**For existing image:** -```bash -ssh [target] "podman pull [image-reference]" -``` - -### Phase 4a-2: Container Configuration - -```markdown -## Container Configuration - -**Container Settings:** -| Setting | Value | -|---------|-------| -| Name | [app-name] | -| Image | [image-ref] | -| Port Mapping | [host-port]:[container-port] | -| Volume Mounts | [list any persistent data paths] | -| Environment | [list env vars] | -| Run Mode | [rootless / rootful] | - -**SELinux Volume Labels:** Use `:z` for shared volumes, `:Z` for private volumes. See [docs/rhel-deployment.md](../../docs/rhel-deployment.md) for SELinux configuration details. - -Proceed with this configuration? (yes/modify/cancel) -``` - -Proceed automatically without waiting for user input. - -### Phase 4a-3: Systemd Unit Creation - -```markdown -## Systemd Service Configuration - -Creating systemd unit for Podman container. - -**Template to use:** -- Rootful: `templates/systemd/systemd-container-rootful.service` -- Rootless: `templates/systemd/systemd-container-rootless.service` - -**Variables to substitute:** -| Variable | Value | -|----------|-------| -| `${APP_NAME}` | [app-name] | -| `${PORT}` | [container-port] | -| `${IMAGE}` | [container-image] | - -**Target locations:** -- Rootful: `/etc/systemd/system/[app-name].service` -- Rootless: `~/.config/systemd/user/[app-name].service` - -Proceed with creating this service? (yes/no) -``` - -Proceed automatically without waiting for user input. - -**Steps to execute:** - -1. Read the appropriate template from `templates/systemd/` -2. Substitute `${APP_NAME}`, `${PORT}`, `${IMAGE}` with session state values -3. Transfer the generated unit file to the target host -4. Enable and start the service - -```bash -# Rootful: transfer to /etc/systemd/system/, daemon-reload, enable --now -# Rootless: transfer to ~/.config/systemd/user/, daemon-reload, enable --now, enable-linger -ssh [target] "sudo systemctl daemon-reload && sudo systemctl enable --now [app-name]" -``` - -### Phase 4a-4: Firewall Configuration - -```markdown -## Firewall Configuration - -Opening port [port] for application access. - -**Commands to execute:** -```bash -# Open port permanently -sudo firewall-cmd --permanent --add-port=[port]/tcp - -# Reload firewall -sudo firewall-cmd --reload - -# Verify -sudo firewall-cmd --list-ports -``` - -Proceed with firewall configuration? (yes/skip) -``` - -Proceed automatically without waiting for user input. - ---- - -## NATIVE PATH (If DEPLOYMENT_STRATEGY is "Native") - -### Phase 4b-1: Dependency Installation - -```markdown -## Installing Dependencies - -**Runtime packages for [language]:** - -See [docs/rhel-deployment.md](../../docs/rhel-deployment.md) for the complete runtime package mapping by language and RHEL version (Node.js, Python, Java, Go, Ruby, PHP). - -**Commands to execute:** -```bash -ssh [target] "sudo dnf install -y [packages]" -``` - -Proceed with installation? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Phase 4b-2: Application Deployment - -```markdown -## Deploying Application - -**Deployment location:** `/opt/[app-name]` - -**Steps:** -1. Create application directory -2. Transfer source code via rsync -3. Install application dependencies -4. Set ownership and permissions -5. Configure SELinux context - -```bash -ssh [target] "sudo mkdir -p /opt/[app-name]" -rsync -avz --exclude node_modules --exclude .git --exclude __pycache__ ./ [target]:/tmp/[app-name]-deploy/ -ssh [target] "sudo cp -r /tmp/[app-name]-deploy/* /opt/[app-name]/" -ssh [target] "cd /opt/[app-name] && npm install --production" # language-specific -ssh [target] "sudo chown -R [service-user]:[service-user] /opt/[app-name]" -ssh [target] "sudo semanage fcontext -a -t bin_t '/opt/[app-name](/.*)?'" -ssh [target] "sudo restorecon -Rv /opt/[app-name]" -``` - -Proceed with deployment? (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Phase 4b-3: Native Systemd Unit - -```markdown -## Systemd Service Configuration - -**Template to use:** `templates/systemd/systemd-native.service` - -**Variables to substitute:** -| Variable | Value | Notes | -|----------|-------|-------| -| `${APP_NAME}` | [app-name] | Application name | -| `${SERVICE_USER}` | [service-user] | User to run the service as | -| `${APP_PATH}` | /opt/[app-name] | Application install path | -| `${PORT}` | [container-port] | Application listen port | -| `${START_COMMAND}` | [see below] | Language-specific start command | - -**Start commands by language:** See [docs/rhel-deployment.md](../../docs/rhel-deployment.md) for language-specific systemd unit templates (Node.js, Python, Java, Go). - -**Target location:** `/etc/systemd/system/[app-name].service` - -**Note:** The template includes security hardening (NoNewPrivileges, ProtectSystem, ProtectHome, PrivateTmp). - -Proceed with creating this service? (yes/no) -``` - -Proceed automatically without waiting for user input. - -**Steps to execute:** - -1. Read the template from `templates/systemd/systemd-native.service` -2. Substitute all variables with session state values -3. Transfer the generated unit file to the target host -4. Enable and start the service - -### Phase 4b-4: Firewall Configuration - -Same as container path - open required port with firewall-cmd. - ---- - -## COMPLETION (Both paths converge here) - -### Phase 5: Completion - -```markdown -## Deployment Complete! - -Your application is now running on [host]. - -**Application Summary:** -| Setting | Value | -|---------|-------| -| Name | [app-name] | -| Host | [host] | -| Strategy | [Container/Native] | -| Service | [app-name].service | - -**Access URLs:** -| Type | URL | -|------|-----| -| HTTP | http://[host]:[port] | -| SSH | ssh [user]@[host] | - -**Service Status:** [systemctl status output] - -**Quick Commands:** - -Show quick commands for: view logs (journalctl), restart/stop/status (systemctl), container logs (if container), and removal steps. -``` - -### Phase 5a: Handle Deployment Failure - -If the service fails to start or is not accessible: - -```markdown -## Deployment Failed - -The service did not start successfully. - -**Service Status:** [systemctl status output showing failure] - -**Recent Errors:** -| Time | Error | -|------|-------| -| [time] | [error from journalctl] | - -**Would you like me to diagnose the issue?** -1. **Debug RHEL** (`/debug-rhel`) - Full system diagnosis (systemd, journal, SELinux, firewall) -2. **Debug Container** (`/debug-container`) - Container state, logs, exit codes -3. **View full logs** - Complete journalctl output -4. **Check SELinux** - Quick SELinux denial check -5. **Check firewall** - Quick firewall port check -6. **Stop and clean up** - -Select an option: -``` - -Proceed automatically without waiting for user input. - -- If user selects "Debug RHEL" → Invoke `/debug-rhel` skill -- If user selects "Debug Container" → Invoke `/debug-container` skill -- After debugging → Offer to retry deployment - -## Dependencies - -### Required MCP Servers -- `lightspeed-mcp` (optional) - Red Hat Insights pre-deploy checks - -### Related Skills -- `/debug-rhel` - systemd failures, SELinux denials, firewall blocking -- `/debug-container` - Container startup issues on RHEL host - -### Reference Documentation -- [docs/rhel-deployment.md](../../docs/rhel-deployment.md) - Systemd templates, SELinux, firewall, runtime packages -- [docs/selinux-troubleshooting.md](../../docs/selinux-troubleshooting.md) - SELinux denial analysis and fixes -- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common error patterns and troubleshooting -- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (ssh, podman) diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__s2i-build/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/with_skills/rh-developer__s2i-build/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__s2i-build/environment/skills/s2i-build/SKILL.md b/evaluation/with_skills/rh-developer__s2i-build/environment/skills/s2i-build/SKILL.md deleted file mode 100644 index d5b179fa..00000000 --- a/evaluation/with_skills/rh-developer__s2i-build/environment/skills/s2i-build/SKILL.md +++ /dev/null @@ -1,391 +0,0 @@ ---- -name: s2i-build -description: | - Create BuildConfig and ImageStream resources on OpenShift and trigger a Source-to-Image (S2I) build. Use this skill after /detect-project to build container images from source code on the cluster. Handles namespace verification, resource creation with user confirmation, build monitoring with log streaming, and failure recovery. Triggers on /s2i-build command. Run before /deploy. -model: inherit -color: green -metadata: - user_invocable: "true" ---- - -# /s2i-build Skill - -Create the necessary OpenShift resources (BuildConfig, ImageStream) and trigger a Source-to-Image build on the cluster. - -## Prerequisites - -Before running this skill, ensure: -1. User is logged into OpenShift cluster -2. Target namespace/project exists or can be created -3. Git repository URL is available (or will use binary build) - -## When to Use This Skill - -Use this skill after `/detect-project` to build container images from source code on OpenShift using Source-to-Image. It creates BuildConfig and ImageStream resources, triggers the build, and monitors progress with log streaming. - -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -## Workflow - -### Step 1: Check OpenShift Connection - -Use kubernetes MCP to verify connection: - -```markdown -## Checking OpenShift Connection... - -**Cluster:** [cluster-url from kubeconfig] -**User:** [current user] -**Current Namespace:** [current namespace] - -Is this the correct cluster and namespace for the build? -- yes - Continue -- no - Let me switch context -``` - -Proceed automatically without waiting for user input. - -### Step 2: Gather Build Information - -Collect required information (from /detect-project or ask user): - -```markdown -## S2I Build Configuration - -I need the following information: - -| Setting | Current Value | Source | -|---------|---------------|--------| -| App Name | `[name]` | [from detect-project / folder name] | -| Git URL | `[url]` | [from .git/config / needs input] | -| Git Branch | `main` | [default] | -| S2I Builder | `[image]` | [from detect-project / needs input] | -| Namespace | `[ns]` | [from current context] | - -[For Python projects only - include these rows if PYTHON_ENTRY_FILE is set] -| Entry Point | `[PYTHON_ENTRY_FILE]` | [from detect-project] | -| APP_MODULE | `[PYTHON_APP_MODULE]` | [Python only - required if entry point != app.py] | -| gunicorn | [Found / Missing] | [from detect-project] | - -Please confirm these values or tell me what to change. -``` - -**Python Entry Point Warning:** - -If `PYTHON_ENTRY_FILE` is NOT `app.py` AND `PYTHON_HAS_GUNICORN` is `false`: - -```markdown -## Python Configuration Issue - -Your application uses `[PYTHON_ENTRY_FILE]` as entry point, but `gunicorn` is not in your requirements. - -**This build will FAIL** because: -- The S2I Python builder requires `gunicorn` to use `APP_MODULE` -- Without gunicorn, it looks for `app.py` (which doesn't exist) - -**Please choose:** -1. **Add gunicorn** - Add `gunicorn` to requirements.txt and retry -2. **Rename entry point** - Rename `[main.py]` to `app.py` -3. **Continue anyway** - Proceed (build will likely fail) -``` - -Proceed automatically without waiting for user input. - -**To detect Git URL:** -- Read `.git/config` and extract `[remote "origin"]` url - -### Step 3: Verify Namespace - -Use kubernetes MCP `resources_list` to check if namespace exists: - -```markdown -## Namespace Check - -Checking if namespace `[namespace]` exists... - -[If exists] -Namespace `[namespace]` exists and you have access. - -[If not exists] -Namespace `[namespace]` does not exist. - -Would you like me to create it? (yes/no) -``` - -Proceed automatically without waiting for user input. - -If creating namespace, use `resources_create_or_update`: -```yaml -apiVersion: v1 -kind: Namespace -metadata: - name: [namespace] -``` - -### Step 4: Create ImageStream - -Show the ImageStream that will be created: - -```markdown -## Step 1 of 3: Create ImageStream - -An ImageStream stores references to your built container images. - -```yaml -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: [app-name] - namespace: [namespace] - labels: - app: [app-name] - app.kubernetes.io/name: [app-name] -spec: - lookupPolicy: - local: false -``` - -**Proceed with creating this ImageStream?** (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Step 5: Create BuildConfig - -Show the BuildConfig: - -**For non-Python projects OR Python with app.py entry point:** - -```markdown -## Step 2 of 3: Create BuildConfig - -A BuildConfig defines how to build your application using S2I. - -```yaml -apiVersion: build.openshift.io/v1 -kind: BuildConfig -metadata: - name: [app-name] - namespace: [namespace] - labels: - app: [app-name] - app.kubernetes.io/name: [app-name] -spec: - source: - type: Git - git: - uri: [git-url] - ref: [git-branch] - strategy: - type: Source - sourceStrategy: - from: - kind: DockerImage - name: [builder-image] - output: - to: - kind: ImageStreamTag - name: [app-name]:latest - triggers: - - type: ConfigChange - - type: ImageChange - runPolicy: Serial -``` - -**This BuildConfig will:** -- Pull source from: `[git-url]` (branch: `[git-branch]`) -- Build using S2I with: `[builder-image]` -- Push result to: `[app-name]:latest` ImageStream - -**Proceed with creating this BuildConfig?** (yes/no) -``` - -**For Python projects with non-default entry point (e.g., main.py):** - -```markdown -## Step 2 of 3: Create BuildConfig - -A BuildConfig defines how to build your application using S2I. - -```yaml -apiVersion: build.openshift.io/v1 -kind: BuildConfig -metadata: - name: [app-name] - namespace: [namespace] - labels: - app: [app-name] - app.kubernetes.io/name: [app-name] -spec: - source: - type: Git - git: - uri: [git-url] - ref: [git-branch] - strategy: - type: Source - sourceStrategy: - from: - kind: DockerImage - name: [builder-image] - # Python S2I: Required when entry point is not app.py - env: - - name: APP_MODULE - value: "[PYTHON_APP_MODULE]" # e.g., "main:app" - output: - to: - kind: ImageStreamTag - name: [app-name]:latest - triggers: - - type: ConfigChange - - type: ImageChange - runPolicy: Serial -``` - -**This BuildConfig will:** -- Pull source from: `[git-url]` (branch: `[git-branch]`) -- Build using S2I with: `[builder-image]` -- Push result to: `[app-name]:latest` ImageStream - -**Python Entry Point Configuration:** -- Entry point file: `[PYTHON_ENTRY_FILE]` -- APP_MODULE: `[PYTHON_APP_MODULE]` -- This tells the S2I Python builder how to start your application with gunicorn. - -**Proceed with creating this BuildConfig?** (yes/no) -``` - -Proceed automatically without waiting for user input. - -### Step 6: Start Build - -```markdown -## Step 3 of 3: Start Build - -Resources created successfully! - -| Resource | Name | Status | -|----------|------|--------| -| ImageStream | [app-name] | Created | -| BuildConfig | [app-name] | Created | - -**Would you like me to start a build now?** (yes/no) - -(You can also trigger builds later with: oc start-build [app-name]) -``` - -Proceed automatically without waiting for user input. - -If yes, create a Build resource: -```yaml -apiVersion: build.openshift.io/v1 -kind: Build -metadata: - generateName: [app-name]- - namespace: [namespace] - labels: - app: [app-name] - buildconfig: [app-name] - annotations: - openshift.io/build-config.name: [app-name] -spec: - serviceAccount: builder - source: - type: Git - git: - uri: [git-url] - ref: [git-branch] - strategy: - type: Source - sourceStrategy: - from: - kind: DockerImage - name: [builder-image] - output: - to: - kind: ImageStreamTag - name: [app-name]:latest - triggeredBy: - - message: Manually triggered -``` - -### Step 7: Monitor Build - -Stream build logs using kubernetes MCP `pod_logs`: - -```markdown -## Build Progress - -**Build:** [app-name]-1 -**Status:** Running -**Phase:** [current phase] - ---- -[Streaming build logs here] ---- - -[When complete] - -## Build Complete! - -**Build:** [app-name]-1 -**Status:** Complete -**Duration:** [X]m [Y]s -**Image:** image-registry.openshift-image-registry.svc:5000/[namespace]/[app-name]:latest - -**CRITICAL: Ensure the build status is 'Complete' before proceeding to deployment.** - -The image is ready for deployment. -Run `/deploy` to create Deployment, Service, and Route. -``` - -### Step 8: Handle Build Failure - -If build fails: - -```markdown -## Build Failed - -**Build:** [app-name]-1 -**Status:** Failed -**Phase:** [phase where it failed] - -**Error:** -``` -[Last 20 lines of build log] -``` - -**Common causes for [phase] failure:** -- [relevant troubleshooting tips] - -**Options:** -1. **Debug Build** (`/debug-build`) - Full build diagnosis - - Analyzes BuildConfig, build logs, source access, registry auth - - Identifies root cause and suggests remediation -2. View full build logs -3. Delete failed build and retry -4. Update BuildConfig and retry -5. Cancel and troubleshoot - -What would you like to do? -``` - -- If user selects "Debug Build" → Invoke `/debug-build` skill with build name -- After debugging → Offer to retry build - -## Dependencies - -### Required MCP Servers -- `openshift` - Kubernetes/OpenShift resource access for BuildConfigs, ImageStreams, and build monitoring - -### Related Skills -- `/debug-build` - Build failures (source access, dependencies, registry issues) -- `/deploy` - After successful build, to deploy the image - -### Reference Documentation -- [docs/builder-images.md](../../docs/builder-images.md) - S2I builder image selection, version mapping -- [docs/python-s2i-entrypoints.md](../../docs/python-s2i-entrypoints.md) - Python APP_MODULE configuration, entry point troubleshooting -- [docs/debugging-patterns.md](../../docs/debugging-patterns.md) - Common build error patterns and troubleshooting -- [docs/prerequisites.md](../../docs/prerequisites.md) - Required tools (oc) diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/builder-images.md b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/debugging-patterns.md b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/dynamic-validation.md b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/human-in-the-loop.md b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/image-selection-criteria.md b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/prerequisites.md b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/python-s2i-entrypoints.md b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/rhel-deployment.md b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/selinux-troubleshooting.md b/evaluation/with_skills/rh-developer__validate-environment/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/with_skills/rh-developer__validate-environment/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/with_skills/rh-developer__validate-environment/environment/skills/validate-environment/SKILL.md b/evaluation/with_skills/rh-developer__validate-environment/environment/skills/validate-environment/SKILL.md deleted file mode 100644 index 33020fe0..00000000 --- a/evaluation/with_skills/rh-developer__validate-environment/environment/skills/validate-environment/SKILL.md +++ /dev/null @@ -1,232 +0,0 @@ ---- -name: validate-environment -description: | - Check and report the status of required tools and environment for rh-developer skills. Validates tool installation (oc, helm, podman, git, skopeo, etc.), cluster connectivity, and permissions. Use this skill before running other deployment skills to ensure prerequisites are met. Triggers on /validate-environment command or when user asks to check their environment setup. -model: inherit -color: cyan -metadata: - user_invocable: "true" ---- - -# Validate Environment Skill - -Check that required tools and environment are properly configured. - -## When to Use This Skill - -- User wants to verify their environment before running deployment skills -- User encounters tool-related errors and needs a diagnostic check -- First-time setup or after environment changes to confirm readiness - -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. Proceed automatically without waiting for user input. -2. Present results clearly and ask if user wants to proceed with fixes -3. Never auto-fix issues without user approval - -## Workflow - -### Step 1: Determine Validation Scope - -Ask user if not clear: - -```markdown -## Environment Validation - -What would you like to validate? - -1. **All** - Check all tools and connections -2. **OpenShift** - Check oc, helm, cluster connectivity -3. **RHEL/Containers** - Check podman, ssh, container tools -4. **Minimal** - Just check core tools (git, curl) - -Select an option (1-4): -``` - -### Step 2: Check Core Tools - -Run these checks using Bash: - -```bash -# Check each tool and capture version -check_tool() { - if command -v "$1" &> /dev/null; then - echo "INSTALLED: $1 ($($1 --version 2>&1 | head -1))" - else - echo "MISSING: $1" - fi -} -``` - -**Tools to check:** git, curl, jq, oc, helm, podman, docker, skopeo, ssh - -> **See [docs/prerequisites.md](../../docs/prerequisites.md)** for the complete tool requirements by skill, check commands, and installation instructions. - -### Step 3: Check OpenShift Connectivity (if TARGET includes openshift) - -```bash -# Check if logged in -oc whoami - -# Check current project -oc project - -# Check permissions -oc auth can-i create deployments -oc auth can-i create buildconfigs -oc auth can-i create imagestreams -``` - -### Step 4: Check Container Runtime (if TARGET includes containers) - -```bash -# Check Podman -podman info --format '{{.Host.OS}} {{.Host.Arch}}' - -# Or Docker -docker info --format '{{.OSType}} {{.Architecture}}' - -# Check if can pull images -podman pull --quiet registry.access.redhat.com/ubi9/ubi-minimal:latest || echo "WARN: Cannot pull images" -``` - -### Step 5: Generate Report - -Present results in this format: - -```markdown -## Environment Validation Report - -### Core Tools - -| Tool | Status | Version | -|------|--------|---------| -| git | OK | 2.43.0 | -| curl | OK | 8.5.0 | -| jq | OK | 1.7.1 | -| oc | OK | 4.14.0 | -| helm | OK | 3.14.0 | -| podman | OK | 4.9.0 | -| skopeo | MISSING | - | -| ssh | OK | OpenSSH_9.6 | - -### OpenShift Cluster - -| Check | Status | Details | -|-------|--------|---------| -| Logged in | OK | user@cluster.example.com | -| Project | OK | my-project | -| Create Deployments | OK | Allowed | -| Create BuildConfigs | OK | Allowed | -| Create ImageStreams | OK | Allowed | - -### Container Runtime - -| Check | Status | Details | -|-------|--------|---------| -| Runtime | OK | Podman 4.9.0 | -| Pull images | OK | Can access registries | - ---- - -### Summary - -**Ready for:** /detect-project, /s2i-build, /deploy, /helm-deploy, /containerize-deploy - -**Missing tools for:** -- /recommend-image (dynamic mode) - Install: `sudo dnf install skopeo` - -### Quick Fix Commands - -```bash -# Install missing tools -sudo dnf install skopeo -``` -``` - -### Step 6: Offer Next Steps - -```markdown -## Next Steps - -Your environment is ready for deployment. - -Would you like to: -1. Run `/detect-project` to analyze your application -2. Run `/containerize-deploy` for end-to-end deployment -3. See detailed prerequisites documentation - -Select an option or describe what you'd like to do: -``` - ---- - -## Validation Status Indicators - -| Status | Meaning | -|--------|---------| -| OK | Tool installed and working | -| MISSING | Tool not found in PATH | -| ERROR | Tool found but not working | -| WARN | Optional tool missing | -| SKIP | Check skipped (not in scope) | - -## Error Handling - -### Tool Not Found - -```markdown -**Missing: [tool-name]** - -This tool is required for [skill-names]. - -See [docs/prerequisites.md](../../docs/prerequisites.md) for installation commands by OS. -``` - -### Cluster Connection Failed - -```markdown -**OpenShift cluster not accessible** - -You are not logged in to an OpenShift cluster. - -To connect: -1. Get login command from OpenShift console -2. Run: `oc login ` - -Or set KUBECONFIG: -```bash -export KUBECONFIG=/path/to/kubeconfig -``` -``` - -### Permission Denied - -```markdown -**Insufficient permissions in namespace [namespace]** - -You need 'edit' or 'admin' role to deploy applications. - -Options: -1. Contact cluster admin for permissions -2. Switch to a different namespace: `oc project ` -3. Create a new project: `oc new-project ` -``` - ---- - -## Dependencies - -### Required MCP Servers -- None required (uses Bash to check tool availability and cluster connectivity) - -### Related Skills -- `/containerize-deploy` - End-to-end deployment workflow (validate environment first) -- `/s2i-build` - S2I build requiring oc and cluster access -- `/deploy` - Deployment requiring oc and cluster access - -### Reference Documentation -- [docs/prerequisites.md](../../docs/prerequisites.md) - Comprehensive tool requirements by skill, installation commands, cluster access verification diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__cve-impact/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 1092dd1d..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- pass rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/SKILL.md b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/SKILL.md deleted file mode 100644 index 9671bc2d..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/SKILL.md +++ /dev/null @@ -1,409 +0,0 @@ ---- -name: cve-impact -description: | - **CRITICAL**: Use for ALL CVE discovery and listing. DO NOT call get_cves directly. - - Use when: "show critical CVEs", "CVEs on hostname X", "remediatable vulnerabilities", "impact of CVE-X", risk assessment. - - NOT for remediation (use `/remediation`). - - System-level: FIRST reply = pagination prompt (Step -1). Parsing: references/01-cve-response-parser.py. ---- - -# CVE Impact Analysis Skill - -This skill helps SREs analyze CVE vulnerabilities to understand their impact on systems before creating remediation playbooks. - -**🚨 SYSTEM-LEVEL (CVEs on device X)**: Your **first reply** to the user MUST be the pagination prompt (Step -1). Do NOT call `inventory__find_host_by_name` or `vulnerability__get_system_cves` until the user responds. Do not validate MCP or resolve hostname first—HITL comes first. - -**Integration with Remediation Skill**: The `/remediation` skill orchestrates this skill as part of its Step 1 (Impact Analysis) workflow for complex remediation scenarios. For simple standalone impact analysis, you can invoke this skill directly. - -## Invocation Note (Host-Specific) - -When invoked by another skill (e.g. remediation), use the Skill tool—do NOT use "Task Output" with the skill name as task ID. That causes "No task found with ID: cve-impact". See [skill-invocation.md](../../docs/references/skill-invocation.md). - -## Prerequisites - -**Required MCP Servers**: `lightspeed-mcp` ([setup guide](https://console.redhat.com/)) - -**Required MCP Tools**: -- `get_cves` (from lightspeed-mcp) - List/query CVEs by severity -- `get_cve` (from lightspeed-mcp) - Get specific CVE details -- `get_cve_systems` (from lightspeed-mcp) - Find systems affected by CVEs -- `get_system_cves` (from lightspeed-mcp) - List CVEs affecting a specific system (uses `system_uuid` only) - -**Required Environment Variables**: -- `LIGHTSPEED_CLIENT_ID` - Red Hat Lightspeed service account client ID -- `LIGHTSPEED_CLIENT_SECRET` - Red Hat Lightspeed service account secret - -### Prerequisite Validation - -**CRITICAL**: Before executing any operations, execute the `/mcp-lightspeed-validator` skill to verify MCP server availability. - -**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-lightspeed-validator/SKILL.md#validation-freshness-policy). - -**How to invoke**: Execute the `/mcp-lightspeed-validator` skill - -**Handle validation result**: -- **If validation PASSED**: Continue with CVE impact analysis -- **If validation PARTIAL**: Warn user and ask to proceed -- **If validation FAILED**: Stop execution, provide setup instructions - -## When to Use This Skill - -**Use this skill directly when you need**: -- Standalone CVE impact analysis without remediation planning -- Quick risk assessment for a single CVE -- Understanding which systems are affected by a CVE -- CVE severity assessments for change management documentation -- Risk assessment reports for management - -**Use the `/remediation` skill when you need**: -- CVE analysis followed by remediation playbook generation -- Complex workflows involving multiple CVEs and systems -- Integrated risk assessment + remediation planning + execution guidance -- Batch remediation across infrastructure -- End-to-end CVE management (analysis → validation → remediation → verification) - -**To invoke**: Execute the `/remediation` skill - -**How they work together**: The `/remediation` skill orchestrates this skill as part of its comprehensive workflow, combining impact analysis with context gathering, playbook generation, and execution guidance. - -## Workflow - -### Step -1: System-Level Gate — HITL FIRST (MANDATORY) - -**If the user asked for CVEs on a device** (e.g. "CVEs on ip-172-31-32-201", "remediatable CVEs on hostname X", "most critical CVEs on system Y"): - -**Your first response to the user MUST be the pagination prompt below. Do not run Step 0, do not call `inventory__find_host_by_name`, do not call `vulnerability__get_system_cves` until the user responds.** - -Reply to the user with: - -``` -To fetch remediatable CVEs on this system, I will: -- Paginate through vulnerability__get_system_cves (limit=100 per page) -- Filter each page for advisory_available === true -- Systems often have 1,700+ CVEs (~18 API calls) - -⚠️ First page only often returns 0 remediatable CVEs—they may be on any page. For "remediatable" queries, recommend "all pages". - -Options: -- **First page only**: Fetch 100 CVEs, filter for remediatable (may be 0) -- **All pages**: Fetch until no more results (recommended for remediatable) -- **N pages**: Fetch up to N pages (e.g. "3 pages" = up to 300 CVEs scanned) - -How would you like to proceed? (first page / all pages / N pages) -``` - -**Wait for the user to respond.** Only after they reply may you proceed to Step 0. - -**If account-level** (e.g. "CVEs on my account"): Skip this step, go to Step 0. - ---- - -### Step 0: Validate Lightspeed MCP Prerequisites - -**Action**: Execute the `/mcp-lightspeed-validator` skill - -**Note**: Can skip if validation was performed earlier in this session and succeeded. See [Validation Freshness Policy](../mcp-lightspeed-validator/SKILL.md#validation-freshness-policy). - -**How to invoke**: Execute the `/mcp-lightspeed-validator` skill - -**Handle validation result**: -- **If validation PASSED**: Continue to Step 1 -- **If validation PARTIAL**: Warn user and ask to proceed -- **If validation FAILED**: Stop execution, user must set up MCP server - -### Step 1: CVE Discovery — Choose Flow - -Select the appropriate flow based on user request. - -| Flow | When to Use | Flow File | -|------|-------------|-----------| -| **Account-level** | devices=all (account-wide CVEs) | [01-account-cves.md](flows/01-account-cves.md) | -| **System-level (all CVEs)** | devices=selected, remediation=does not matter | [02-system-all-cves.md](flows/02-system-all-cves.md) | -| **System-level (remediatable)** | devices=selected, remediation=available | [03-system-remediatable-cves.md](flows/03-system-remediatable-cves.md) | - -**Variable mapping**: -- **devices**: all → account flow; selected → system flow -- **severities**: all, most critical, or selected → parameter/filter in each flow -- **remediation**: available → remediatable flow; does not matter → all-cves flow - ---- - -#### CRITICAL: System-Level — HITL FIRST (Before Any Other Action) - -**For system-level flows (02 or 03)**: Your **first** action MUST be to display the HITL prompt below and **wait for user confirmation**. Do NOT resolve hostname, do NOT call any MCP tool, until the user responds. - -**Order of operations**: -1. **STOP. Display HITL prompt. Wait for user.** -2. Only after user confirms → document consultation → resolve hostname → call `vulnerability__get_system_cves` - -*For remediatable CVEs on system (flow 03):* -``` -To fetch remediatable CVEs on this system, I will: -- Paginate through vulnerability__get_system_cves (limit=100 per page) -- Filter each page for advisory_available === true -- Systems often have 1,700+ CVEs (~18 API calls) - -⚠️ First page only often returns 0 remediatable CVEs—they may be on any page. For "remediatable" queries, recommend "all pages". - -Options: -- **First page only**: Fetch 100 CVEs, filter for remediatable (may be 0) -- **All pages**: Fetch until no more results (recommended for remediatable) -- **N pages**: Fetch up to N pages (e.g. "3 pages" = up to 300 CVEs scanned) - -How would you like to proceed? (first page / all pages / N pages) -``` - -*For all CVEs on system (flow 02):* -``` -This system may have many CVEs. I will paginate through vulnerability__get_system_cves (limit=100 per page). - -Options: -- **First page only**: Fetch 100 CVEs, then stop (quick overview) -- **All pages**: Fetch until no more results (systems with 1,700+ CVEs may require ~18 API calls) -- **N pages**: Fetch up to N pages (e.g. "3 pages" = 300 CVEs) - -How would you like to proceed? (first page / all pages / N pages) -``` - -**Handle response**: Wait for user reply. Only after user confirms (and specifies strategy) may you proceed to resolve hostname and call `vulnerability__get_system_cves`. If user says "no" or cancels, stop execution. - -**Anti-pattern**: Do NOT call `vulnerability__get_system_cves` or `inventory__find_host_by_name` before completing HITL. Calling with only the first page (limit=100, no offset loop) misses remediatable CVEs on later pages. - ---- - -**Action**: Read and follow the selected flow file. For system-level, HITL is Step 1 (before all other steps). - -### Step 2: CVE Information Retrieval (For Specific CVE Analysis) - -**CRITICAL**: Document consultation MUST happen BEFORE tool invocation. - -**Document Consultation** (REQUIRED - Execute FIRST): -1. **Action**: Read [cvss-scoring.md](../../docs/references/cvss-scoring.md) using the Read tool to understand CVSS score interpretation and severity mapping -2. **Output to user**: "I consulted [cvss-scoring.md](../../docs/references/cvss-scoring.md) to understand CVSS score interpretation and severity mapping." - -**MCP Tool**: `get_cve` or `vulnerability__get_cve` (from lightspeed-mcp) - -**Parameters**: -- `cve_id`: Exact CVE identifier from user query (format: `"CVE-YYYY-NNNNN"`) - - Example: `"CVE-2024-1234"` -- `include_details`: `true` (retrieve complete metadata including CVSS vector, affected packages, references) - -**Expected Output**: Comprehensive CVE metadata including CVSS score, severity, attack vector, affected packages, remediation status - -Retrieve comprehensive CVE metadata: -``` -CVE ID: CVE-YYYY-NNNNN -CVSS Score: X.X (Base score from NIST) -Severity: Critical/High/Medium/Low -Attack Vector: Network/Adjacent/Local/Physical -Attack Complexity: Low/High -Privileges Required: None/Low/High -User Interaction: None/Required -Scope: Unchanged/Changed -Confidentiality Impact: None/Low/High -Integrity Impact: None/Low/High -Availability Impact: None/Low/High - -Description: [CVE description from NVD/Red Hat] -Affected Packages: [List of packages and versions] -Published Date: YYYY-MM-DD -Last Modified: YYYY-MM-DD -``` - -### Step 3: Affected Systems Identification - -**CRITICAL**: Document consultation MUST happen BEFORE tool invocation. - -**Document Consultation** (REQUIRED - Execute FIRST): -1. **Action**: Read [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) using the Read tool to understand the system identification response format -2. **Output to user**: "I consulted vulnerability-logic.md to understand the system identification response format." - -**MCP Tool**: `get_cve_systems` or `vulnerability__get_cve_systems` (from lightspeed-mcp) - -**Parameters**: -- `cve_id`: Exact CVE identifier (format: `"CVE-YYYY-NNNNN"`) - - Example: `"CVE-2024-1234"` -- `include_patched`: `false` (exclude already-patched systems to focus on vulnerable systems) - -**Expected Output**: List of affected systems with UUID, hostname, IP address, package version, vulnerability status - -Identify which systems in your infrastructure are affected by the CVE: -``` -Affected Systems: -- system-uuid-1 (hostname: web-server-01, IP: 10.0.1.10) - - Package: httpd-2.4.37-1.el8 - - Status: Vulnerable - -- system-uuid-2 (hostname: web-server-02, IP: 10.0.1.11) - - Package: httpd-2.4.37-1.el8 - - Status: Vulnerable - -Total Affected Systems: N -``` - -### Step 4: System Classification - -**CRITICAL**: Document consultation MUST happen BEFORE classification logic. - -**Document Consultation** (REQUIRED - Execute FIRST): -1. **Action**: Read [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) using the Read tool to understand system tagging and classification strategies -2. **Output to user**: "I consulted vulnerability-logic.md to understand system tagging and classification strategies." - -Classify affected systems by environment and criticality based on tags from Red Hat Lightspeed: -``` -System Classification: -- Production systems: N (highest priority) -- Staging systems: M (test before prod) -- Development systems: K (lowest priority) -- Bare metal: X systems -- Virtual machines: Y systems -``` - -### Step 5: Risk Assessment - -**CRITICAL**: Document consultation MUST happen BEFORE risk assessment. - -**Document Consultation** (REQUIRED - Execute FIRST): -1. **Action**: Read [cvss-scoring.md](../../docs/references/cvss-scoring.md) using the Read tool to understand risk prioritization based on CVSS vectors -2. **Output to user**: "I consulted [cvss-scoring.md](../../docs/references/cvss-scoring.md) to understand risk prioritization based on CVSS vectors." - -Provide a risk assessment based on: -- CVSS score and severity -- Number of affected systems -- System criticality (production vs non-production) -- Attack vector and exploitability -- Known exploits in the wild - -**Output Format**: -``` -Risk Assessment Summary -━━━━━━━━━━━━━━━━━━━━━━━ - -Overall Risk Level: Critical/High/Medium/Low - -Factors: -✓ CVSS Score: X.X (Severity: High) -✓ Affected Systems: N systems -✓ Environment: Production (Critical) -✓ Attack Vector: Network (Remote exploitable) -✓ Exploitability: Proof-of-concept exists -✓ User Interaction: None required - -Recommendation: Immediate remediation recommended -Priority: P0 (within 24 hours) / P1 (within 7 days) / P2 (within 30 days) -``` - -### Step 6: Impact Analysis - -Analyze business impact (service, confidentiality/integrity/availability, compliance). - -### Step 7: Remediation Readiness Check - -Check if automated playbook or manual steps are available. - -## Output and Examples - -**Read [references/03-output-templates.md](references/03-output-templates.md)** for report format. -**Read [references/04-examples.md](references/04-examples.md)** for query-type examples and remediation integration. - -## Error Handling - -**Read [references/05-error-handling.md](references/05-error-handling.md)** for CVE not found, no affected systems, and Lightspeed tool failures. - -## Reference Files - -| File | Use When | -|------|----------| -| [01-cve-response-parser.py](references/01-cve-response-parser.py) | Parse/filter MCP vulnerability responses | -| [02-cve-parsing-guide.md](references/02-cve-parsing-guide.md) | Parser invocation, filter options | -| [03-output-templates.md](references/03-output-templates.md) | Report format | -| [04-examples.md](references/04-examples.md) | Query-type examples | -| [05-error-handling.md](references/05-error-handling.md) | CVE not found, no systems, Lightspeed failures | -| [lightspeed-mcp-tool-failures.md](../../docs/references/lightspeed-mcp-tool-failures.md) | explain_cves dnf_modules workaround | - -## Parsing MCP Responses - -**REQUIRED**: Use the skill's parser script for all vulnerability response parsing. Do NOT use jq, inline Python, or other ad-hoc JSON parsing. - -**Do NOT generate inline Python** to aggregate multiple page files—the parser accepts multiple file paths and produces aggregated reports. - -**Read** [references/02-cve-parsing-guide.md](references/02-cve-parsing-guide.md) for: -- Parser location: `references/01-cve-response-parser.py` -- Single page: `python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py ` -- Multiple pages: `python3 .../01-cve-response-parser.py page1.json page2.json page3.json ...` (merges, dedupes, aggregated report) -- Filter options: `FILTER_REMEDIATABLE=1`, `FILTER_IMPACT=Critical,Important` -- Report format: `OUTPUT=report`, `SYSTEM_NAME=hostname` for aggregated multi-page reports - -Save each MCP tool result to a file, then run the parser with one or more paths. Use parser output for summary tables and reports. - -## Best Practices - -1. **Always start with risk assessment** before deciding on remediation -2. **Check for known exploits** using CVE description and references -3. **Consider business impact** not just CVSS scores -4. **Group related CVEs** for batch remediation when possible -5. **Document findings** for compliance and audit purposes -6. **Test in staging first** for high-impact changes - -## Dependencies - -### Required MCP Servers -- `lightspeed-mcp` - Red Hat Lightspeed platform access - -### Required MCP Tools -- `get_cves` (from lightspeed-mcp) - List/query CVEs by severity and filters - - Parameters: severity (array), sort_by (string), limit (number) - - Returns: List of CVEs with CVSS scores, severity, affected systems count - -- `get_cve` (from lightspeed-mcp) - Get specific CVE details - - Parameters: cve_id (string, format CVE-YYYY-NNNNN), include_details (boolean) - - Returns: Complete CVE metadata with CVSS vector, affected packages, references - -- `get_cve_systems` (from lightspeed-mcp) - Find systems affected by CVE - - Parameters: cve_id (string), include_patched (boolean) - - Returns: List of affected systems with UUID, hostname, package version, status - -- `get_system_cves` (from lightspeed-mcp) - List CVEs affecting a specific system - - Parameters: **system_uuid** (string, required) - use `system_uuid`, NOT `system_id` - - Does NOT support: impact, limit, severity filters - filter results client-side - - Returns: List of CVEs affecting the system - -### Related Skills -- `mcp-lightspeed-validator` - **PREREQUISITE** - Validates Lightspeed MCP server before operations - - Use before: ALL cve-impact operations (Step 0 in workflow) - - Purpose: Ensures MCP server is available before attempting tool calls - -- `cve-validation` - Validate CVE IDs before impact analysis - - Use before: Step 2 if CVE format/existence is uncertain - - Purpose: Confirms CVE is valid and remediable before expensive analysis - -- `system-context` - Get detailed system configuration after identifying affected systems - - Use after: Step 3 when deeper system investigation is needed - - Purpose: Understand deployment architecture for remediation planning - -- `fleet-inventory` - Get comprehensive fleet information before CVE analysis - - Use before: Step 1 when starting from fleet discovery - - Purpose: Understand overall infrastructure before assessing CVE impact - -### Reference Documentation -- [cvss-scoring.md](../../docs/references/cvss-scoring.md) - CVSS score interpretation and severity mapping -- [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) - CVE severity classification and filtering -- [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) - CVE severity classification, system identification, and filtering -- [references/02-cve-parsing-guide.md](references/02-cve-parsing-guide.md) - Parse MCP vulnerability responses; use the parser script instead of generating inline Python - -## Tools Reference - -This skill primarily uses: -- `get_cve` (vulnerability toolset) - Get details about a specific CVE -- `get_cve_systems` (vulnerability toolset) - Get list of systems affected by a CVE -- `get_cves` (vulnerability toolset) - Get list of all CVEs affecting the account (optional) -- `get_system_cves` (vulnerability toolset) - Get list of CVEs affecting a specific system - - **CRITICAL**: Use `system_uuid` (required), NOT `system_id` - - Does NOT support `impact`, `limit`, or severity filters - filter client-side -- `inventory__find_host_by_name` (inventory toolset) - Resolve hostname to system UUID before get_system_cves -- `get_host_details` (inventory toolset) - Get detailed system information (optional) - -All tools are provided by the lightspeed-mcp MCP server configured in `.mcp.json`. diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/01-account-cves.md b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/01-account-cves.md deleted file mode 100644 index d981a9e9..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/01-account-cves.md +++ /dev/null @@ -1,92 +0,0 @@ -# Flow: Account-Level CVEs - -**Scope**: devices=all (account-wide) | severities=all, most critical, or selected | remediation=available or does not matter - -**Tool**: `vulnerability__get_cves` (single request; no offset pagination in API) - -## When to Use - -- "What are the critical vulnerabilities on my account?" -- "Show me high-severity CVEs" -- "List all vulnerabilities affecting my account" -- "Which CVEs can I remediate?" (account-wide) - -## Step 1: Document Consultation (REQUIRED - Execute FIRST) - -1. **Action**: Read [vulnerability-logic.md](../../../docs/insights/vulnerability-logic.md) using the Read tool -2. **Output to user**: "I consulted [vulnerability-logic.md](../../../docs/insights/vulnerability-logic.md) to understand CVE severity classification and filtering." - -## Step 2: HITL Checkpoint — Pagination / Limit - -**Before** calling the API, display and wait for confirmation: - -``` -For account-level CVEs, I will fetch up to {limit} CVEs per request. -The API returns a single page (no offset pagination). - -Options: -- Default: limit=20 (top CVEs by CVSS score) -- You may request a different limit (e.g. 10, 50) before I proceed - -Proceed with limit=20? (yes/no) Or specify a different limit. -``` - -**Handle response**: -- **yes** or no limit specified → use `limit=20` -- **User specifies N** → use `limit=N` -- **no** → Stop execution - -## Step 3: MCP Tool Invocation - -**Tool**: `vulnerability__get_cves` (from lightspeed-mcp) - -**Parameters**: - -| Parameter | Severity=all | Severity=most critical | Severity=selected | -|-----------|-------------|------------------------|-------------------| -| `impact` | `"7,6,5,4"` | `"7,6"` | e.g. `"7"` (Critical only) or `"6,5"` | - -| Parameter | Remediation=available | Remediation=does not matter | -|-----------|----------------------|------------------------------| -| `advisory_available` | `"true"` | `"true,false"` | - -**Common parameters**: -- `sort`: `"-cvss_score"` (descending by CVSS) -- `limit`: From HITL (default 20) - -**Example** (most critical, remediatable): -``` -vulnerability__get_cves( - impact="7,6", - sort="-cvss_score", - limit=20, - advisory_available="true" -) -``` - -**Example** (all severities, remediation doesn't matter): -``` -vulnerability__get_cves( - impact="7,6,5,4", - sort="-cvss_score", - limit=20, - advisory_available="true,false" -) -``` - -## Step 4: After Listing - -- **Parse response** (if needed): Use [references/01-cve-response-parser.py](../references/01-cve-response-parser.py). Do NOT use jq or inline Python. See [02-cve-parsing-guide.md](../references/02-cve-parsing-guide.md). -- Sort by CVSS score (highest first) or by affected system count -- Provide summary table: CVE ID, severity, affected systems count, remediation availability -- Offer to analyze a specific CVE (see [SKILL.md](../SKILL.md) — Step 2: CVE Information Retrieval) -- Offer to create remediation plan (invoke `/remediation` skill) - -## Impact Level Reference - -| impact | Severity | -|--------|----------| -| 7 | Critical | -| 6 | High | -| 5 | Important | -| 4 | Moderate | diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/02-system-all-cves.md b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/02-system-all-cves.md deleted file mode 100644 index 6a819454..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/02-system-all-cves.md +++ /dev/null @@ -1,89 +0,0 @@ -# Flow: System-Level CVEs (All — Remediation Does Not Matter) - -**Scope**: devices=selected | severities=all, most critical, or selected | remediation=does not matter - -**Tool**: `vulnerability__get_system_cves` (paginated via `limit` + `offset`) - -**BLOCKING**: HITL (Step -1 in SKILL.md) MUST be your first action. Reply to the user with the pagination prompt before ANY tool call. Do NOT proceed to this flow until the user has responded to the HITL prompt. - -## When to Use - -- "What CVEs affect hostname X?" -- "What vulnerabilities are on system Y?" -- "Show CVEs for ip-172-31-32-201.eu-west-3.compute.internal" -- (When user does NOT specifically ask for remediatable-only) - -## Step 1: HITL — Pagination Strategy (Done in SKILL.md Step -1) - -**If you have not yet replied to the user with the pagination prompt**: Stop. Go to [SKILL.md](../SKILL.md) Step -1. Your first reply to the user must be the prompt below. Do NOT call `inventory__find_host_by_name` or `vulnerability__get_system_cves` until the user responds. - -``` -This system may have many CVEs. I will paginate through vulnerability__get_system_cves (limit=100 per page). - -Options: -- **First page only**: Fetch 100 CVEs, then stop (quick overview) -- **All pages**: Fetch until no more results (systems with 1,700+ CVEs may require ~18 API calls) -- **N pages**: Fetch up to N pages (e.g. "3 pages" = 300 CVEs) - -How would you like to proceed? (first page / all pages / N pages) -``` - -**Handle response**: -- **first page** → Single call: `limit=100`, `offset=0`; stop after first response -- **all pages** → Loop: `offset=0, 100, 200, ...` until `len(data) < 100` or empty -- **N pages** → Loop N times: `offset=0`, then `offset=100`, ... up to N pages -- **no** or cancel → Stop execution - -## Step 2: Document Consultation (REQUIRED - Execute AFTER HITL) - -1. **Action**: Read [insights-api.md](../../../docs/insights/insights-api.md) using the Read tool -2. **Output to user**: "I consulted [insights-api.md](../../../docs/insights/insights-api.md) to understand system identification." - -## Step 3: Resolve Hostname to System UUID - -**If user provided hostname** (not UUID): -- **Tool**: `inventory__find_host_by_name` (preferred) or `inventory__list_hosts` -- **inventory__list_hosts**: Use `per_page` (integer), NOT `page_size`; pass `display_name=""` if no filter -- If multiple matches: ask user to disambiguate or use first match with a note - -## Step 4: MCP Tool Invocation - -**Tool**: `vulnerability__get_system_cves` (from lightspeed-mcp) - -**Parameters**: -- `system_uuid`: Required (from Step 2) -- `limit`: `100` (fewer pages) -- `offset`: `0`, `100`, `200`, ... per pagination strategy - -**First call** (to get total estimate if available): -``` -vulnerability__get_system_cves(system_uuid="", limit=100, offset=0) -``` -Check `meta.count` in response for total estimate. - -**Pagination loop** (if user chose "all pages" or "N pages"): -``` -offset = 0 -all_cves = [] -while (strategy allows): - result = vulnerability__get_system_cves(system_uuid="...", limit=100, offset=offset) - all_cves.extend(result.data) - if len(result.data) < 100: break - offset += 100 -``` - -## Step 5: Filter by Severity (Client-Side) - -`get_system_cves` does NOT support severity filters. Filter results client-side: - -| Severity choice | Filter | -|-----------------|--------| -| all | No filter | -| most critical | Keep items where severity in (Critical, High) | -| selected | Keep items matching user-specified severity | - -## Step 6: After Retrieval - -- **Parse response**: Use [references/01-cve-response-parser.py](../references/01-cve-response-parser.py). Do NOT use jq or inline Python. See [02-cve-parsing-guide.md](../references/02-cve-parsing-guide.md). -- Sort by CVSS score (highest first) -- Provide summary table; offer to analyze specific CVEs or create remediation plan (`/remediation` skill) diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/03-system-remediatable-cves.md b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/03-system-remediatable-cves.md deleted file mode 100644 index a1351e79..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/flows/03-system-remediatable-cves.md +++ /dev/null @@ -1,96 +0,0 @@ -# Flow: System-Level CVEs (Remediatable Only) - -**Scope**: devices=selected | severities=all, most critical, or selected | remediation=available - -**Tool**: `vulnerability__get_system_cves` (paginated; filter client-side for `advisory_available === true`) - -**BLOCKING**: HITL (Step -1 in SKILL.md) MUST be your first action. Reply to the user with the pagination prompt before ANY tool call. Do NOT proceed to this flow until the user has responded to the HITL prompt. - -## When to Use - -- "Remediatable CVEs on system X" -- "CVEs with available remediation on device Y" -- "Which CVEs can I fix on hostname Z?" - -**CRITICAL**: `get_system_cves` does NOT support `advisory_available` as a request parameter. We must paginate through ALL CVEs and filter client-side for `attributes.advisory_available === true`. Do NOT use `get_cves` + `get_cve_systems` per CVE—does not scale. - -## Step 1: HITL — Pagination Strategy (Done in SKILL.md Step -1) - -**If you have not yet replied to the user with the pagination prompt**: Stop. Go to [SKILL.md](../SKILL.md) Step -1. Your first reply to the user must be the prompt below. Do NOT call `inventory__find_host_by_name` or `vulnerability__get_system_cves` until the user responds. - -``` -To fetch remediatable CVEs on this system, I will: -- Paginate through vulnerability__get_system_cves (limit=100 per page) -- Filter each page for advisory_available === true -- Systems often have 1,700+ CVEs (~18 API calls) - -⚠️ First page only often returns 0 remediatable CVEs—they may be on any page. For "remediatable" queries, recommend "all pages". - -Options: -- **First page only**: Fetch 100 CVEs, filter for remediatable (may be 0) -- **All pages**: Fetch until no more results (recommended for remediatable) -- **N pages**: Fetch up to N pages (e.g. "3 pages" = up to 300 CVEs scanned) - -How would you like to proceed? (first page / all pages / N pages) -``` - -**Handle response**: -- **first page** → Single call: `limit=100`, `offset=0`; filter for remediatable; stop -- **all pages** → Loop until empty; filter each page for remediatable -- **N pages** → Loop N times; filter each page for remediatable -- **no** or cancel → Stop execution - -## Step 2: Document Consultation (REQUIRED - Execute AFTER HITL) - -1. **Action**: Read [insights-api.md](../../../docs/insights/insights-api.md) using the Read tool -2. **Output to user**: "I consulted [insights-api.md](../../../docs/insights/insights-api.md) to understand system identification." - -## Step 3: Resolve Hostname to System UUID - -**If user provided hostname** (not UUID): -- **Tool**: `inventory__find_host_by_name` (preferred) or `inventory__list_hosts` -- **inventory__list_hosts**: Use `per_page` (integer), NOT `page_size`; pass `display_name=""` if no filter -- If multiple matches: ask user to disambiguate or use first match with a note - -## Step 4: MCP Tool Invocation - -**Tool**: `vulnerability__get_system_cves` (from lightspeed-mcp) - -**Parameters**: -- `system_uuid`: Required (from Step 2) -- `limit`: `100` -- `offset`: `0`, `100`, `200`, ... per pagination strategy - -**First call** (to get total estimate if available): -``` -vulnerability__get_system_cves(system_uuid="", limit=100, offset=0) -``` - -**Pagination loop** (filter for remediatable): -``` -offset = 0 -all_remediatable = [] -while (strategy allows): - result = vulnerability__get_system_cves(system_uuid="...", limit=100, offset=offset) - for item in result.data: - if item.attributes.advisory_available is True: - all_remediatable.append(item) - if len(result.data) < 100: break - offset += 100 -``` - -## Step 5: Filter by Severity (Client-Side) - -After filtering for remediatable, optionally filter by severity: - -| Severity choice | Filter | -|-----------------|--------| -| all | No additional filter | -| most critical | Keep items where severity in (Critical, High) | -| selected | Keep items matching user-specified severity | - -## Step 6: After Retrieval - -- **Parse response**: Use [references/01-cve-response-parser.py](../references/01-cve-response-parser.py) with `FILTER_REMEDIATABLE=1`. Do NOT use jq or inline Python. See [02-cve-parsing-guide.md](../references/02-cve-parsing-guide.md). -- Sort by CVSS score (highest first) -- Provide summary table; offer to analyze specific CVEs or create remediation plan (`/remediation` skill) diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/01-cve-response-parser.py b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/01-cve-response-parser.py deleted file mode 100644 index d9235f25..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/01-cve-response-parser.py +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env python3 -""" -Parse Red Hat Lightspeed vulnerability MCP tool responses. - -Handles: vulnerability__get_system_cves, vulnerability__get_cves - -Usage: - python3 01-cve-response-parser.py < response.json - python3 01-cve-response-parser.py /path/to/response.json - python3 01-cve-response-parser.py page1.json page2.json page3.json # Multiple pages → aggregated report - -Options (env vars or args): - FILTER_REMEDIATABLE=1 - Only CVEs with advisory_available=true - FILTER_IMPACT=Critical,Important - Only these severities (comma-separated) - SORT_BY=cvss|impact|public_date - OUTPUT=summary|table|json|report - SYSTEM_NAME=hostname - For report header (when OUTPUT=report) - PAGES_SCANNED=5 - For report header (when OUTPUT=report, multiple files) -""" -import json -import sys -import os -from pathlib import Path - -# Response structure: {"result": {"data": [...]}, "meta": {"count": N}} -# Each CVE: {"id": "CVE-...", "type": "cve", "url": "...", "attributes": {...}} -# attributes: advisory_available, impact, cvss3_score, cvss2_score, description, synopsis, public_date, business_risk, etc. - -IMPACT_ORDER = {"Critical": 0, "High": 1, "Important": 2, "Moderate": 3, "Low": 4, "None": 5} - - -def load_response(src): - """Load JSON from file path or stdin.""" - if src and src != "-": - with open(src, "r") as f: - return json.load(f) - return json.load(sys.stdin) - - -def extract_cves(data): - """Extract CVE list from Lightspeed response (handles result.data or result.results).""" - result = data.get("result", data) - cves = result.get("data", result.get("results", [])) - meta = data.get("meta", {}) - total = meta.get("count", meta.get("total", len(cves))) - return cves, total - - -def get_attr(cve, key, default=None): - """Get attribute from CVE (handles nested attributes).""" - attrs = cve.get("attributes", cve) - return attrs.get(key, attrs.get(key.replace("_", "-"), default)) - - -def filter_cves(cves, remediatable_only=None, impact_filter=None): - """Filter CVEs by advisory_available and/or impact.""" - filtered = [] - for cve in cves: - if remediatable_only and not get_attr(cve, "advisory_available", False): - continue - if impact_filter: - impact = get_attr(cve, "impact", "") - if impact not in impact_filter: - continue - filtered.append(cve) - return filtered - - -def sort_cves(cves, sort_by="cvss"): - """Sort CVEs by cvss (desc), impact, or public_date.""" - def key_fn(cve): - if sort_by == "cvss": - score = get_attr(cve, "cvss3_score") or get_attr(cve, "cvss2_score") or "0" - return -(float(score) if score else 0) - if sort_by == "impact": - return IMPACT_ORDER.get(get_attr(cve, "impact", "None"), 99) - if sort_by == "public_date": - return get_attr(cve, "public_date", "") or "" - return 0 - return sorted(cves, key=key_fn) - - -def format_summary(cves, total_in_api, remediatable_only=False, impact_filter=None): - """Print summary counts by impact and remediation.""" - by_impact = {} - by_remediation = {"with_remediation": 0, "without": 0} - for cve in cves: - impact = get_attr(cve, "impact", "None") or "None" - by_impact[impact] = by_impact.get(impact, 0) + 1 - if get_attr(cve, "advisory_available", False): - by_remediation["with_remediation"] += 1 - else: - by_remediation["without"] += 1 - - lines = [ - "CVE Response Summary", - "=" * 60, - f"Total in this page/batch: {len(cves)}", - f"Total in API (meta.count): {total_in_api}", - ] - if remediatable_only or impact_filter: - lines.append(f"Filter: remediatable={remediatable_only}, impact={impact_filter}") - lines.append("") - lines.append("By Impact:") - for impact in ["Critical", "Important", "High", "Moderate", "Low", "None"]: - if impact in by_impact: - lines.append(f" {impact}: {by_impact[impact]}") - lines.append("") - lines.append("By Remediation:") - lines.append(f" With advisory: {by_remediation['with_remediation']}") - lines.append(f" Without: {by_remediation['without']}") - return "\n".join(lines) - - -def format_table(cves, limit=20): - """Print CVE table (CVE ID, CVSS, Impact, Remediation).""" - lines = [ - "CVE ID | CVSS | Impact | Remediation", - "-" * 60, - ] - for cve in cves[:limit]: - cve_id = cve.get("id", "?") - cvss = get_attr(cve, "cvss3_score") or get_attr(cve, "cvss2_score") or "-" - impact = get_attr(cve, "impact", "-") or "-" - rem = "Yes" if get_attr(cve, "advisory_available", False) else "No" - lines.append(f"{cve_id:<19} | {str(cvss):<6} | {impact:<9} | {rem}") - if len(cves) > limit: - lines.append(f"... and {len(cves) - limit} more") - return "\n".join(lines) - - -def format_report(cves, total_in_api, system_name=None, pages_scanned=None): - """Print aggregated report format (for multi-page results).""" - lines = ["=" * 80] - title = "CVEs WITH AVAILABLE REMEDIATION" - if system_name: - title = f"{title.upper()} — System: {system_name}" - lines.append(title) - if pages_scanned: - lines.append(f"Scanned: {pages_scanned} page(s)") - lines.append("=" * 80) - lines.append("") - if not cves: - lines.append("No CVEs with available remediation found.") - if pages_scanned: - lines.append("") - lines.append("Note: Try scanning more pages or use FILTER_IMPACT=Critical,Important for severity filter.") - else: - lines.append(f"Found {len(cves)} CVE(s) with available remediation:\n") - for i, cve in enumerate(cves, 1): - cve_id = cve.get("id", "?") - cvss = get_attr(cve, "cvss3_score") or get_attr(cve, "cvss2_score") or "-" - impact = get_attr(cve, "impact", "-") or "-" - synopsis = get_attr(cve, "synopsis", "") or cve_id - url = cve.get("url", "") - lines.append(f"{i}. CVE ID: {cve_id}") - lines.append(f" CVSS v3 Score: {cvss}") - lines.append(f" Severity: {impact}") - lines.append(f" Synopsis: {synopsis}") - if url: - lines.append(f" View in Insights: {url}") - lines.append("") - return "\n".join(lines) - - -def load_and_merge_files(paths): - """Load multiple JSON files, extract CVEs, merge and dedupe by id.""" - all_cves = {} - max_total = 0 - for p in paths: - if not os.path.exists(p): - continue - with open(p, "r") as f: - try: - data = json.load(f) - except json.JSONDecodeError: - continue - cves, total = extract_cves(data) - max_total = max(max_total, total) - for cve in cves: - cid = cve.get("id") - if cid and cid not in all_cves: - all_cves[cid] = cve - return list(all_cves.values()), max_total - - -def main(): - # Parse args — multiple files = aggregated multi-page mode - paths = [a for a in sys.argv[1:] if not a.startswith("-")] - if not paths and not sys.stdin.isatty(): - paths = ["-"] - - remediatable_only = os.environ.get("FILTER_REMEDIATABLE", "").lower() in ("1", "true", "yes") - impact_str = os.environ.get("FILTER_IMPACT", "") - impact_filter = [s.strip() for s in impact_str.split(",") if s.strip()] if impact_str else None - sort_by = os.environ.get("SORT_BY", "cvss") - output = os.environ.get("OUTPUT", "report" if len(paths) > 1 else "summary") - system_name = os.environ.get("SYSTEM_NAME", "") - pages_scanned = os.environ.get("PAGES_SCANNED", str(len(paths)) if len(paths) > 1 else None) - - if len(paths) > 1 and "-" not in paths: - cves, total = load_and_merge_files(paths) - else: - src = paths[0] if paths else "-" - data = load_response(src) - cves, total = extract_cves(data) - - cves = filter_cves(cves, remediatable_only=remediatable_only, impact_filter=impact_filter) - cves = sort_cves(cves, sort_by=sort_by) - - if output == "json": - print(json.dumps({"data": cves, "total": total, "filtered_count": len(cves)}, indent=2)) - elif output == "table": - print(format_table(cves)) - elif output == "report": - print(format_report(cves, total, system_name=system_name or None, pages_scanned=pages_scanned)) - else: - print(format_summary(cves, total, remediatable_only, impact_filter)) - print("") - print("Top CVEs (by CVSS):") - print(format_table(cves, limit=15)) - - -if __name__ == "__main__": - main() diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/02-cve-parsing-guide.md b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/02-cve-parsing-guide.md deleted file mode 100644 index f4608f3b..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/02-cve-parsing-guide.md +++ /dev/null @@ -1,147 +0,0 @@ -# CVE Response Parsing Guide - -**Use this reference** when you need to parse/filter MCP vulnerability tool responses. - -**Do NOT use** jq, inline Python, or other ad-hoc JSON parsing. Use the skill's parser script only. - -**Do NOT generate inline Python** to aggregate multiple page files—the parser supports multiple file paths. - -## When to Use the Parser - -- After `vulnerability__get_system_cves` returns a large response -- After `vulnerability__get_cves` returns a response -- When filtering for `advisory_available === true` (remediatable CVEs) -- When filtering by impact (Critical, Important, Moderate, Low) -- When summarizing CVE counts by severity and remediation status -- **When aggregating multiple paginated pages**—pass all page files as arguments - -**Do NOT use jq or inline Python**—use the skill's parser script. - -## Parser Location - -``` -rh-sre/skills/cve-impact/references/01-cve-response-parser.py -``` - -From workspace root: `rh-sre/skills/cve-impact/references/01-cve-response-parser.py` - -## How to Invoke - -### Option 1: JSON file path - -Save the MCP tool result to a file, then run: - -```bash -python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py /path/to/response.json -``` - -### Option 2: stdin - -```bash -python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py < /path/to/response.json -``` - -### Option 3: From MCP tool result file - -When the MCP tool writes to a file (e.g. `tool-results/toolu_xxx.txt`): - -```bash -python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py /path/to/tool-results/toolu_xxx.txt -``` - -### Option 4: Multiple page files (aggregated report) - -When you have multiple paginated responses (e.g. pages 1–5 from `vulnerability__get_system_cves`), pass all files. The parser merges, dedupes, and produces an aggregated report: - -```bash -FILTER_REMEDIATABLE=1 FILTER_IMPACT=Critical,Important OUTPUT=report SYSTEM_NAME=ip-172-31-32-201.eu-west-3.compute.internal \ - python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py page1.json page2.json page3.json page4.json page5.json -``` - -**Do NOT generate inline Python** to loop over page files—use the parser with multiple paths. - -## Filter Options (Environment Variables) - -| Variable | Values | Effect | -|----------|--------|--------| -| `FILTER_REMEDIATABLE` | `1`, `true`, `yes` | Only CVEs with `advisory_available=true` | -| `FILTER_IMPACT` | `Critical,Important` | Only these severities (comma-separated) | -| `SORT_BY` | `cvss`, `impact`, `public_date` | Sort order (default: cvss) | -| `OUTPUT` | `summary`, `table`, `json`, `report` | Output format. `report` = aggregated format (default when multiple files) | -| `SYSTEM_NAME` | hostname string | For report header (e.g. `ip-172-31-32-201.eu-west-3.compute.internal`) | -| `PAGES_SCANNED` | number | For report header (e.g. `5`). Auto-set when multiple files. | - -### Examples - -**Remediatable CVEs only:** -```bash -FILTER_REMEDIATABLE=1 python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py response.json -``` - -**Critical/Important only:** -```bash -FILTER_IMPACT=Critical,Important python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py response.json -``` - -**Remediatable + Critical/Important:** -```bash -FILTER_REMEDIATABLE=1 FILTER_IMPACT=Critical,Important python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py response.json -``` - -**Aggregated report from multiple pages:** -```bash -FILTER_REMEDIATABLE=1 FILTER_IMPACT=Critical,Important OUTPUT=report SYSTEM_NAME=ip-172-31-32-201.eu-west-3.compute.internal \ - python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py page1.json page2.json page3.json page4.json page5.json -``` - -**JSON output (for further processing):** -```bash -OUTPUT=json python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py response.json -``` - -## Response Structure (Lightspeed MCP) - -The parser expects this structure (from `vulnerability__get_system_cves` or `vulnerability__get_cves`): - -```json -{ - "result": { - "data": [ - { - "id": "CVE-2024-1234", - "type": "cve", - "url": "https://console.redhat.com/insights/vulnerability/cves/CVE-2024-1234", - "attributes": { - "advisory_available": true, - "impact": "Important", - "cvss3_score": "7.5", - "cvss2_score": null, - "description": "...", - "synopsis": "CVE-2024-1234", - "public_date": "2024-01-15", - "business_risk": "Low" - } - } - ] - }, - "meta": { - "count": 1735 - } -} -``` - -Key fields: -- `result.data` — Array of CVE objects -- `meta.count` — Total CVEs (for pagination context) -- `attributes.advisory_available` — Boolean, remediatable -- `attributes.impact` — Critical, Important, Moderate, Low, None -- `attributes.cvss3_score` — CVSS 3.x score string - -## Workflow Integration - -1. Call MCP tool (`vulnerability__get_system_cves` or `vulnerability__get_cves`) — one or more times (paginated) -2. Save each response to file (MCP may write to `tool-results/` or you save from result) -3. **Run parser** (required—do not use jq or inline Python): - - Single page: `python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py ` - - Multiple pages: `python3 rh-sre/skills/cve-impact/references/01-cve-response-parser.py page1.json page2.json ...` -4. Use parser output for summary tables and user-facing reports diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/03-output-templates.md b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/03-output-templates.md deleted file mode 100644 index 4f41c54b..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/03-output-templates.md +++ /dev/null @@ -1,39 +0,0 @@ -# CVE Impact Output Templates - -Read when completing a CVE impact analysis to format the report. - -## Report Format - -```markdown -# CVE Impact Analysis Report - -## CVE Information -**CVE ID**: CVE-YYYY-NNNNN -**CVSS Score**: X.X -**Severity**: Critical/High/Medium/Low -**Published**: YYYY-MM-DD - -**Description**: [Brief description] -**Affected Packages**: package-name version-range - -## Affected Systems -**Total Systems**: N -| System | Hostname | Environment | Package | Status | - -## Risk Assessment -**Overall Risk**: Critical/High/Medium/Low -**Priority**: P0/P1/P2 -**Recommendation**: [Immediate remediation / Schedule maintenance / Monitor] - -## Business Impact -- Confidentiality, Integrity, Availability - -## Remediation Options -- Automated playbook / Manual steps / Testing required - -## Next Steps -1. Approve remediation plan -2. Schedule maintenance (if needed) -3. Create playbook (use `/remediation` skill) -4. Test in staging → Execute in production → Verify -``` diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/04-examples.md b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/04-examples.md deleted file mode 100644 index 325e9850..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/04-examples.md +++ /dev/null @@ -1,37 +0,0 @@ -# CVE Impact Examples - -Read when handling specific query types. - -## Example 0: Account-Level Critical CVEs - -**Request**: "What are the most critical vulnerabilities on my account?" -- Follow [01-account-cves.md](../flows/01-account-cves.md) -- Call `vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20)` -- Return summary table; offer remediation via `/remediation` - -## Example 1: CVEs on a System - -**Request**: "What CVEs affect ip-172-31-32-201?" -- Follow [02-system-all-cves.md](../flows/02-system-all-cves.md) -- HITL: pagination (first page / all pages / N pages) -- Resolve hostname: `inventory__find_host_by_name` -- Call `vulnerability__get_system_cves(system_uuid=..., limit=100, offset=0)` - -## Example 2: Single CVE Analysis - -**Request**: "Analyze CVE-2024-1234" -- `get_cve` → `get_cve_systems` → classify → risk assessment → suggest `/remediation` - -## Example 3: Compare CVEs - -**Request**: "Compare CVE-2024-1234 and CVE-2024-5678" -- Retrieve both; comparison table; prioritization; batch remediation if appropriate - -## Example 4: Production-Only - -**Request**: "Which production systems are affected by CVE-2024-1234?" -- Retrieve CVE; filter by environment tag; production-specific impact - -## Integration with Remediation - -After analysis, suggest: "Would you like me to create a remediation playbook?" (invoke `/remediation`). Provide CVE ID, system UUIDs, execution method, maintenance window. diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/05-error-handling.md b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/05-error-handling.md deleted file mode 100644 index 006aee55..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/cve-impact/references/05-error-handling.md +++ /dev/null @@ -1,24 +0,0 @@ -# CVE Impact Error Handling - -Read when errors occur during CVE analysis. - -## CVE Not Found - -``` -CVE-YYYY-NNNNN was not found in the Red Hat CVE database. - -Possible reasons: CVE ID incorrect, too recent, doesn't affect RHEL. -Suggestions: Verify format (CVE-YYYY-NNNNN), check NVD: https://nvd.nist.gov/vuln/search -``` - -## No Affected Systems - -``` -CVE-YYYY-NNNNN Analysis Complete — No systems affected. -Possible reasons: Already patched, packages not installed, different versions. -No action required. -``` - -## Lightspeed Tool Failures - -If explain_cves fails with `'dnf_modules'`: Do NOT show raw error. Use workaround from [lightspeed-mcp-tool-failures.md](../../../docs/references/lightspeed-mcp-tool-failures.md) (get_cve + get_host_details synthesis). diff --git a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/mcp-lightspeed-validator/SKILL.md b/evaluation/with_skills/rh-sre__cve-impact/environment/skills/mcp-lightspeed-validator/SKILL.md deleted file mode 100644 index e1f1528e..00000000 --- a/evaluation/with_skills/rh-sre__cve-impact/environment/skills/mcp-lightspeed-validator/SKILL.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -name: mcp-lightspeed-validator -description: | - Validate Red Hat Lightspeed MCP server connectivity. Use when the user asks to "validate Lightspeed MCP", "check Lightspeed connection", or when other skills need to verify lightspeed-mcp availability before CVE operations. -model: haiku -color: yellow ---- - -# MCP Lightspeed Validator - -Validates connectivity to the Red Hat Lightspeed MCP server by running a lightweight tool call. - -## When to Use This Skill - -Use when validating Lightspeed MCP before CVE operations, troubleshooting connection issues, or when other skills (e.g. remediation) need to verify availability. Do NOT use for actual CVE queries—use cve-impact or cve-validation. - -## Workflow - -1. **Test connectivity**: Call `vulnerability__get_cves` with **no parameters** (uses default limit=10). Do NOT pass `limit`—some MCP clients incorrectly serialize it as `limit_`, causing validation errors. -2. **If it fails**: Provide a comprehensive message with possible root causes (see below). -3. **Report**: Output a table with validated servers and outcome (emojis). - -## Failure Message (Root Causes) - -When the tool call fails, include: - -``` -❌ Lightspeed MCP connection failed - -**Possible root causes:** -- **Credentials**: LIGHTSPEED_CLIENT_ID or LIGHTSPEED_CLIENT_SECRET not set or invalid -- **Expired credentials**: Red Hat Console tokens may have expired -- **Server not running**: MCP server/container may be stopped -- **Network**: Firewall or proxy blocking console.redhat.com -- **Configuration**: .mcp.json misconfigured or server not registered - -**Troubleshooting:** -1. Verify env vars: LIGHTSPEED_CLIENT_ID, LIGHTSPEED_CLIENT_SECRET (never echo values) -2. Check credentials at: https://console.redhat.com/settings/integrations -3. Restart MCP server or host after config changes -4. Check container logs if using podman/docker -``` - -## Report Format - -Always end with a table: - -| Server | Outcome | -|--------|---------| -| lightspeed-mcp | ✅ PASSED | -| lightspeed-mcp | ❌ FAILED | - -Use ✅ for success, ❌ for failure, ⚠️ for partial (e.g. connected but error on tool). - -## Dependencies - -### Required MCP Tools -- `vulnerability__get_cves` or `get_cves` (from lightspeed-mcp) - Connectivity test - -### Required MCP Servers -- `lightspeed-mcp` - Red Hat Lightspeed vulnerability and inventory data diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__cve-validation/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 1092dd1d..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- pass rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/SKILL.md b/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/SKILL.md deleted file mode 100644 index f76c22be..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/SKILL.md +++ /dev/null @@ -1,340 +0,0 @@ ---- -name: cve-validation -description: | - **CRITICAL**: This skill must be used for CVE validation queries. DO NOT use raw MCP tools like get_cve directly. - - Validate CVE identifiers and check remediation availability in Red Hat Lightspeed. Use this skill when you need to verify a CVE exists, check its severity, and confirm automated remediation is available before proceeding with remediation planning. - - **DO NOT use this skill when** user requests full remediation - use `/remediation` skill instead: - - "Create a remediation playbook for CVE-X" → `/remediation` skill - - "Create playbook and execute it" → `/remediation` skill - - "Remediate CVE-X" / "Patch CVE-X" → `/remediation` skill - - This skill orchestrates MCP tools (get_cve) to provide comprehensive CVE validation. The `/remediation` skill invokes this skill as Step 2 of its workflow. ---- - -# CVE Validation Skill - -This skill validates CVE identifiers and checks remediation availability in Red Hat Lightspeed, ensuring CVEs are valid and remediable before investing effort in remediation planning. - -**Integration with Remediation Skill**: The `/remediation` skill orchestrates this skill as part of its Step 2 (Validate CVE) workflow. For standalone CVE validation, you can invoke this skill directly. - -## Invocation Note (Host-Specific) - -When invoked by another skill (e.g. remediation), use the Skill tool—do NOT use "Task Output" with the skill name as task ID. That causes "No task found with ID: cve-validation". See [skill-invocation.md](../../docs/references/skill-invocation.md). - -## Prerequisites - -**Required MCP Servers**: `lightspeed-mcp` ([setup guide](https://console.redhat.com/)) - -**Required MCP Tools**: -- `get_cve` (from lightspeed-mcp) - Get CVE metadata and validation - -**Required Environment Variables**: -- `LIGHTSPEED_CLIENT_ID` - Red Hat Lightspeed service account client ID -- `LIGHTSPEED_CLIENT_SECRET` - Red Hat Lightspeed service account secret - -### Prerequisite Validation - -**CRITICAL**: Before executing any operations, execute the `/mcp-lightspeed-validator` skill to verify MCP server availability. - -**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-lightspeed-validator/SKILL.md#validation-freshness-policy). - -**How to invoke**: Execute the `/mcp-lightspeed-validator` skill - -**Handle validation result**: -- **If validation PASSED**: Continue with CVE validation -- **If validation PARTIAL**: Warn user and ask to proceed -- **If validation FAILED**: Stop execution, provide setup instructions - -## When to Use This Skill - -**Use this skill directly when you need**: -- Quick validation of CVE identifier format and existence (standalone query) -- Check if automated remediation is available -- Verify CVE metadata before analysis -- Validate CVE lists for batch operations - -**DO NOT use this skill when** - use `/remediation` skill instead: -- User says "create a remediation playbook" or "remediate CVE-X" or "patch CVE-X" -- User says "create playbook and execute it" - agent orchestrates full workflow -- Any request that implies playbook generation or execution - -**Use the `/remediation` skill when you need**: -- Full remediation workflow (validation + analysis + playbook + execution) -- Integrated CVE validation as part of remediation planning - -**How they work together**: The `/remediation` skill invokes this skill early in the workflow to fail fast if a CVE is invalid or has no automated remediation, saving time and effort. - -**When invoked by remediation**: Return remediatable status prominently so the orchestrator can gate. Include `remediation_status.automated_remediation_available` (boolean) and `validation_status` ("valid" | "not_remediable" | "invalid" | "not_found") in the output. - -## Workflow - -### Step 0: Validate Lightspeed MCP Prerequisites - -**Action**: Execute the `/mcp-lightspeed-validator` skill - -**Note**: Can skip if validation was performed earlier in this session and succeeded. See [Validation Freshness Policy](../mcp-lightspeed-validator/SKILL.md#validation-freshness-policy). - -**How to invoke**: Execute the `/mcp-lightspeed-validator` skill - -**Handle validation result**: -- **If validation PASSED**: Continue to Step 1 -- **If validation PARTIAL**: Warn user and ask to proceed -- **If validation FAILED**: Stop execution, user must set up MCP server - -### Step 1: CVE Format Validation - -Validate CVE identifier format before calling MCP tools. **Format only**—do NOT reject based on year or sequence magnitude. - -```python -CVE Format: CVE-YYYY-NNNNN -Where: -- YYYY = 4-digit year (1999-2030; current and recent years are valid) -- NNNNN = 4-7 digit sequence number (e.g. 1234, 24882, 1234567) - -Valid Examples: -- CVE-2024-1234 -- CVE-2026-24882 # 2026 CVEs exist; 24882 is 5 digits (valid) -- CVE-2023-12345 -- CVE-2021-1234567 - -Invalid Examples (format only): -- CVE-24-1234 (year must be 4 digits) -- CVE-2024-ABC (sequence must be numeric) -- 2024-1234 (missing CVE- prefix) -- CVE-2024-123 (sequence must be 4-7 digits) -``` - -**Quick Regex Check**: -``` -Pattern: ^CVE-\d{4}-\d{4,7}$ - -If invalid format: -→ Return error immediately -→ Suggest format correction -→ Do not proceed to MCP tool calls -``` - -**CRITICAL - Do NOT add extra checks**: If the format matches the regex, you MUST call `get_cve`. Do NOT reject based on: -- "Future" or "current year" assumptions (e.g. "2026 CVE might not exist yet") -- Sequence number magnitude (e.g. "24882 seems high")—5 digits is valid -- Your training data about typical CVE ranges - -Let the API determine existence. A 404 from get_cve means "not found"; format validation only catches malformed IDs. - -### Step 2: CVE Metadata Retrieval - -**CRITICAL**: Document consultation MUST happen BEFORE tool invocation. - -**Document Consultation** (REQUIRED - Execute FIRST): -1. **Action**: Read [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) using the Read tool to understand CVE validation criteria -2. **Action**: Read [references/01-remediation-indicators.md](references/01-remediation-indicators.md) to interpret get_cve response—**CRITICAL** to avoid misinterpreting remediation availability -3. **Output to user**: "I consulted [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) and [01-remediation-indicators.md](references/01-remediation-indicators.md) for CVE validation and remediation indicator interpretation." - -**MCP Tool**: `get_cve` or `vulnerability__get_cve` (from lightspeed-mcp) - -**Do NOT use** `vulnerability__explain_cves` for validation. That tool requires `system_uuid` from inventory; at validation you may not have it. `get_cve` provides remediation availability. Never pass `system_uuid: "undefined"` or placeholders. - -**Parameters**: -- `cve`: Exact CVE identifier from user query (format: `"CVE-YYYY-NNNNN"`) - - Example: `"CVE-2024-1234"` -- `advisory_available`: `"true"` (retrieve CVE with advisory/remediation info) - -**Expected Output**: CVE metadata including CVSS score, severity, affected packages, remediation availability - -Retrieve CVE metadata from Red Hat Lightspeed: - -```json -{ - "cve_id": "CVE-2024-1234", - "cvss_score": 7.5, - "cvss_vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - "severity": "Important", # Red Hat severity rating - "description": "A vulnerability in Apache HTTPD...", - "published_date": "2024-01-15", - "modified_date": "2024-01-20", - - "affected_packages": [ - { - "name": "httpd", - "version": "2.4.37-1.el8", - "fixed_version": "2.4.37-2.el8" - } - ], - - "references": [ - "https://access.redhat.com/errata/RHSA-2024:1234", - "https://nvd.nist.gov/vuln/detail/CVE-2024-1234" - ], - - "cwe": "CWE-400: Uncontrolled Resource Consumption", - - "exploitability": "Proof of concept available", - "remediation_available": true, # KEY FIELD - "reboot_required": false -} -``` - -### Step 3: Validation Checks - -**CRITICAL**: Document consultation MUST happen BEFORE validation logic. - -**Document Consultation** (REQUIRED - Execute FIRST): -1. **Action**: Read [cvss-scoring.md](../../docs/references/cvss-scoring.md) using the Read tool to understand Red Hat severity classification and CVSS score ranges -2. **Output to user**: "I consulted [cvss-scoring.md](../../docs/references/cvss-scoring.md) to understand Red Hat severity classification and CVSS score ranges." - -Perform comprehensive validation: - -**A. Existence Check**: -``` -✓ CVE exists in Red Hat CVE database -✗ CVE not found → Return error with suggestions -``` - -**B. Red Hat Relevance Check**: -``` -✓ CVE affects RHEL systems -✗ CVE is Windows/macOS specific → Not applicable to RHEL -``` - -**C. Severity Classification**: -``` -Red Hat Severity Levels: -- Critical (CVSS 9.0-10.0): Immediate action required -- Important (CVSS 7.0-8.9): Urgent remediation needed -- Moderate (CVSS 4.0-6.9): Plan remediation -- Low (CVSS 0.1-3.9): Low priority -``` - -**D. Remediation Availability Check** (READ [references/01-remediation-indicators.md](references/01-remediation-indicators.md)): -``` -Key Question: Can Red Hat Lightspeed generate an automated playbook? - -✅ USE these fields: - - advisory_available === true → Remediation available - - remediation === 2 → Automated remediation available - - advisories_list non-empty → RHSA exists, remediation available - -❌ DO NOT use rules[] for remediation decision: - - rules: [] (empty) does NOT mean "no remediation" - - Advisor rules are separate from vulnerability remediation - - Remediation comes from Security Advisories (RHSA), not Advisor rules - -✓ If advisory_available=true OR remediation=2 OR advisories_list has entries - → Proceed with automated remediation - → Use create_vulnerability_playbook tool - -✗ If advisory_available=false AND (remediation=0 or advisories_list empty) - → Manual remediation required - → Provide manual steps based on affected packages -``` - -**E. Package Information Validation**: -``` -Check affected_packages array: -✓ Packages identified: httpd-2.4.37-1.el8 -✓ Fixed version available: httpd-2.4.37-2.el8 -✓ Package exists in RHEL repositories - -This information will be used by playbook-generator skill. -``` - -### Step 4: Return Validation Result - -Return structured validation result. **When invoked by remediation skill**: Ensure `validation_status` and `remediation_status.automated_remediation_available` are explicit—the orchestrator gates on these. - -```json -{ - "validation_status": "valid", # or "invalid", "not_found", "not_remediable" - - "cve_metadata": { - "cve_id": "CVE-2024-1234", - "cvss_score": 7.5, - "severity": "Important", - "description": "Brief description...", - "published_date": "2024-01-15" - }, - - "remediation_status": { - "automated_remediation_available": true, - "reboot_required": false, - "affected_packages": [ - { - "name": "httpd", - "current_version": "2.4.37-1.el8", - "fixed_version": "2.4.37-2.el8" - } - ] - }, - - "recommendations": [ - "Automated remediation available via Red Hat Lightspeed", - "No reboot required for this CVE", - "Severity: Important - Urgent remediation recommended", - "Test in staging environment before production deployment" - ], - - "next_steps": [ - "Analyze CVE impact (use cve-impact skill)", - "Gather system context (use system-context skill)", - "Generate remediation playbook (use playbook-generator skill)" - ] -} -``` - -## Output, Examples, Error Handling - -**Read [references/03-output-template.md](references/03-output-template.md)** for report format. -**Read [references/04-examples.md](references/04-examples.md)** for validation examples. -**Read [references/05-error-handling.md](references/05-error-handling.md)** for format, not-found, no-remediation, and API errors. - -## Best Practices - -Validate format first; if regex matches, ALWAYS call get_cve (do not reject on year/sequence). Check remediation availability; fail fast if none. Provide clear next steps and manual guidance when automated unavailable. Link to NVD and Red Hat Security. Cache results to avoid redundant calls. - -## Dependencies - -### Required MCP Servers -- `lightspeed-mcp` - Red Hat Lightspeed platform access - -### Required MCP Tools -- `get_cve` (from lightspeed-mcp) - Get CVE metadata and validation - - Parameters: cve_id (string, format CVE-YYYY-NNNNN), include_details (boolean), validate_format (boolean) - - Returns: CVE metadata with CVSS score, severity, affected packages, remediation availability - -### Related Skills -- `mcp-lightspeed-validator` - **PREREQUISITE** - Validates Lightspeed MCP server before operations - - Use before: ALL cve-validation operations (Step 0 in workflow) - - Purpose: Ensures MCP server is available before attempting tool calls - -- `cve-impact` - Analyze CVE impact after validation - - Use after: Step 4 when CVE is validated and user wants impact analysis - - Purpose: Assess risk and affected systems for validated CVE - -- `system-context` - Get system details after validating CVE affects systems - - Use after: Validation confirms CVE has affected systems - - Purpose: Understand deployment context before remediation - -- `playbook-generator` - Generate remediation playbooks for validated CVEs - - Use after: Validation confirms remediation_available = true - - Purpose: Create automated remediation for valid, remediable CVEs - -### Reference Documentation -- [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) - CVE validation criteria -- [references/01-remediation-indicators.md](references/01-remediation-indicators.md) - **REQUIRED** - Correct interpretation of get_cve response (advisory_available, remediation, advisories_list). Do NOT use rules[] for remediation decision. -- [cvss-scoring.md](../../docs/references/cvss-scoring.md) - Red Hat severity classification and CVSS score ranges -- [cve-remediation-templates.md](../../docs/ansible/cve-remediation-templates.md) - Manual remediation templates for CVEs without automated playbooks - -## Tools Reference - -This skill uses: -- `get_cve` (vulnerability toolset) - Get CVE metadata and remediation availability from Red Hat Lightspeed - -**Do NOT use** `vulnerability__explain_cves` in this skill—it requires `system_uuid` which may not be available at validation time. Use `get_cve` only. - -All tools are provided by the lightspeed-mcp MCP server configured in `.mcp.json`. - -## Integration with Other Skills - -cve-impact, playbook-generator, system-context, remediation-verifier all depend on validation first. The `/remediation` skill invokes cve-validation as Step 2. Validate → proceed if valid; stop and return error if invalid. diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/01-remediation-indicators.md b/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/01-remediation-indicators.md deleted file mode 100644 index 17f9afe8..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/01-remediation-indicators.md +++ /dev/null @@ -1,66 +0,0 @@ -# Remediation Availability Indicators (get_cve Response) - -Read this reference when interpreting `vulnerability__get_cve` or `get_cve` response to determine if automated remediation is available. - -## CRITICAL: Correct vs Incorrect Indicators - -### ✅ USE These Fields for Remediation Availability - -| Field | Meaning | Remediation Available When | -|-------|---------|-----------------------------| -| `advisory_available` | Red Hat Security Advisory exists | `true` | -| `remediation` | Remediation status code | `2` = automated remediation available | -| `advisories_list` | List of RHSA/errata IDs | Non-empty (e.g. `["RHSA-2026:2719"]`) | - -**Decision rule**: Remediation IS available when `advisory_available === true` OR `remediation === 2` OR `advisories_list` is non-empty. - -### ❌ DO NOT Use These Fields for Remediation - -| Field | Meaning | Why NOT to Use | -|-------|---------|----------------| -| `rules` | Red Hat Insights **Advisor** rules | Advisor rules are separate from vulnerability remediation. Empty `rules: []` does NOT mean no remediation. Remediation comes from Security Advisories (RHSA), not Advisor rules. | - -**Common mistake**: Agent sees `rules: []` (empty) and incorrectly concludes "no remediation available." This is WRONG. Always check `advisory_available` and `remediation` first. - -## Interpretation Checklist - -When evaluating `get_cve` response: - -1. **Check `advisory_available`**: If `true` → remediation available ✓ -2. **Check `remediation`**: If `2` → automated remediation available ✓ -3. **Check `advisories_list`**: If non-empty (e.g. RHSA-*) → remediation available ✓ -4. **Ignore `rules`**: Do NOT use for remediation decision. Empty rules ≠ no remediation. - -## Example: Remediation Available (rules empty) - -```json -{ - "advisory_available": true, - "advisories_list": ["RHSA-2026:2719"], - "remediation": 2, - "rules": [] -} -``` - -**Correct interpretation**: Remediation IS available. `rules: []` only means no Advisor rule—remediation comes from RHSA-2026:2719. - -## Example: No Remediation - -```json -{ - "advisory_available": false, - "advisories_list": [], - "remediation": 0, - "rules": [] -} -``` - -**Correct interpretation**: No automated remediation. Manual steps required. - -## get_cve_systems Response (per-system) - -When using `get_cve_systems` for system-level check, each system entry may include: -- `attributes.advisory_available` — same meaning as get_cve -- `attributes.remediation` — same meaning as get_cve - -Use the same interpretation rules. Do NOT use `rules` for remediation decision. diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/03-output-template.md b/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/03-output-template.md deleted file mode 100644 index 51bb3992..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/03-output-template.md +++ /dev/null @@ -1,36 +0,0 @@ -# CVE Validation Output Template - -Read when completing CVE validation to format the report. - -```markdown -# CVE Validation Result - -## CVE: CVE-YYYY-NNNNN -**Status**: ✓ Valid - -## CVE Information -**CVSS Score**: X.X (Severity) -**Published**: YYYY-MM-DD -**Description**: [Brief description] - -## Affected Packages -- package-current → package-fixed (fixed) - -## Remediation Status -✓ **Automated Remediation Available** (or ✗ Manual required) -✓ Package updates available -✗ Reboot NOT required - -## Severity Assessment -**Red Hat Severity**: Critical/Important/Moderate/Low -**Priority**: P0/P1/P2 -**Response Time**: [guidance] - -## Recommendations -1. [Automated/manual remediation guidance] -2. Test in staging first -3. Schedule deployment during change window - -## Next Steps -1. cve-impact → system-context → playbook-generator → remediation-verifier -``` diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/04-examples.md b/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/04-examples.md deleted file mode 100644 index 2a16ce85..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/04-examples.md +++ /dev/null @@ -1,35 +0,0 @@ -# CVE Validation Examples - -## Example 1: Valid CVE with Automated Remediation - -**Request**: "Validate CVE-2024-1234" -1. Format check → Valid -2. get_cve → found -3. advisory_available/remediation/advisories_list → remediation available (ignore rules[]) -4. Return: "Valid, automated remediation available" - -## Example 2: Valid CVE, No Automated Remediation - -**Request**: "Validate CVE-2024-5678" -1. Format → Valid, get_cve → found -2. advisory_available/remediation/advisories_list → no remediation -3. Return: "Valid but manual remediation: yum update custom-app" - -## Example 3: Invalid Format - -**Request**: "Validate CVE-24-1234" -1. Format → Invalid (year must be 4 digits) -2. Return error without MCP call; suggest CVE-2024-1234 - -## Example 4: CVE Not Found - -**Request**: "Validate CVE-2024-999999" -1. Format → Valid, get_cve → 404 -2. Return: "Not found. Check NVD, access.redhat.com, or wait 24-48h if recent" - -## Example 5: Batch Validation - -**Request**: "Validate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -1. Validate each sequentially -2. Return summary per CVE (valid/remediable, valid/manual, invalid format) -3. Suggest next steps per CVE diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/05-error-handling.md b/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/05-error-handling.md deleted file mode 100644 index 201c193a..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/skills/cve-validation/references/05-error-handling.md +++ /dev/null @@ -1,37 +0,0 @@ -# CVE Validation Error Handling - -Read when errors occur during CVE validation. - -## CVE Format Invalid - -``` -CVE Validation Failed: Invalid Format -Provided: CVE-24-1234 -Expected: CVE-YYYY-NNNNN (YYYY=4-digit year, NNNNN=4-7 digit sequence) -Suggestion: Did you mean CVE-2024-1234? -``` - -## CVE Not Found in Database - -``` -CVE-YYYY-NNNNN was not found in Red Hat CVE database. -Possible reasons: Too recent, doesn't affect RHEL, incorrect ID. -Next steps: Verify at NVD, access.redhat.com/security/cve/CVE-YYYY-NNNNN, wait 24-48h if recent. -``` - -## CVE Exists But No Automated Remediation - -``` -CVE Validation: Valid (No Automated Remediation) -CVE-YYYY-NNNNN is valid but has no automated playbook. -Manual steps: dnf/yum update package-name, restart service if needed, verify fix. -Offer: "Would you like a manual playbook template?" -``` - -## API Access Error - -``` -CVE Validation Failed: API Access Error -Possible causes: Network, auth failure, service unavailable. -Troubleshooting: ping console.redhat.com, verify credentials, status.redhat.com, retry. -``` diff --git a/evaluation/with_skills/rh-sre__cve-validation/environment/skills/mcp-lightspeed-validator/SKILL.md b/evaluation/with_skills/rh-sre__cve-validation/environment/skills/mcp-lightspeed-validator/SKILL.md deleted file mode 100644 index e1f1528e..00000000 --- a/evaluation/with_skills/rh-sre__cve-validation/environment/skills/mcp-lightspeed-validator/SKILL.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -name: mcp-lightspeed-validator -description: | - Validate Red Hat Lightspeed MCP server connectivity. Use when the user asks to "validate Lightspeed MCP", "check Lightspeed connection", or when other skills need to verify lightspeed-mcp availability before CVE operations. -model: haiku -color: yellow ---- - -# MCP Lightspeed Validator - -Validates connectivity to the Red Hat Lightspeed MCP server by running a lightweight tool call. - -## When to Use This Skill - -Use when validating Lightspeed MCP before CVE operations, troubleshooting connection issues, or when other skills (e.g. remediation) need to verify availability. Do NOT use for actual CVE queries—use cve-impact or cve-validation. - -## Workflow - -1. **Test connectivity**: Call `vulnerability__get_cves` with **no parameters** (uses default limit=10). Do NOT pass `limit`—some MCP clients incorrectly serialize it as `limit_`, causing validation errors. -2. **If it fails**: Provide a comprehensive message with possible root causes (see below). -3. **Report**: Output a table with validated servers and outcome (emojis). - -## Failure Message (Root Causes) - -When the tool call fails, include: - -``` -❌ Lightspeed MCP connection failed - -**Possible root causes:** -- **Credentials**: LIGHTSPEED_CLIENT_ID or LIGHTSPEED_CLIENT_SECRET not set or invalid -- **Expired credentials**: Red Hat Console tokens may have expired -- **Server not running**: MCP server/container may be stopped -- **Network**: Firewall or proxy blocking console.redhat.com -- **Configuration**: .mcp.json misconfigured or server not registered - -**Troubleshooting:** -1. Verify env vars: LIGHTSPEED_CLIENT_ID, LIGHTSPEED_CLIENT_SECRET (never echo values) -2. Check credentials at: https://console.redhat.com/settings/integrations -3. Restart MCP server or host after config changes -4. Check container logs if using podman/docker -``` - -## Report Format - -Always end with a table: - -| Server | Outcome | -|--------|---------| -| lightspeed-mcp | ✅ PASSED | -| lightspeed-mcp | ❌ FAILED | - -Use ✅ for success, ❌ for failure, ⚠️ for partial (e.g. connected but error on tool). - -## Dependencies - -### Required MCP Tools -- `vulnerability__get_cves` or `get_cves` (from lightspeed-mcp) - Connectivity test - -### Required MCP Servers -- `lightspeed-mcp` - Red Hat Lightspeed vulnerability and inventory data diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__execution-summary/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 1092dd1d..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- pass rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__execution-summary/environment/skills/execution-summary/SKILL.md b/evaluation/with_skills/rh-sre__execution-summary/environment/skills/execution-summary/SKILL.md deleted file mode 100644 index 6474c392..00000000 --- a/evaluation/with_skills/rh-sre__execution-summary/environment/skills/execution-summary/SKILL.md +++ /dev/null @@ -1,357 +0,0 @@ ---- -name: execution-summary -description: | - This skill should be used when the user asks to "generate execution summary", "create execution report", "summarize what was used", "show execution summary", or "what agents/skills/tools were used". Generates a concise report of agents, skills, tools, and documentation accessed during a workflow for audit and learning purposes. -model: haiku -color: blue ---- - -# Execution Summary Skill - -Generate a concise execution report summarizing all agents, skills, tools, and documentation accessed during a workflow. Useful for audit trails, learning reviews, and workflow documentation. - -## When to Use This Skill - -Use this skill when: -- User requests an execution summary or report -- At the end of a remediation workflow to document what was executed -- Tracking resource usage for audit or compliance purposes -- Creating a learning record of a complex workflow -- Documenting which components contributed to a result - -Do NOT use when: -- User wants detailed logs (use native logging instead) -- User wants performance metrics (use monitoring tools) -- Just listing available skills/agents (use documentation instead) - -## Workflow - -### Step 1: Analyze Conversation History - -**Action**: Review the current conversation to identify all agents, skills, tools, and documentation used - -**What to extract**: - -1. **Agents invoked** - Look for agent invocations in the conversation - - Example: Skill `remediation` (orchestration) → `rh-sre:remediation` - - Include plugin prefix: `rh-sre:` - -2. **Skills invoked** - Look for Skill tool calls - - Example: `Skill(skill="fleet-inventory")` → `rh-sre:fleet-inventory` - - Example: `Skill(skill="mcp-lightspeed-validator")` → `rh-sre:mcp-lightspeed-validator` - - Include plugin prefix: `rh-sre:` - -3. **MCP Tools called** - Look for MCP tool invocations - - Example: `get_host_details` → `lightspeed-mcp:get_host_details` - - Example: `vulnerability__get_cve` → `lightspeed-mcp:vulnerability__get_cve` - - Example: `job_templates_launch_retrieve` → `aap-mcp-job-management:job_templates_launch_retrieve` - - Include server prefix - -4. **Documentation consulted** - Look for Read tool calls on documentation files - - Pattern: Files under `rh-sre/docs/` or `rh-sre/skills/*/SKILL.md` - - Extract only from `docs/` onwards - - Example: `/path/to/rh-sre/docs/ansible/playbook-templates.md` → `docs/ansible/playbook-templates.md` - - Example: `/path/to/rh-sre/skills/fleet-inventory/SKILL.md` → `skills/fleet-inventory/SKILL.md` - - Include "I consulted [filename]" statements in conversation - -**How to analyze**: -- Review the conversation from start to current message -- Track tool invocations in chronological order -- Deduplicate: each resource should appear only once -- Maintain original order of first appearance - -### Step 2: Categorize and Deduplicate - -**Action**: Organize extracted resources into categories and remove duplicates - -**Categories**: -- **Agents**: Agent invocations -- **Skills**: Skill invocations -- **Tools**: MCP tool calls (group by server) -- **Docs**: Documentation files read - -**Deduplication rules**: -- If an agent was invoked multiple times, list it once -- If a skill was invoked multiple times, list it once -- If a tool was called multiple times, list it once -- If a doc was read multiple times, list it once - -**Sorting**: -- Within each category, maintain chronological order (order of first use) -- Do not alphabetize (preserve workflow sequence) - -### Step 3: Format Output - -**Action**: Generate the execution summary using the standard template - -**Output template**: -``` -**** EXECUTION SUMMARY START **** -Agents: ,,... -Skills: ,,... -Tools: ,,... -Docs: ,,... -**** EXECUTION SUMMARY END **** -``` - -**Formatting rules**: - -1. **Agent names**: Include plugin prefix - - Format: `rh-sre:agent-name` - - Example: `rh-sre:remediation` - - Separate with commas (no spaces): `rh-sre:remediation,rh-sre:validator` - -2. **Skill names**: Include plugin prefix - - Format: `rh-sre:skill-name` - - Example: `rh-sre:fleet-inventory` - - Separate with commas: `rh-sre:fleet-inventory,rh-sre:cve-impact` - -3. **Tool names**: Include MCP server prefix - - Format: `server-name:tool-name` - - Example: `lightspeed-mcp:get_host_details` - - Example: `aap-mcp-job-management:job_templates_list` - - Separate with commas: `lightspeed-mcp:get_cve,lightspeed-mcp:get_host_details` - -4. **Doc names**: Path from `docs/` onwards - - Format: `docs/category/filename.md` or `skills/skill-name/SKILL.md` - - Example: `docs/ansible/cve-remediation-templates.md` - - Example: `skills/fleet-inventory/SKILL.md` - - Separate with commas: `docs/ansible/playbook-templates.md,docs/insights/vulnerability-logic.md` - -5. **Empty categories**: If no resources used, show "None" - - Example: `Agents: None` - -6. **Spacing**: No spaces after commas (compact format) - -### Step 4: Present Summary - -**Action**: Output the formatted summary to the user - -**Presentation**: -``` -I've generated the execution summary for this workflow: - -**** EXECUTION SUMMARY START **** -Agents: None -Skills: rh-sre:remediation,rh-sre:fleet-inventory,rh-sre:cve-impact,rh-sre:playbook-generator,rh-sre:job-template-creator -Tools: lightspeed-mcp:get_host_details,lightspeed-mcp:get_cve,aap-mcp-job-management:job_templates_list -Docs: docs/ansible/cve-remediation-templates.md,docs/insights/vulnerability-logic.md,skills/fleet-inventory/SKILL.md -**** EXECUTION SUMMARY END **** - -This summary shows all agents, skills, tools, and documentation used during the workflow. -``` - -**Additional context** (optional): -- Brief explanation of what each category represents -- Note any notable patterns (e.g., "Heavy use of Lightspeed MCP tools") -- Mention if summary is for audit/learning purposes - -## Dependencies - -### Required Files -- None (analyzes conversation history only) - -### Related Skills -- None (standalone reporting skill) - -### Reference Documentation -- None required (execution tracking skill) - -## Example Usage - -### Example 1: Simple Fleet Query Workflow - -**User Request**: "Show the managed fleet, then generate execution summary" - -**Workflow executed**: -1. Invoked `mcp-lightspeed-validator` skill -2. Invoked `fleet-inventory` skill -3. Called `get_host_details` MCP tool -4. Read `docs/insights/insights-api.md` - -**Skill Response**: -``` -I've generated the execution summary for this workflow: - -**** EXECUTION SUMMARY START **** -Agents: None -Skills: rh-sre:mcp-lightspeed-validator,rh-sre:fleet-inventory -Tools: lightspeed-mcp:get_host_details -Docs: docs/insights/insights-api.md,skills/fleet-inventory/SKILL.md -**** EXECUTION SUMMARY END **** - -This workflow used 2 skills, 1 MCP tool, and consulted 2 documentation files. -``` - -### Example 2: CVE Remediation Workflow - -**User Request**: "Remediate CVE-2024-1234 on production systems, then show execution summary" - -**Workflow executed**: -1. Invoked `remediation` skill -2. Remediation skill delegated to: `cve-validation`, `cve-impact`, `system-context`, `playbook-generator`, `playbook-executor` skills -3. Called multiple MCP tools: `get_cve`, `get_cve_systems`, `get_host_details`, `create_vulnerability_playbook`, `execute_playbook` -4. Read multiple docs: CVE scoring, Ansible templates, remediation verification - -**Skill Response**: -``` -I've generated the execution summary for this workflow: - -**** EXECUTION SUMMARY START **** -Agents: None -Skills: rh-sre:remediation,rh-sre:cve-validation,rh-sre:cve-impact,rh-sre:system-context,rh-sre:playbook-generator,rh-sre:job-template-creator -Tools: lightspeed-mcp:get_cve,lightspeed-mcp:get_cve_systems,lightspeed-mcp:get_host_details,lightspeed-mcp:create_vulnerability_playbook,aap-mcp-job-management:job_templates_launch_retrieve -Docs: docs/references/cvss-scoring.md,docs/ansible/cve-remediation-templates.md,docs/insights/vulnerability-logic.md,skills/playbook-generator/SKILL.md -**** EXECUTION SUMMARY END **** - -This comprehensive remediation workflow used the remediation skill that orchestrated 5 skills, invoked 5 MCP tools, and consulted 4 documentation files. -``` - -### Example 3: Validation Only Workflow - -**User Request**: "Validate Lightspeed MCP and generate summary" - -**Workflow executed**: -1. Invoked `mcp-lightspeed-validator` skill -2. No MCP tools called (connectivity test unavailable) -3. No docs consulted - -**Skill Response**: -``` -I've generated the execution summary for this workflow: - -**** EXECUTION SUMMARY START **** -Agents: None -Skills: rh-sre:mcp-lightspeed-validator -Tools: None -Docs: None -**** EXECUTION SUMMARY END **** - -This lightweight validation workflow used 1 skill with no MCP tool invocations or documentation consultation. -``` - -## Error Handling - -### Incomplete Conversation History - -**Scenario**: Conversation history is not available or truncated - -**Response**: -``` -⚠ Execution Summary: INCOMPLETE - -Unable to generate complete summary due to limited conversation history. - -Partial summary based on available context: -**** EXECUTION SUMMARY START **** -Agents: [partial list] -Skills: [partial list] -Tools: [partial list] -Docs: [partial list] -**** EXECUTION SUMMARY END **** - -Note: This summary may not include all resources used earlier in the conversation. -``` - -### No Workflow Executed - -**Scenario**: User requests summary but no workflow has been executed - -**Response**: -``` -ℹ️ No execution to summarize - -No agents, skills, or tools have been invoked in this conversation yet. - -To generate a meaningful execution summary: -1. Execute a workflow (e.g., "Show the fleet", "Remediate CVE-X") -2. Then request the execution summary - -Would you like to start a workflow now? -``` - -### Ambiguous Resource Names - -**Scenario**: Uncertain about which plugin/server a resource belongs to - -**Response**: -Include the resource with a note: -``` -**** EXECUTION SUMMARY START **** -Agents: None -Skills: rh-sre:remediation,rh-sre:fleet-inventory,unknown-plugin:custom-skill -Tools: lightspeed-mcp:get_cve -Docs: docs/ansible/playbook-templates.md -**** EXECUTION SUMMARY END **** - -Note: "unknown-plugin:custom-skill" origin unclear - verify plugin source. -``` - -## Best Practices - -1. **Analyze entire conversation** - Don't miss early invocations -2. **Deduplicate resources** - Each resource appears once -3. **Maintain chronological order** - Preserve workflow sequence -4. **Use exact prefixes** - `rh-sre:`, `lightspeed-mcp:`, etc. -5. **Compact format** - No spaces after commas -6. **Include all categories** - Even if "None" -7. **Extract docs from "I consulted" statements** - These indicate documentation usage -8. **Path from docs/ onwards** - Not full filesystem paths -9. **Brief explanation** - Help user understand the summary -10. **Handle edge cases gracefully** - Empty workflows, incomplete history - -## Use Cases - -**Audit Trail**: -- Document which components were used for compliance -- Track MCP tool access patterns -- Record skill usage for billing/metrics - -**Learning & Training**: -- Show new users which resources solve specific problems -- Demonstrate skill orchestration patterns -- Illustrate skill orchestration workflows - -**Troubleshooting**: -- Identify which tools were called before an error -- Trace skill invocation sequence -- Document successful workflows for reproduction - -**Workflow Documentation**: -- Create records of complex remediation processes -- Document resource usage for similar future tasks -- Build a library of workflow patterns - -## Integration with Other Skills - -This skill complements other rh-sre skills: - -**After `/remediation` skill**: -``` -User: "Remediate CVE-X" -→ `/remediation` skill executes full workflow (invoked) -User: "Generate execution summary" -→ execution-summary shows complete resource usage -``` - -**For learning workflows**: -``` -User: "Show the fleet" -→ fleet-inventory skill executes -User: "What did you use to do that?" -→ execution-summary shows skills and tools invoked -``` - -**For audit purposes**: -``` -User: "Create playbook for CVE-X and generate audit trail" -→ Workflow executes -→ execution-summary provides compliance record -``` - -The summary output format is designed to be: -- **Machine-readable**: Parseable by scripts/tools -- **Human-readable**: Clear and concise for users -- **Compact**: Minimal token usage -- **Complete**: All resource categories represented -- **Auditable**: Chronological order preserved diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 1092dd1d..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- pass rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/SKILL.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/SKILL.md deleted file mode 100644 index 530bf58b..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/SKILL.md +++ /dev/null @@ -1,254 +0,0 @@ ---- -name: fleet-inventory -description: | - Query and display Red Hat Lightspeed managed system inventory. Use this skill for information-gathering requests about the fleet, registered systems, or inventory queries. This skill focuses on discovery and listing only - for remediation actions, transition to the `/remediation` skill. - - **When to use this skill**: - - "Show the managed fleet" - - "List all systems registered in Lightspeed" - - "What systems are affected by CVE-X?" - - "How many RHEL 8 systems do we have?" - - "Show me production systems" - - **When NOT to use this skill** (use `/remediation` skill instead): - - "Remediate CVE-X on these systems" - - "Create a playbook for..." - - "Patch system Y" - - This skill orchestrates MCP tools from lightspeed-mcp to provide comprehensive fleet visibility and system inventory management. -model: inherit -color: blue ---- - -# Fleet Inventory Skill - -This skill queries Red Hat Lightspeed to retrieve and display information about managed systems, registered hosts, and fleet inventory. - -## Prerequisites - -**Required MCP Servers**: `lightspeed-mcp` ([setup guide](https://console.redhat.com/)) - -**Required MCP Tools**: -- `get_host_details` (from lightspeed-mcp) - Retrieve system inventory -- `get_cve_systems` (from lightspeed-mcp) - Find CVE-affected systems - -**Required Environment Variables**: -- `LIGHTSPEED_CLIENT_ID` - Red Hat Lightspeed service account client ID -- `LIGHTSPEED_CLIENT_SECRET` - Red Hat Lightspeed service account secret - -### Prerequisite Validation - -**CRITICAL**: Before executing any operations, execute the `/mcp-lightspeed-validator` skill to verify MCP server availability. - -See **Step 0** in the Workflow section below for implementation details. - -**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-lightspeed-validator/SKILL.md#validation-freshness-policy). - -## When to Use This Skill - -**Use this skill directly when you need**: -- List all systems registered in Red Hat Lightspeed -- Show systems affected by specific CVEs -- Display system details (OS version, tags, last check-in) -- Filter systems by environment, RHEL version, or tags -- Count systems matching criteria -- Verify system registration status - -**Use the `/remediation` skill when you need**: -- Remediate vulnerabilities on systems -- Generate or execute playbooks -- Perform infrastructure changes -- End-to-end CVE remediation workflows - -**How they work together**: Use this skill for discovery ("What systems are affected?"), then transition to the `/remediation` skill for action ("Remediate those systems"). - -## Workflow - -### Step 0: Validate Lightspeed MCP Prerequisites - -**Action**: Execute the `/mcp-lightspeed-validator` skill - -**Note**: Can skip if validation was performed earlier in this session and succeeded. See [Validation Freshness Policy](../mcp-lightspeed-validator/SKILL.md#validation-freshness-policy). - -**How to invoke**: Execute the `/mcp-lightspeed-validator` skill - -**Handle validation result**: -- **If validation PASSED**: Continue to Step 1 -- **If validation PARTIAL** (connectivity test unavailable): - - Warn user: "Configuration appears correct but connectivity could not be tested" - - Ask: "Do you want to proceed? (yes/no)" - - If yes: Continue to Step 1 - - If no: Stop execution -- **If validation FAILED**: - - The validator provides error details and setup instructions - - Wait for user decision (setup/skip/abort) - - If user chooses "skip": Attempt Step 1 anyway (may fail) - - If user chooses "setup" or "abort": Stop execution - -**Example**: -``` -Before retrieving fleet inventory, I'll validate the Lightspeed MCP server configuration. - -[Invoke mcp-lightspeed-validator skill] - -✓ Lightspeed MCP validation successful. -Proceeding with fleet inventory query... -``` - -### Step 1: Retrieve System Inventory - -**Document Consultation** (REQUIRED - Execute FIRST): -1. **Action**: Read [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) using the Read tool to understand the `get_host_details` response format and pagination handling -2. **Output to user**: "I consulted vulnerability-logic.md to understand the `get_host_details` response format and pagination handling." - -**MCP Tool**: `get_host_details` (from lightspeed-mcp) - -**Purpose**: Query Lightspeed for comprehensive system information - -**Parameters**: See [references/01-parameter-reference.md](references/01-parameter-reference.md) for get_host_details/get_cve_systems parameters and response fields. - -**Verification Checklist**: -- ✓ Systems list returned with metadata -- ✓ Total count matches expectation -- ✓ System details include RHEL version, tags, status -- ✓ No authentication errors (401/403) - -**Key Fields to Extract**: -- `id`: Unique system identifier (use for remediation workflows) -- `display_name` / `fqdn`: Human-readable hostname -- `rhel_version`: OS version (critical for remediation compatibility) -- `tags`: Environment labels (production, staging, dev) -- `stale`: Whether system recently checked in (< 7 days) -- `last_seen`: Last Lightspeed client run timestamp - -### Step 2: Filter and Organize Systems - -**Document Consultation** (REQUIRED - Execute FIRST): -1. **Action**: Read [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) using the Read tool to understand fleet inventory reporting structure and best practices -2. **Output to user**: "I consulted vulnerability-logic.md to structure this inventory report." - -Apply user-requested filters and grouping. See [references/01-parameter-reference.md](references/01-parameter-reference.md) for filtering and sorting patterns. - -### Step 3: Query CVE-Affected Systems - -**MCP Tool**: `get_cve_systems` (from lightspeed-mcp) - -**Purpose**: Find systems affected by specific CVEs - -**Parameters**: `cve_id` (CVE-YYYY-NNNNN, uppercase). See [references/01-parameter-reference.md](references/01-parameter-reference.md). - -**Verification Checklist**: -- ✓ CVE ID matches request exactly -- ✓ System list includes remediation status for each -- ✓ Counts are accurate (affected, remediated, still vulnerable) -- ✓ `remediation_available` flag is present - -**Status Interpretation**: -``` -Status: "Vulnerable" -→ CVE affects this system, patch not applied -→ Action: Suggest remediation via `/remediation` skill - -Status: "Patched" -→ CVE previously affected, now remediated -→ Action: No action needed, informational only - -Status: "Not Affected" -→ System not vulnerable to this CVE -→ Action: Exclude from affected count -``` - -### Step 4: Generate Fleet Summary - -Create organized output. **Read [references/03-output-templates.md](references/03-output-templates.md)** for report format (Overview, RHEL/Environment breakdown, System Details, Stale Systems). - -### Step 5: Offer Remediation Transition - -When appropriate, suggest transitioning to the `/remediation` skill: - -```markdown -## Next Steps - -**For CVE Remediation**: -If you need to remediate vulnerabilities on any of these systems, I can help using the `/remediation` skill: - -Examples: -- "Remediate CVE-2024-1234 on web-server-01" -- "Create playbook for all RHEL 8 production systems affected by CVE-2024-5678" -- "Batch remediate critical CVEs on staging environment" - -**For System Investigation**: -- "Show CVEs affecting web-server-01" (use cve-impact skill) -- "Analyze risk for production systems" (use cve-impact skill) -- "List critical vulnerabilities across the fleet" (use cve-impact skill) -``` - -## Dependencies - -### Required MCP Servers -- `lightspeed-mcp` - Red Hat Lightspeed platform access for system inventory and CVE data - -### Required MCP Tools -- `get_host_details` (from lightspeed-mcp) - Retrieve all registered systems with metadata - - Parameters: Optional filters (system_id, hostname_pattern, tags, operating_system) - - Returns: List of systems with id, display_name, fqdn, rhel_version, tags, stale status - -- `get_cve_systems` (from lightspeed-mcp) - Find systems affected by specific CVEs - - Parameters: cve_id (string, format: CVE-YYYY-NNNNN) - - Returns: List of affected systems with vulnerability and remediation status - -### Related Skills -- `mcp-lightspeed-validator` - **PREREQUISITE** - Validates Lightspeed MCP server configuration and connectivity - - Use before: ALL fleet-inventory operations (Step 0 in workflow) - - Purpose: Ensures MCP server is available before attempting tool calls - - Prevents errors from missing configuration or credentials - -- `cve-impact` - Analyze CVE severity and risk after identifying affected systems - - Use after: "What systems are affected by CVE-X?" → "What's the risk of CVE-X?" - -- `cve-validation` - Validate CVE IDs before querying affected systems - - Use before: If CVE ID format is unclear, validate first - -- `system-context` - Get detailed system configuration for specific hosts - - Use after: Fleet discovery identifies systems needing deeper investigation - -- `/remediation` (skill) - Transition to remediation workflows after discovery - - Use after: "Show affected systems" → "Remediate those systems" - -### Reference Documentation -- [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) - CVE severity classification, API response patterns, and filtering strategies - -### Skill Orchestration Pattern - -**Information-First Workflow**: -``` -User Query: "Show the managed fleet" - ↓ -fleet-inventory skill (discovery) - ↓ -Systems identified: 42 total, 15 affected by CVE-2024-1234 - ↓ -User: "What's the risk of CVE-2024-1234?" - ↓ -cve-impact skill (analysis) - ↓ -CVSS 8.1, Critical severity, affects httpd package - ↓ -User: "Remediate CVE-2024-1234 on all production systems" - ↓ -`/remediation` skill (action) - ↓ -Playbook generated and executed -``` - -**Key Principle**: Always start with discovery before taking remediation actions. This ensures informed decisions based on actual fleet state. - -## Output, Examples, Error Handling - -**Read [references/03-output-templates.md](references/03-output-templates.md)** for report format. -**Read [references/04-examples.md](references/04-examples.md)** for fleet, CVE-affected, and environment-filter examples. -**Read [references/05-error-handling.md](references/05-error-handling.md)** for no-results, API errors, and stale system handling. - -## Best Practices - -Start broad then filter; group by environment/RHEL/tier; highlight stale systems; offer `/remediation` transitions; use tables and percentages; declare document consultations; verify prerequisites first. diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/01-parameter-reference.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/01-parameter-reference.md deleted file mode 100644 index 6909aa85..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/01-parameter-reference.md +++ /dev/null @@ -1,49 +0,0 @@ -# Fleet Inventory Parameter Reference - -Read when calling `get_host_details` or `get_cve_systems` to ensure correct parameters. - -## get_host_details - -**Parameters** (based on user query): - -```python -# No filters -get_host_details() - -# Specific system -get_host_details(system_id="abc-123") - -# Hostname pattern -get_host_details(hostname_pattern="web-*") - -# Tag filter -get_host_details(tags=["production"]) - -# RHEL version filter -get_host_details(operating_system__version__startswith="8") - -# Combined -get_host_details(tags=["production", "web-tier"], operating_system__version__startswith="8") -``` - -**Response fields**: id, display_name, fqdn, rhel_version, last_seen, tags, stale, satellite_managed - -## get_cve_systems - -**Parameters**: `cve_id` (string, format CVE-YYYY-NNNNN, uppercase) - -```python -get_cve_systems(cve_id="CVE-2024-1234") -``` - -**Response fields**: cve_id, affected_systems (system_id, display_name, status, remediation_available), total_affected, total_remediated, total_vulnerable - -**Status values**: Vulnerable (patch needed), Patched (no action), Not Affected (exclude) - -## Filtering and Sorting - -**By RHEL**: `[s for s in systems if s['rhel_version'].startswith("8")]` -**By tag**: `[s for s in systems if "production" in s.get('tags', [])]` -**By stale**: `[s for s in systems if not s.get('stale', False)]` -**Sort by last_seen**: `sorted(systems, key=lambda s: s['last_seen'], reverse=True)` -**Sort by display_name**: `sorted(systems, key=lambda s: s['display_name'])` diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/03-output-templates.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/03-output-templates.md deleted file mode 100644 index b0337ed8..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/03-output-templates.md +++ /dev/null @@ -1,80 +0,0 @@ -# Fleet Inventory Output Templates - -Read when completing a fleet inventory report to format the output. - -## Template 1: Full Fleet Listing - -**User Request**: "Show the managed fleet" - -```markdown -# Managed Fleet Inventory - -I consulted [fleet-management.md](../../../docs/insights/fleet-management.md) to structure this inventory report. - -Retrieved from Red Hat Lightspeed on YYYY-MM-DDTHH:MM:SSZ - -## Fleet Overview -- **Total Registered Systems**: N -- **Active (< 24h)**: N -- **Stale (> 7 days)**: N - -## RHEL Version Distribution -| Version | Count | Percentage | - -## Environment Breakdown -| Environment | Count | Systems | - -## Top 20 Systems (by last check-in) -[Table: display_name, rhel_version, tags, last_seen] - -**Would you like to**: Filter by environment/RHEL, view CVEs, create remediation plans -``` - -## Template 2: CVE-Affected Systems - -**User Request**: "What systems are affected by CVE-X?" - -```markdown -# CVE-X Impact Analysis - -## Affected Systems Summary -- **Total Vulnerable**: N -- **Already Patched**: N -- **Impact Rate**: X% of fleet - -## Vulnerable Systems -| System Name | RHEL Version | Environment | Remediation Available | - -## Already Patched (No Action Needed) -[list] - -## Next Steps -- Use `/remediation` skill for remediation -- Use cve-impact for severity analysis -``` - -## Template 3: Environment-Filtered View - -**User Request**: "Show me production systems" - -```markdown -# Production Systems Inventory - -Filtered by tag: "production" - -## Production Fleet Summary -- **Total**: N -- **RHEL 9.x / 8.x / 7.x** breakdown -- **Active / Stale** counts - -## System Tiers -### Web Tier, Database Tier, Application Tier -[grouped lists] - -## Stale System Alert ⚠️ -[list with action: investigate Lightspeed client] - -## Next Steps -- "Show CVEs affecting production systems" -- "Remediate CVE-X on production web tier" -``` diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/04-examples.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/04-examples.md deleted file mode 100644 index 2d08d77a..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/04-examples.md +++ /dev/null @@ -1,32 +0,0 @@ -# Fleet Inventory Examples - -## Example 1: General Fleet Query - -**User Request**: "Show the managed fleet" - -1. Invoke mcp-lightspeed-validator (Step 0) → PASSED -2. Call `get_host_details()` with no filters -3. Consult fleet-management.md for grouping -4. Group by RHEL version, environment tags -5. Generate Template 1 output -6. Offer next steps (CVE analysis, remediation) - -## Example 2: CVE Impact Query - -**User Request**: "What systems are affected by CVE-2024-1234?" - -1. Invoke mcp-lightspeed-validator (Step 0) → PASSED -2. Call `get_cve_systems(cve_id="CVE-2024-1234")` -3. Separate vulnerable vs. patched systems -4. Generate Template 2 output -5. Suggest /remediation for next steps - -## Example 3: Environment Filter - -**User Request**: "Show me staging systems" - -1. Invoke mcp-lightspeed-validator (Step 0) → PARTIAL -2. Ask user: "Proceed? (yes/no)" → yes -3. Call `get_host_details()` → filter by tag "staging" -4. Group by tier (hostname patterns) -5. Generate Template 3 output diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/05-error-handling.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/05-error-handling.md deleted file mode 100644 index e295f0e6..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/fleet-inventory/references/05-error-handling.md +++ /dev/null @@ -1,45 +0,0 @@ -# Fleet Inventory Error Handling - -Read when errors occur during fleet inventory queries. - -## No Systems Found - -``` -Fleet Inventory Query: No Results - -Query: [user's filter criteria] -Result: No systems match the specified criteria - -Possible reasons: No systems registered, filter too restrictive, tag mismatch -Troubleshooting: Verify at console.redhat.com/insights/inventory, try broader filters -Suggested: "Show the managed fleet" (no filters) -``` - -## Lightspeed API Error - -``` -❌ Fleet Inventory Query: API Error - -Possible causes: MCP not running, auth failure, network, service outage - -Troubleshooting: -1. Run /mcp-lightspeed-validator skill -2. Check LIGHTSPEED_CLIENT_ID and LIGHTSPEED_CLIENT_SECRET (never echo values) -3. Verify at console.redhat.com/settings/service-accounts -4. Check status.redhat.com - -Options: retry | setup | abort -``` - -## Stale System Warning - -``` -⚠️ Stale Systems Detected - -Systems not checked in > 7 days: [list] - -Impact: Vulnerability data may be outdated - -Actions: Verify insights-client, check connectivity, review logs, re-register if needed -Note: Stale systems included but may have outdated CVE data -``` diff --git a/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/mcp-lightspeed-validator/SKILL.md b/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/mcp-lightspeed-validator/SKILL.md deleted file mode 100644 index e1f1528e..00000000 --- a/evaluation/with_skills/rh-sre__fleet-inventory/environment/skills/mcp-lightspeed-validator/SKILL.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -name: mcp-lightspeed-validator -description: | - Validate Red Hat Lightspeed MCP server connectivity. Use when the user asks to "validate Lightspeed MCP", "check Lightspeed connection", or when other skills need to verify lightspeed-mcp availability before CVE operations. -model: haiku -color: yellow ---- - -# MCP Lightspeed Validator - -Validates connectivity to the Red Hat Lightspeed MCP server by running a lightweight tool call. - -## When to Use This Skill - -Use when validating Lightspeed MCP before CVE operations, troubleshooting connection issues, or when other skills (e.g. remediation) need to verify availability. Do NOT use for actual CVE queries—use cve-impact or cve-validation. - -## Workflow - -1. **Test connectivity**: Call `vulnerability__get_cves` with **no parameters** (uses default limit=10). Do NOT pass `limit`—some MCP clients incorrectly serialize it as `limit_`, causing validation errors. -2. **If it fails**: Provide a comprehensive message with possible root causes (see below). -3. **Report**: Output a table with validated servers and outcome (emojis). - -## Failure Message (Root Causes) - -When the tool call fails, include: - -``` -❌ Lightspeed MCP connection failed - -**Possible root causes:** -- **Credentials**: LIGHTSPEED_CLIENT_ID or LIGHTSPEED_CLIENT_SECRET not set or invalid -- **Expired credentials**: Red Hat Console tokens may have expired -- **Server not running**: MCP server/container may be stopped -- **Network**: Firewall or proxy blocking console.redhat.com -- **Configuration**: .mcp.json misconfigured or server not registered - -**Troubleshooting:** -1. Verify env vars: LIGHTSPEED_CLIENT_ID, LIGHTSPEED_CLIENT_SECRET (never echo values) -2. Check credentials at: https://console.redhat.com/settings/integrations -3. Restart MCP server or host after config changes -4. Check container logs if using podman/docker -``` - -## Report Format - -Always end with a table: - -| Server | Outcome | -|--------|---------| -| lightspeed-mcp | ✅ PASSED | -| lightspeed-mcp | ❌ FAILED | - -Use ✅ for success, ❌ for failure, ⚠️ for partial (e.g. connected but error on tool). - -## Dependencies - -### Required MCP Tools -- `vulnerability__get_cves` or `get_cves` (from lightspeed-mcp) - Connectivity test - -### Required MCP Servers -- `lightspeed-mcp` - Red Hat Lightspeed vulnerability and inventory data diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 1092dd1d..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- pass rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/SKILL.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/SKILL.md deleted file mode 100644 index 510776ca..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/SKILL.md +++ /dev/null @@ -1,321 +0,0 @@ ---- -name: job-template-creator -description: | - Create AAP (Ansible Automation Platform) job templates for executing playbooks. Use when users request: - - "Create a job template for this playbook" - - "Set up a template to run remediation playbooks" - - "Configure AAP to execute this playbook" - - "Add a new job template for CVE remediation" - - This skill guides through adding playbooks to Git projects and creating job templates via AAP Web UI. -model: inherit -color: blue ---- - -# AAP Job Template Creator Skill - -This skill helps SREs create AAP job templates for executing Ansible playbooks, particularly for CVE remediation workflows. - -## Prerequisites - -**Required AAP Components**: -- AAP (Ansible Automation Platform) instance with API access -- Projects configured with playbooks -- Inventories with target hosts -- Credentials for authentication - -**Required MCP Servers**: `aap-mcp-job-management` ([setup guide](https://docs.redhat.com/)) - -**Currently Available MCP Tools** (read-only): -- `job_templates_list` - List existing templates -- `job_templates_retrieve` - Get template details -- `projects_list` - List available projects -- `inventories_list` - List available inventories - -**Missing MCP Tools** (needed for creation): -- ⚠️ `job_templates_create` - **NOT CURRENTLY AVAILABLE** -- ⚠️ `job_templates_update` - **NOT CURRENTLY AVAILABLE** - -**Required Environment Variables**: -- `AAP_MCP_SERVER` - Base URL for the MCP endpoint of the AAP server (must point to the AAP MCP gateway) -- `AAP_API_TOKEN` - AAP API authentication token - -### Prerequisite Validation - -**CRITICAL**: Before executing operations, execute the `/mcp-aap-validator` skill to verify AAP MCP server availability. - -**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-aap-validator/SKILL.md#validation-freshness-policy). - -**How to invoke**: Execute the `/mcp-aap-validator` skill - -**Handle validation result**: -- **If validation PASSED**: Continue with job template creation workflow -- **If validation PARTIAL**: Warn user and ask to proceed -- **If validation FAILED**: Stop execution, provide setup instructions from validator - -## Current Limitation: No Create Tools Available - -⚠️ **IMPORTANT**: The current AAP MCP implementation does **NOT** include tools to create job templates programmatically. The available MCP tools are read-only (list, retrieve, launch). - -**Current Approach**: -- Job templates must be created through the **AAP Web UI** -- This skill provides step-by-step instructions for Web UI creation -- Future MCP tool additions will enable programmatic template creation - -This skill documents both the **current manual workflow** and the **intended automated workflow** for when creation tools become available. - -## When to Use This Skill - -**Use this skill when you need**: -- Create a new job template for a remediation playbook -- Configure AAP to execute dynamically generated playbooks -- Set up templates for CVE remediation workflows -- Automate job template creation as part of remediation setup - -**Do NOT use this skill when**: -- Job templates already exist (use `/playbook-executor` skill instead) -- Only need to execute existing templates (use `job_templates_launch_retrieve`) -- Need to modify existing templates (requires AAP Web UI currently) - -## Invocation from playbook-executor - -When invoked from the [playbook-executor](../playbook-executor/SKILL.md) skill (Scenario 3 - No suitable template), this skill receives playbook content in context. The playbook-executor invokes with an instruction such as: - -``` -Create a job template for this remediation playbook. Playbook: [content]. Filename: [filename]. Path: [our_playbook_path]. CVE: [cve_id]. Target systems: [list]. -``` - -**When playbook content is provided**: -- Use the provided content for Phase 1 (Prepare Playbook in Git) instead of asking the user to supply it -- Write the playbook to the specified path in the user's Git repository (ask for repo path if not provided) -- Follow the Git flow: add, commit (with checkpoint for confirmation), push -- Then guide template creation via AAP Web UI (Phase 4) -- **Output**: Include the created template ID and name in the final report so playbook-executor can retrieve and validate it - -**Phase 0 - Check context**: If playbook content is provided by the invoking skill, execute the git flow (write, add, commit with confirmation checkpoint, push) before guiding template creation. Otherwise, use the existing manual flow where the user supplies the playbook. - -## Workflow - -### Phase 0: Validate AAP MCP Prerequisites - -**Action**: Execute the `/mcp-aap-validator` skill - -**Note**: Can skip if validation was performed earlier in this session and succeeded. See [Validation Freshness Policy](../mcp-aap-validator/SKILL.md#validation-freshness-policy). - -**How to invoke**: Execute the `/mcp-aap-validator` skill - -**Handle validation result**: -- **If validation PASSED**: Continue to Phase 1 -- **If validation PARTIAL**: Warn user and ask to proceed -- **If validation FAILED**: Stop execution, user must set up AAP MCP servers - -### Phase 1: Prepare Playbook in Git Project - -**Goal**: Add playbook to a Git repository AAP can access. - -**Read [references/01-git-setup.md](references/01-git-setup.md)** for Option A (existing repo) and Option B (new repo). - -**Verification**: Playbook committed, pushed, AAP synced, playbook path noted. - -### Phase 2: Gather Required Information - -Before creating a job template, collect: - -1. **Playbook Information**: - - Playbook name/path (e.g., `remediation-CVE-2025-49794.yml`) - - Project where playbook is stored - - Required variables/parameters - -2. **Target Information**: - - Inventory containing target hosts - - Host groups or specific hosts to target - - Any host limits or filters - -3. **Credentials**: - - SSH credentials for host access - - Vault passwords (if playbook uses Ansible Vault) - - Cloud credentials (if targeting cloud resources) - -4. **Execution Settings**: - - Job type (run/check) - - Verbosity level - - Concurrent execution limits - - Timeout settings - -### Phase 3: Verify Prerequisites - -**Step 1: List Available Projects** - -**MCP Tool**: `projects_list` (from aap-mcp-job-management) - -**Parameters**: -- `page_size`: 50 (retrieve up to 50 projects) -- `search`: "remediation" (optional - filter by keyword) - -**Expected Output**: -```json -{ - "count": 1, - "results": [ - { - "id": 6, - "name": "Remediation Playbooks", - "scm_type": "git", - "scm_url": "https://github.com/org/playbooks.git", - "status": "successful" - } - ] -} -``` - -**Action**: Identify the project ID where your playbook is stored. - -**Step 2: List Available Inventories** - -**MCP Tool**: `inventories_list` (from aap-mcp-inventory-management) - -**Parameters**: -- `page_size`: 50 -- `search`: "production" (optional - filter by keyword) - -**Expected Output**: -```json -{ - "count": 1, - "results": [ - { - "id": 1, - "name": "Production Inventory", - "total_hosts": 150, - "has_active_failures": false - } - ] -} -``` - -**Action**: Identify the inventory ID containing your target hosts. - -**Step 3: Verify Credentials** - -**Note**: The current AAP MCP doesn't expose credential listing tools. You'll need credential IDs from AAP Web UI or administrator. - -### Phase 4: Create Job Template via AAP Web UI - -⚠️ **CURRENT LIMITATION**: AAP MCP has no create tools. Template creation must be done via AAP Web UI. - -**Read [references/02-web-ui-form.md](references/02-web-ui-form.md)** for form fields and steps. - -**Required**: Name, Inventory, Project, Playbook, Credentials. Enable Privilege Escalation. Prompt on Launch: Job Type (REQUIRED), Variables, Limit. - -### Phase 5: Verify Template Creation - -**MCP Tool**: `job_templates_list` (from aap-mcp-job-management) - -**Parameters**: -- `search`: "CVE-2025-49794" (search for your template) -- `page_size`: 10 - -**Expected Output**: -```json -{ - "results": [ - { - "id": 42, - "name": "Remediate CVE-2025-49794", - "playbook": "remediation-CVE-2025-49794.yml", - "project": 6, - "inventory": 1, - "status": "never updated" - } - ] -} -``` - -**Success Criteria**: -- ✓ Template appears in search results -- ✓ Playbook path matches your playbook -- ✓ Project and inventory IDs are correct -- ✓ Template status is valid - -### Phase 6: Test Template Execution (Optional) - -**MCP Tool**: `job_templates_launch_retrieve` (from aap-mcp-job-management) - -**Parameters**: -- `id`: "42" (template ID from Phase 5) - -**Expected Output**: -```json -{ - "job": 1234, - "status": "pending", - "url": "/api/controller/v2/jobs/1234/" -} -``` - -**Follow-up**: Use `playbook-executor` skill to track job execution. - -## Output and Examples - -**Read [references/03-output-template.md](references/03-output-template.md)** for report format. -**Read [references/04-examples.md](references/04-examples.md)** for CVE remediation and dynamic variable examples. - -## Dependencies - -### Required MCP Servers -- `aap-mcp-job-management` - AAP job management API access - -### Required MCP Tools (Current) -- `job_templates_list` - List existing templates (verification) -- `job_templates_retrieve` - Get template details (verification) -- `projects_list` - List available projects (prerequisite) -- `inventories_list` (from aap-mcp-inventory-management) - List inventories (prerequisite) - -### Missing MCP Tools (Needed for Full Automation) -- `job_templates_create` - Create new job templates -- `job_templates_update` - Modify existing templates -- `credentials_list` - List available credentials - -### Related Skills -- `mcp-aap-validator` - **PREREQUISITE** - Validates AAP MCP server before creation (invoke in Phase 0 if not validated in session) -- `job-template-remediation-validator` - Validates created template meets remediation requirements -- `playbook-executor` - Execute templates after creation -- `playbook-generator` - Generate remediation playbooks for templates -- `system-context` - Identify target systems for inventory selection - -### Reference Documentation -- [AAP 2.6 Job Templates Documentation](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates) -- [AAP 2.6 Creating Projects](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects) - -## Best Practices - -1. **Use descriptive template names** - Include CVE ID or purpose: "Remediate CVE-2025-49794" -2. **Enable variable prompts for flexibility** - Check "Variables" in the "Prompt on Launch" section for dynamic values -3. **Set appropriate timeouts** - CVE remediation can take time; set generous timeouts -4. **Use privilege escalation** - Most remediation requires sudo/root access -5. **Document template purpose** - Use description field to explain usage -6. **Version playbooks** - Keep playbooks in Git for change tracking -7. **Test templates first** - Use check mode or test inventory before production -8. **Set concurrent limits** - Prevent overwhelming infrastructure with simultaneous jobs -9. **Enable notifications** - Configure email/webhook alerts for job completion -10. **Regular template audits** - Review and update templates as playbooks evolve - -## Human-in-the-Loop Requirements - -This skill requires user confirmation for: - -1. **Git Operations** (adding playbook to repository): - - Display: "I'll help you add the playbook to your Git repository" - - Ask: "Proceed with Git operations (clone, commit, push)?" - - Wait for confirmation - -2. **Manual Template Creation** (AAP Web UI): - - Display: "Template creation requires using the AAP Web UI" - - Ask: "I'll provide step-by-step instructions. Ready to proceed?" - - Wait for confirmation - -3. **Test Execution** (optional verification): - - Ask: "Should I test the template by launching a job?" - - Wait for confirmation before launching - -**Never assume approval** - always wait for explicit user confirmation. diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/01-git-setup.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/01-git-setup.md deleted file mode 100644 index 1a367a2e..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/01-git-setup.md +++ /dev/null @@ -1,25 +0,0 @@ -# Git Setup for Playbooks - -Read when guiding user through Phase 1 (Prepare Playbook in Git). - -## Option A: Add to Existing Project - -1. Ask: repo URL, local path, or "I don't have one" -2. Clone or `cd` to repo -3. `mkdir -p playbooks/remediation`; copy playbook; `git add`; `git commit`; `git push` -4. Sync AAP project (Automation Execution → Projects → Sync) -5. Note playbook path: `playbooks/remediation/remediation-CVE-YYYY-NNNNN.yml` - -## Option B: Create New Repository - -1. `mkdir ansible-remediation-playbooks`; `git init`; `mkdir -p playbooks/remediation` -2. Copy playbook; create README, .gitignore; `git add .`; `git commit` -3. Create remote repo; `git remote add origin `; `git push -u origin main` -4. Add project in AAP Web UI (Automation Execution → Projects → Add) -5. Note playbook path - -## Verification Checklist - -- Playbook committed and pushed -- AAP project synced -- Playbook path noted for template creation diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/02-web-ui-form.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/02-web-ui-form.md deleted file mode 100644 index 690d63ec..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/02-web-ui-form.md +++ /dev/null @@ -1,24 +0,0 @@ -# AAP Web UI Job Template Form - -Read when guiding Phase 4 (Create Template via Web UI). AAP MCP has no create tools—use Web UI. - -## Form Fields - -**Required**: Name, Inventory, Project, Playbook, Credentials (Machine/SSH) -**Job Type**: Run (or Check for dry-run) -**Options**: Enable Privilege Escalation: Yes -**Prompt on Launch** (check): Job Type (REQUIRED), Variables, Limit - -**Extra Variables** (optional): -```yaml -target_cve: "CVE-YYYY-NNNNN" -remediation_mode: "automated" -verify_after: true -``` - -## Steps - -1. Automation Execution → Templates → Add → Job Template -2. Fill form; Save -3. Note template ID from URL or details -4. Verify via `job_templates_list(search="CVE-ID")` diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/03-output-template.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/03-output-template.md deleted file mode 100644 index 496d2c45..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/03-output-template.md +++ /dev/null @@ -1,20 +0,0 @@ -# Job Template Creation Output - -Read when completing template creation. - -## Report Format - -```markdown -# AAP Job Template Created - -**Name**: Remediate CVE-YYYY-NNNNN -**ID**: [template_id] -**Project**: [name] (ID: [id]) -**Playbook**: playbooks/remediation/remediation-CVE-YYYY-NNNNN.yml -**Inventory**: [name] (ID: [id]) - -## Next Steps -1. Execute via AAP Web UI or job_templates_launch_retrieve -2. Monitor via jobs_retrieve, jobs_stdout_retrieve -3. Verify via remediation-verifier skill -``` diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/04-examples.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/04-examples.md deleted file mode 100644 index d19c66c6..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/job-template-creator/references/04-examples.md +++ /dev/null @@ -1,19 +0,0 @@ -# Job Template Creator Examples - -Read when handling specific request types. - -## Example 1: CVE Remediation Template - -**Request**: "Create a job template for CVE-2025-49794 playbook" -- Phase 1: Git setup (see 01-git-setup.md)—add playbook, commit, push, sync AAP -- Phase 2: Gather playbook path, project, inventory -- Phase 3: projects_list, inventories_list -- Phase 4: Web UI instructions (see 02-web-ui-form.md) -- Phase 5: job_templates_list to verify - -## Example 2: Dynamic CVE Template - -**Request**: "Template with variable CVE ID" -- Enable "Prompt on Launch" → Variables -- Extra vars: cve_id, remediation_mode, verify_after -- Override at launch for different CVEs diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/mcp-aap-validator/SKILL.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/mcp-aap-validator/SKILL.md deleted file mode 100644 index a1c4f708..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/mcp-aap-validator/SKILL.md +++ /dev/null @@ -1,66 +0,0 @@ ---- -name: mcp-aap-validator -description: | - Validate AAP (Ansible Automation Platform) MCP server connectivity. Use when the user asks to "validate AAP MCP", "check AAP connection", or when other skills need to verify AAP MCP availability before job management or inventory operations. -model: haiku -color: yellow ---- - -# MCP AAP Validator - -Validates connectivity to AAP MCP servers by running lightweight tool calls. - -## When to Use This Skill - -Use when validating AAP MCP before job template operations, troubleshooting connection issues, or when other skills (e.g. playbook-executor) need to verify availability. Do NOT use for creating templates—use job-template-creator. - -## Workflow - -1. **Test connectivity**: Call these tools to verify each server responds: - - `job_templates_list` (page_size: 10) from aap-mcp-job-management - - `inventories_list` (page_size: 10) from aap-mcp-inventory-management -2. **If any fails**: Provide a comprehensive message with possible root causes (see below). -3. **Report**: Output a table with validated servers and outcome (emojis). - -## Failure Message (Root Causes) - -When a tool call fails, include: - -``` -❌ AAP MCP connection failed - -**Possible root causes:** -- **Credentials**: AAP_MCP_SERVER or AAP_API_TOKEN not set or invalid -- **401 Unauthorized**: Token expired or invalid → regenerate in AAP Web UI -- **403 Forbidden**: Token lacks RBAC permissions (need Job Templates, Inventories) -- **404 Not Found**: Wrong AAP_MCP_SERVER URL (must point to MCP gateway, not main AAP UI) -- **Connection timeout**: Server unreachable, firewall, or network issue -- **SSL/TLS error**: Certificate verification problem - -**Troubleshooting:** -1. Verify env vars: AAP_MCP_SERVER, AAP_API_TOKEN (never echo values) -2. Get token: AAP Web UI → Users → [Your User] → Tokens → Create -3. Ensure AAP_MCP_SERVER points to MCP gateway endpoint -4. Restart host after config changes -``` - -## Report Format - -Always end with a table: - -| Server | Outcome | -|--------|---------| -| aap-mcp-job-management | ✅ PASSED | -| aap-mcp-inventory-management | ✅ PASSED | - -Use ✅ for success, ❌ for failure, ⚠️ for partial (e.g. one server OK, one failed). - -## Dependencies - -### Required MCP Tools -- `job_templates_list` (from aap-mcp-job-management) - Connectivity test -- `inventories_list` (from aap-mcp-inventory-management) - Connectivity test - -### Required MCP Servers -- `aap-mcp-job-management` - AAP job template and execution -- `aap-mcp-inventory-management` - AAP inventory management diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/SKILL.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/SKILL.md deleted file mode 100644 index a29c9443..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/SKILL.md +++ /dev/null @@ -1,499 +0,0 @@ ---- -name: playbook-executor -description: | - **CRITICAL**: Use for Ansible playbook execution via AAP. DO NOT call AAP MCP tools directly. - - Execute remediation playbooks with job management, dry-run, and reporting. Use after playbook-generator. - - **Git Flow**: If template playbook path ≠ generated playbook, perform Git Flow (commit, push, sync) BEFORE launch. ---- - -# AAP Playbook Executor Skill - -This skill executes Ansible remediation playbooks through AAP (Ansible Automation Platform) with full job management capabilities. - -**Integration with Remediation Skill**: The `/remediation` skill orchestrates this skill as part of its Step 5 (Execute Playbook) workflow. For standalone playbook execution, you can invoke this skill directly. - -## Prerequisites - -**Required MCP Servers**: `aap-mcp-job-management`, `aap-mcp-inventory-management` ([setup guide](https://docs.redhat.com/)) - -**Required MCP Tools**: -- `job_templates_list` (from aap-mcp-job-management) - List job templates -- `job_templates_retrieve` (from aap-mcp-job-management) - Get template details -- `projects_list` (from aap-mcp-job-management) - Get project name and scm_url for Git Flow -- `job_templates_launch_retrieve` (from aap-mcp-job-management) - Launch jobs -- `jobs_retrieve` (from aap-mcp-job-management) - Get job status -- `jobs_stdout_retrieve` (from aap-mcp-job-management) - Get console output -- `jobs_job_events_list` (from aap-mcp-job-management) - Get task events -- `jobs_job_host_summaries_list` (from aap-mcp-job-management) - Get host statistics -- `inventories_list` (from aap-mcp-inventory-management) - List inventories -- `hosts_list` (from aap-mcp-inventory-management) - List inventory hosts - -**Required Environment Variables**: -- `AAP_MCP_SERVER` - Base URL for the MCP endpoint of the AAP server (must point to the AAP MCP gateway) -- `AAP_API_TOKEN` - AAP API authentication token - -### Prerequisite Validation - -**CRITICAL**: Before executing operations, execute the `/mcp-aap-validator` skill to verify AAP MCP server availability. - -**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-aap-validator/SKILL.md#validation-freshness-policy). - -**How to invoke**: Execute the `/mcp-aap-validator` skill - -**Handle validation result**: -- **If validation PASSED**: Continue with playbook execution workflow -- **If validation PARTIAL**: Warn user and ask to proceed -- **If validation FAILED**: Stop execution, provide setup instructions from validator - -**Human Notification on Failure**: -If prerequisites are not met: -- ❌ "Cannot proceed: AAP MCP servers are not available" -- 📋 "Setup required: Configure AAP_MCP_SERVER and AAP_API_TOKEN environment variables" -- ❓ "How would you like to proceed? (setup now / skip / abort)" -- ⏸️ Wait for user decision - -## When to Use This Skill - -**Use this skill directly when you need**: -- Execute a previously generated Ansible playbook via AAP -- Track the status of a running AAP job -- Monitor playbook job completion -- Run dry-run (check mode) before production execution -- Verify playbook execution succeeded - -**Use the `/remediation` skill when you need**: -- Full remediation workflow including playbook execution -- Integrated CVE analysis → playbook generation → execution → verification -- End-to-end remediation orchestration - -**How they work together**: The `/remediation` skill invokes this skill after generating a remediation playbook, managing the full workflow from analysis to verification. - -## Workflow - -**Git Flow is MANDATORY**: When the job template's playbook path differs from the generated playbook (or content must be updated), you MUST perform Git Flow (write, commit, push, sync) and receive "sync complete" from the user BEFORE launching any job. Do NOT skip this—launching without it executes the wrong playbook. - -### Phase 0: Validate AAP MCP Prerequisites - -**Action**: Execute the `/mcp-aap-validator` skill - -**Note**: Can skip if validation was performed earlier in this session and succeeded. - -**How to invoke**: Execute the `/mcp-aap-validator` skill - -**Handle validation result**: -- **If validation PASSED**: Continue to Phase 1 -- **If validation PARTIAL**: Warn user and ask to proceed -- **If validation FAILED**: Stop execution, user must set up AAP MCP servers - -### Phase 1: Job Template Selection and Playbook Preparation - -**Goal**: Identify an AAP job template suitable for executing the remediation playbook. **Git Flow is MANDATORY** before Phase 3 when the template points to a different playbook or when content must be updated. - -**Input**: Playbook content and metadata from playbook-generator (filename, CVE ID, target systems). The playbook YAML is already in context—do NOT regenerate it during Git Flow. Playbook path is derived from metadata: `playbooks/remediation/` (e.g., `playbooks/remediation/remediation-CVE-2025-49794.yml`). - -**BLOCKING**: You MUST NOT launch any job (dry-run or production) until the playbook is in the Git repo and the user has confirmed "sync complete". AAP executes from the synced project—there is no "override at launch". Launching without Git Flow executes the WRONG playbook. - -#### Step 1.1: Derive Playbook Path - -From playbook metadata (filename from playbook-generator): -- Use convention `playbooks/remediation/` -- Support both `remediation-CVE-*.yml` and `remediation-CVE-*-playbook.yml` patterns. -- Example: CVE-2026-26103 → `playbooks/remediation/remediation-CVE-2026-26103.yml` - -#### Step 1.2: List Templates and Validate Each Candidate - -**MCP Tool**: `job_templates_list` (from aap-mcp-job-management) - -**Parameters**: -- `page_size`: 50 (retrieve up to 50 templates) -- `search`: "" (search for all templates) - -**REQUIRED**: For each template in results: -1. Call `job_templates_retrieve(id)` to get full details -2. **Invoke the `/job-template-remediation-validator` skill** with the template ID to verify it meets remediation requirements (inventory, project, playbook, credentials, become_enabled) -3. Only include templates that PASS validation in the lists below - -Build two lists: -- **exact_match**: `template.playbook` equals `our_playbook_path` (normalize slashes; match if equal or basenames match) -- **compatible_other**: Passes job-template-remediation-validator but **different playbook path** (template points to e.g. `cve-remediation.yml` while we have `remediation-CVE-2026-26103.yml`) - -**Path normalization**: Normalize slashes, handle `playbooks/remediation/` prefix. Match if `template.playbook` equals `our_playbook_path` or if basenames match. **Different filenames = different path = Scenario 2.** - -#### Step 1.3: Scenario Selection (MANDATORY - Do Not Skip) - -**Scenario 1 - Same playbook path** (exact_match not empty): - -The template already points to our playbook path. The project may need the latest content. **Read [references/05-git-flow-prompts.md](references/05-git-flow-prompts.md)** for Scenario 1 prompt, options (A/B), and Git Flow steps. - -- **If A**: Execute Git Flow (see Git Flow section below). **BLOCK Phase 3** until user confirms "sync complete" or "done". -- **If B**: Wait for user confirmation. **BLOCK Phase 3** until user confirms. - -**Scenario 2 - Different playbook path** (compatible_other not empty, exact_match empty): - -**CRITICAL**: The template points to a DIFFERENT playbook than our generated playbook. You MUST NOT launch the job without Git Flow—AAP executes from synced content; there is no override at launch. **Read [references/05-git-flow-prompts.md](references/05-git-flow-prompts.md)** for Scenario 2 prompt and Git Flow steps. - -- **If yes**: Execute Git Flow. **BLOCK Phase 3** until Git Flow completes and user confirms "sync complete". -- **If no**: Fall through to Scenario 3. - -**Anti-pattern**: Do NOT say "I'll override with our playbook" and then launch—that is impossible. The playbook MUST be in the repo before launch. - -**Scenario 3 - No suitable template** (exact_match and compatible_other both empty, or user chose "no" in Scenario 2): - -Execute the `/job-template-creator` skill with instruction: -``` -"Create a job template for this remediation playbook. Playbook: [content]. Filename: [filename]. Path: [our_playbook_path]. CVE: [cve_id]. Target systems: [list]." -``` - -The job-template-creator skill guides the user through: (1) Adding playbook to Git repository, (2) Syncing AAP project, (3) Creating job template via AAP Web UI with correct path, inventory, credentials, privilege escalation. - -After `/job-template-creator` completes, retrieve the template ID (from skill output or user confirmation). Execute `/job-template-remediation-validator` to validate the newly created template. If passed, proceed to Phase 3 (Dry-Run). If failed, report issues and ask user to fix in AAP Web UI. - -**Multiple matches**: If multiple exact matches, present list and ask user to choose by number. If multiple different-path matches, prefer by project name containing "remediation" or "CVE", else first. - -**Phase 1 Checkpoint** (BLOCKING - must pass before Phase 3): -- **Git Flow required**: If Scenario 1 or 2, you MUST complete Git Flow and receive "sync complete" from the user before proceeding. Do NOT skip. -- **No override**: There is no way to "override" the playbook at launch. AAP runs whatever is in the synced project. -- **Never launch** if the playbook has not been committed, pushed, and synced - -#### Git Flow (for Scenario 1 Override and Scenario 2) - MANDATORY HITL - -**When**: Scenario 1 (same path, update content) or Scenario 2 (different path, replace playbook). **Do not skip**—execution with wrong playbook content will remediate the wrong CVE. - -**Target path**: -- Scenario 1: `our_playbook_path` (e.g. `playbooks/remediation/remediation-CVE-2026-26103.yml`) -- Scenario 2: `template.playbook` (e.g. `playbooks/remediation/cve-remediation.yml`)—we replace the template's playbook with our generated content - -**Prerequisite**: Ask user for the local path to the Git repository. Use `projects_list` for project name and `scm_url`. **Read [references/05-git-flow-prompts.md](references/05-git-flow-prompts.md)** for repo path question, HITL checkpoint text, and after-push message. - -**Steps** (execute in order; HITL at checkpoint): -1. **Write playbook to file** (FAST—do NOT regenerate): - - The playbook content is ALREADY in context from playbook-generator (or remediation skill). Use it directly. - - **⚠️ ABSOLUTE PATH REQUIRED**: The Write path MUST start with `/`. Use: `/`. Example: `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project/playbooks/remediation/cve-remediation.yml` - - **WRONG** (causes "Error writing file"): `test-aap-project/playbooks/...` or `playbooks/remediation/...` — these are relative and fail when repo is outside workspace. - - **Before Write**: Confirm path starts with `/`. If not, prepend the user's repo path. - - Do NOT invoke playbook-generator, do NOT call MCP tools, do NOT re-fetch. This should take seconds, not minutes. -2. Use Run tool: `git add ` (from repo root, e.g. `git add playbooks/remediation/cve-remediation.yml`) -3. **HITL Checkpoint** (REQUIRED): Display summary per reference file. Wait for "yes" or "proceed" -4. If confirmed: `git commit -m "Add/update remediation playbook for CVE-YYYY-NNNNN"` -5. `git push origin main` (or branch from project's scm_branch if available) - -**Note**: Git must be configured. Use Run tool for git commands. - -**Do NOT proceed to Phase 3 (Dry-Run) until user confirms sync complete.** - -### Phase 2: Git Flow (MANDATORY before Phase 3) - -**BLOCKING**: You MUST NOT proceed to Phase 3 (Dry-Run) until Git Flow is complete. - -**When**: Scenario 1 (same path, update content) or Scenario 2 (different path). See Phase 1 Step 1.3. - -**Checkpoint**: Before Phase 3, confirm: -- [ ] Playbook written to repo at target path -- [ ] Git commit and push completed (with user confirmation) -- [ ] User confirmed "sync complete" after AAP project sync - -**If any unchecked**: STOP. Do Git Flow. Do NOT launch the job. - -### Phase 3: Dry-Run Execution (Recommended) - -**Prerequisite**: Phase 2 (Git Flow) MUST be complete. User must have confirmed "sync complete". - -**Goal**: Test playbook in check mode before actual execution to simulate changes. - -**Read [references/04-dry-run-display-templates.md](references/04-dry-run-display-templates.md)** for: Playbook Preview, Dry-Run Offer, Dry-Run Results Display, Proceed prompt. - -#### Step 3.1–3.2: Display Preview and Offer Dry-Run - -Show playbook structure per reference. Offer dry-run with options: yes / no / abort. **ONLY if user confirms**, proceed. - -#### Step 3.3: Launch Dry-Run Job - -**Pre-launch check** (BLOCKING): If Scenario 1 or 2 applied, you MUST have completed Git Flow and received "sync complete" from the user. If not, STOP—do not launch. Return to Phase 2 / Git Flow. - -**MCP Tool**: `job_templates_launch_retrieve` (from aap-mcp-job-management) - -**Parameters**: `id`, `requestBody` with `job_type: "check"`, `extra_vars`, `limit` - -**Key**: `job_type: "check"` - Runs Ansible in check mode (dry-run) - -#### Step 3.4: Monitor Dry-Run Progress - -Poll `jobs_retrieve` every 2 seconds. Use `jobs_job_events_list` for live task updates. - -#### Step 3.5: Display Dry-Run Results - -**MCP Tools**: `jobs_stdout_retrieve` (id, format: "txt"), `jobs_job_host_summaries_list` (id). Use display format from reference. - -#### Step 3.6: Proceed to Actual Execution? - -Ask per reference. Wait for "yes" or "execute". - -### Phase 4: Actual Execution - -**ONLY execute if user explicitly confirms** (either after dry-run or directly if they skipped dry-run). - -#### Step 4.1: Final Confirmation - -``` -⚠️ CRITICAL: Playbook Execution Confirmation Required - -This playbook will: -- Execute on: 3 production systems -- Update packages: httpd (2.4.53-7.el9 → 2.4.57-8.el9) -- Restart services: httpd -- Estimated downtime: ~10 seconds per system -- Requires reboot: No - -Job Template: CVE Remediation Template (ID: 10) -AAP URL: https://aap.example.com/jobs/ - -❓ Execute this playbook now? - -Options: -- "yes" or "execute" - Proceed with execution -- "abort" - Cancel execution - -Please respond with your choice. -``` - -Wait for explicit "yes" or "execute" response. - -#### Step 4.2: Launch Production Job - -**Pre-launch check** (BLOCKING): Same as Phase 3—if Scenario 1 or 2 applied, Git Flow must be complete and user must have confirmed "sync complete". Do NOT launch without it. - -**MCP Tool**: `job_templates_launch_retrieve` (from aap-mcp-job-management) - -**Parameters**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run", - "extra_vars": { - "target_cve": "CVE-2025-49794", - "remediation_mode": "automated", - "verify_after": true - }, - "limit": "prod-web-01,prod-web-02,prod-web-03" - } -} -``` - -**Key Parameter**: `job_type: "run"` - Runs Ansible in execution mode (actual changes) - -**Expected Output**: -```json -{ - "job": 1235, - "status": "pending", - "url": "/api/controller/v2/jobs/1235/" -} -``` - -#### Step 4.3: Monitor Execution Progress - -**Polling Strategy**: -1. Call `jobs_retrieve(id=job_id)` every 2 seconds -2. Get task events with `jobs_job_events_list(id=job_id)` for progress updates -3. Display real-time task completion status -4. Continue until status is "successful", "failed", or "error" - -**Progress Display**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running -Elapsed: 1m 23s -AAP URL: https://aap.example.com/#/jobs/playbook/1235 - -Recent Events: -- ✓ Gathering Facts (completed - all hosts) -- ✓ Check Disk Space (completed - all hosts) -- ✓ Backup Configuration (completed - all hosts) -- ⏳ Update Package: httpd (running - prod-web-01, prod-web-02) - └─ prod-web-01: Installing httpd-2.4.57-8.el9... - └─ prod-web-02: Installing httpd-2.4.57-8.el9... -- ⏸ Restart Service: httpd (pending) -``` - -**Update every 2 seconds** until completion. - -### Phase 5: Execution Report - -**Goal**: Generate comprehensive report with job details, per-host results, and full output. - -**Read [references/01-execution-report-templates.md](references/01-execution-report-templates.md)** for JSON examples, comprehensive report template, and Success/Partial Success/Failure output templates. - -#### Step 5.1–5.4: Gather Data - -**MCP Tools** (all from aap-mcp-job-management): -- `jobs_retrieve` (id) - Job details -- `jobs_job_host_summaries_list` (id) - Per-host stats -- `jobs_job_events_list` (id) - Task timeline -- `jobs_stdout_retrieve` (id, format: "txt") - Full console output - -#### Step 5.5: Generate Report - -Format all gathered data per reference. Use Success / Partial Success / Failure template based on job status. - -#### Step 5.6: Validate Job Log for CVE Handling (MANDATORY) - -**Goal**: Confirm from the job stdout that the playbook actually addressed the target CVE(s). - -**Input**: Target CVE ID(s) from invocation (e.g. CVE-2025-49794). Job stdout from `jobs_stdout_retrieve` (already gathered in Step 5.4). - -**Parse stdout for**: -- Target CVE ID(s) in output (vars, task names, audit logs, playbook metadata) -- Package update tasks for affected packages (dnf/yum install/update, package module) -- Remediation-related task names (e.g. "Update package", "Restart service", "remediation") - -**Report** (add to execution report): -- **✓ Job log confirms CVE-XXXX-YYYY was addressed** — CVE ID or package updates found in stdout -- **⚠️ Job log did not show clear evidence of CVE handling** — No CVE ID or package updates found; recommend manual verification or `/remediation-verifier` - -**Batch**: For multiple CVEs, validate each. Report per-CVE confirmation or warning. - -### Phase 6: Error Handling - -**If job status is "failed" or "error"**, provide detailed troubleshooting. - -**Read [references/02-error-handling-guide.md](references/02-error-handling-guide.md)** for: Error categories, error report template, troubleshooting steps, relaunch parameters. - -#### Step 6.1: Parse Error Output - -**MCP Tool**: `jobs_stdout_retrieve`. Analyze output for error categories per reference. - -#### Step 6.2: Generate Error Report - -Use error report template from reference. Include per-host results, failed task details, troubleshooting steps, relaunch options. - -#### Step 6.3: Offer Relaunch - -If user chooses relaunch: **MCP Tool** `jobs_relaunch_retrieve` with `hosts: "failed"`, `job_type: "run"` per reference. - -## Reference Files - -| File | Use When | -|------|----------| -| [01-execution-report-templates.md](references/01-execution-report-templates.md) | Phase 5 reports, Success/Partial/Failure output | -| [02-error-handling-guide.md](references/02-error-handling-guide.md) | Phase 6 error reports, relaunch | -| [03-workflow-examples.md](references/03-workflow-examples.md) | Demo full workflow, failure handling, skip dry-run | -| [04-dry-run-display-templates.md](references/04-dry-run-display-templates.md) | Phase 3 preview, offer, results, proceed prompt | -| [05-git-flow-prompts.md](references/05-git-flow-prompts.md) | Scenario 1/2 prompts, Git Flow HITL, after-push | - -## Dependencies - -### Required MCP Servers -- `aap-mcp-job-management` - AAP job management and execution -- `aap-mcp-inventory-management` - AAP inventory management - -### Required MCP Tools -- `job_templates_list` (from aap-mcp-job-management) - List templates -- `job_templates_retrieve` (from aap-mcp-job-management) - Get template details -- `projects_list` (from aap-mcp-job-management) - Get project name and scm_url for Git Flow -- `job_templates_launch_retrieve` (from aap-mcp-job-management) - Launch jobs -- `jobs_retrieve` (from aap-mcp-job-management) - Get job status -- `jobs_stdout_retrieve` (from aap-mcp-job-management) - Get console output -- `jobs_job_events_list` (from aap-mcp-job-management) - Get task events -- `jobs_job_host_summaries_list` (from aap-mcp-job-management) - Get host statistics -- `inventories_list` (from aap-mcp-inventory-management) - List inventories -- `hosts_list` (from aap-mcp-inventory-management) - List hosts - -### Related Skills -- `mcp-aap-validator` - **PREREQUISITE** - Validates AAP MCP servers (invoke in Phase 0) -- `job-template-remediation-validator` - **REQUIRED** - Invoke for each candidate template in Phase 1 Step 1.2 to verify remediation requirements -- `job-template-creator` - Creates/guides AAP job template setup -- `playbook-generator` - Generates playbooks for execution -- `remediation-verifier` - Verifies success after execution - -### Reference Documentation -- [references/](references/) - Step-numbered reference files (01–05) for templates and examples -- [AAP Job Execution Guide](../../docs/ansible/aap-job-execution.md) - AAP job execution best practices -- [Playbook Integration with AAP](../../docs/ansible/playbook-integration-aap.md) - Playbook-to-AAP workflow - -## Critical: Human-in-the-Loop Requirements - -This skill executes code on production systems. **Explicit user confirmation is REQUIRED** at multiple stages. - -**Before Git commit/push** (Scenario 1 Override, Scenario 2): -1. **Display change summary**: File path, diff or file size -2. **Ask for confirmation**: "Ready to commit and push these changes? Reply 'yes' or 'proceed' to continue, or 'abort' to cancel." -3. **Wait for explicit "yes" or "proceed"**: Do not commit/push without confirmation - -**Before Dry-Run Execution** (if user chooses dry-run): -1. **Display Playbook Preview**: Show tasks and explain changes -2. **Ask for Dry-Run Confirmation**: - ``` - ❓ Run dry-run to simulate changes? - - Options: - - "yes" - Run dry-run (recommended) - - "no" - Skip to actual execution - - "abort" - Cancel - - Please respond with your choice. - ``` -3. **Wait for Explicit Response**: Do not proceed without confirmation - -**Before Actual Execution** (REQUIRED): -1. **Display Execution Summary**: Show systems, changes, downtime estimate -2. **Ask for Final Confirmation**: - ``` - ⚠️ CRITICAL: Execute playbook on production systems? - - This will make real changes to N systems. - - Options: - - "yes" or "execute" - Proceed - - "abort" - Cancel - - Please respond with your choice. - ``` -3. **Wait for Explicit "yes" or "execute"**: Do not proceed without confirmation - -**Never assume approval** - always wait for explicit user confirmation before executing playbooks. - -## Best Practices - -1. **Write path must be absolute** - When Git Flow writes the playbook to the user's repo, use `/playbooks/remediation/`. The path MUST start with `/`. Relative paths cause "Error writing file". -2. **Always validate AAP prerequisites** - Invoke mcp-aap-validator in Phase 0 -3. **Validate each template** - Invoke job-template-remediation-validator for each candidate before selection -4. **Never skip Git Flow** - If template playbook path ≠ generated playbook path (Scenario 2) or content must be updated (Scenario 1), you MUST complete Git Flow and receive "sync complete" before Phase 3. Do NOT launch without it. -5. **Recommend dry-run** - Offer check mode before production execution -6. **Filter compatible templates** - Check inventory, project, and credentials match -7. **Monitor in real-time** - Display task progress during execution -8. **Comprehensive reporting** - Include per-host stats, task timeline, full output -9. **Error categorization** - Parse errors and provide specific troubleshooting -10. **Relaunch capability** - Offer to retry failed hosts -11. **Link to AAP** - Provide direct URL to job in AAP Web UI -12. **Suggest verification** - Always recommend remediation-verifier after success -13. **Document job details** - Save job ID and template info for audit trail - -## Integration with Other Skills - -- **playbook-generator**: Generates playbooks that this skill executes -- **job-template-creator**: Creates AAP job templates when needed -- **remediation-verifier**: Verifies success after this skill completes execution -- **`/remediation` skill**: Orchestrates full workflow including playbook execution - -**Orchestration Example** (from `/remediation` skill): -1. Agent invokes playbook-generator skill → Creates playbook YAML -2. playbook-generator asks for confirmation → User approves playbook content -3. Agent invokes playbook-executor skill (this skill) → Execution workflow -4. Skill validates templates via job-template-remediation-validator → Filters valid candidates -5. Skill checks path match → If different path, offers Git Flow (HITL: commit/push, sync AAP) -6. Skill waits for "sync complete" before proceeding (if Git Flow was used) -7. Skill offers dry-run → User runs check mode -8. Skill asks for execution confirmation → User approves -9. Skill executes and monitors → Reports completion -10. Agent invokes remediation-verifier skill → Confirms CVE resolved - -**Note**: Both playbook-generator and playbook-executor require separate confirmations for different purposes: -- playbook-generator: Confirms playbook content is acceptable -- playbook-executor: Confirms execution on production systems is approved - -This two-step approval ensures user control over both what to run and when to run it. diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/01-execution-report-templates.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/01-execution-report-templates.md deleted file mode 100644 index a6773c5f..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/01-execution-report-templates.md +++ /dev/null @@ -1,168 +0,0 @@ -# Step 01: Execution Report Templates - -Read this reference when generating Phase 5 execution reports or output templates. - -## Phase 5: Job Details (JSON Examples) - -### jobs_retrieve Expected Output - -```json -{ - "id": 1235, - "name": "CVE Remediation Template", - "status": "successful", - "started": "2026-02-24T15:35:02Z", - "finished": "2026-02-24T15:40:25Z", - "elapsed": 323.45, - "job_template": 10, - "inventory": 1, - "limit": "prod-web-01,prod-web-02,prod-web-03", - "playbook": "playbooks/remediation/remediation-CVE-2025-49794.yml" -} -``` - -### jobs_job_host_summaries_list Expected Output - -```json -{ - "results": [ - { - "host_name": "prod-web-01", - "ok": 8, - "changed": 3, - "failed": 0, - "unreachable": 0 - }, - { - "host_name": "prod-web-02", - "ok": 8, - "changed": 3, - "failed": 0, - "unreachable": 0 - }, - { - "host_name": "prod-web-03", - "ok": 5, - "changed": 0, - "failed": 1, - "unreachable": 0 - } - ] -} -``` - -## Comprehensive Report Template - -```markdown -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 5m 23s -**Started**: 2026-02-24 15:35:02 UTC -**Completed**: 2026-02-24 15:40:25 UTC -**Job Template**: CVE Remediation Template -**Playbook**: playbooks/remediation/remediation-CVE-2025-49794.yml -**AAP URL**: [View in AAP](https://aap.example.com/#/jobs/playbook/1235) - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| prod-web-01 | 8 | 3 | 0 | 0 | ✅ Success | -| prod-web-02 | 8 | 3 | 0 | 0 | ✅ Success | -| prod-web-03 | 8 | 3 | 0 | 0 | ✅ Success | - -**Summary**: 3 of 3 hosts successfully remediated - -## Task Timeline -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) - - prod-web-01: 2.4.53-7.el9 → 2.4.57-8.el9 - - prod-web-02: 2.4.53-7.el9 → 2.4.57-8.el9 - - prod-web-03: 2.4.53-7.el9 → 2.4.57-8.el9 -5. ✅ Restart httpd service (15s) -6. ✅ Verify service status (2s) -7. ✅ Update audit log (1s) - -## Full Console Output -
-Click to expand (187 lines) - -[Full stdout from jobs_stdout_retrieve] - -
- -## Job Log CVE Validation (Step 5.6) -✓ Job log confirms CVE-XXXX-YYYY was addressed - -*(Or: ⚠️ Job log did not show clear evidence of CVE handling—verify manually or use remediation-verifier)* - -## Next Steps -1. ✅ All systems successfully remediated -2. ☐ Verify remediation with remediation-verifier skill -3. ☐ Update vulnerability tracking system -4. ☐ Schedule follow-up verification in 24-48 hours - ---- - -**Recommendation**: Run remediation-verifier skill to confirm CVE status has been updated in Red Hat Lightspeed. -``` - -## Output Templates - -### Success Template - -```markdown -✅ Playbook Execution Successful - -Job ID: 1235 -Duration: 5m 23s -Systems Remediated: 3 of 3 - -View full report above for details. - -Next Steps: -- Run remediation-verifier skill to confirm CVE resolution -- Update vulnerability tracking system -- Monitor systems for 24-48 hours - -AAP URL: https://aap.example.com/#/jobs/playbook/1235 -``` - -### Partial Success Template - -```markdown -⚠️ Playbook Execution Completed with Failures - -Job ID: 1235 -Duration: 2m 45s -Systems Remediated: 2 of 3 -Failed Systems: prod-web-03 - -See error details above for troubleshooting steps. - -Options: -- Relaunch for failed hosts -- Manual remediation -- Skip failed hosts - -AAP URL: https://aap.example.com/#/jobs/playbook/1235 -``` - -### Failure Template - -```markdown -❌ Playbook Execution Failed - -Job ID: 1235 -Duration: 1m 15s -Systems Remediated: 0 of 3 - -Critical errors prevented execution. -See error details above for troubleshooting. - -AAP URL: https://aap.example.com/#/jobs/playbook/1235 -``` diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/02-error-handling-guide.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/02-error-handling-guide.md deleted file mode 100644 index 90492f00..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/02-error-handling-guide.md +++ /dev/null @@ -1,108 +0,0 @@ -# Step 02: Error Handling Guide - -Read this reference when generating Phase 6 error reports or troubleshooting. - -## Error Categories - -**Parse error output** from `jobs_stdout_retrieve` for these common patterns: - -1. **Connection Failures**: SSH timeout, host unreachable, authentication failed -2. **Permission Errors**: sudo required, insufficient privileges, SELinux denials -3. **Package Manager Issues**: repo unavailable, package not found, dependency conflicts -4. **Service Failures**: service not found, restart failed, timeout -5. **Disk Space**: insufficient space for updates -6. **General Failures**: playbook syntax errors, task failures - -## Error Report Template - -```markdown -# Playbook Execution Failed - -## Job Summary -**Job ID**: 1235 -**Status**: ❌ Failed -**Duration**: 2m 45s -**Started**: 2026-02-24 15:35:02 UTC -**Failed At**: 2026-02-24 15:37:47 UTC -**Job Template**: CVE Remediation Template -**AAP URL**: [View in AAP](https://aap.example.com/#/jobs/playbook/1235) - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| prod-web-01 | 8 | 3 | 0 | 0 | ✅ Success | -| prod-web-02 | 8 | 3 | 0 | 0 | ✅ Success | -| prod-web-03 | 5 | 0 | 1 | 0 | ❌ Failed | - -**Summary**: 2 of 3 hosts succeeded, 1 failed - -## Failed Tasks Details - -### Host: prod-web-03 - -**Task**: Restart httpd service -**Error**: "Failed to restart httpd.service: Unit httpd.service not found." - -**Error Category**: Service Failure - -**Root Cause**: The httpd service is not installed or not recognized by systemd. - -**Troubleshooting Steps**: -1. Check if httpd is installed: - ```bash - ssh prod-web-03 'rpm -q httpd' - ``` -2. If not installed, the package update may have failed: - ```bash - ssh prod-web-03 'dnf info httpd' - ``` -3. Check systemd service status: - ```bash - ssh prod-web-03 'systemctl status httpd' - ``` -4. Review package manager logs: - ```bash - ssh prod-web-03 'tail -50 /var/log/dnf.log' - ``` - -**Recommended Action**: -- Verify httpd package installation on prod-web-03 -- Check if package update completed successfully -- Manually install httpd if needed: `dnf install httpd` -- Relaunch job for failed host only - -## Console Output (Last 50 Lines) -
-Click to expand error context - -[Relevant error output from jobs_stdout_retrieve] - -
- -## Relaunch Options - -Would you like to: -1. **Relaunch for failed hosts only** - Run job again with limit="prod-web-03" -2. **Fix issues manually and relaunch** - Resolve problems first, then relaunch -3. **View full job output** - See complete execution logs -4. **Abort** - Stop remediation workflow - -Please choose an option (1-4): -``` - -## Relaunch Parameters - -**MCP Tool**: `jobs_relaunch_retrieve` (from aap-mcp-job-management) - -**Parameters**: -```json -{ - "id": "1235", - "requestBody": { - "hosts": "failed", - "job_type": "run" - } -} -``` - -This relaunches the job for only the failed hosts. diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/03-workflow-examples.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/03-workflow-examples.md deleted file mode 100644 index f5caa0d1..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/03-workflow-examples.md +++ /dev/null @@ -1,119 +0,0 @@ -# Step 03: Workflow Examples - -Read this reference when demonstrating end-to-end workflow patterns. - -## Example 1: Full Workflow with Dry-Run - -**User Request**: "Execute the CVE-2025-49794 remediation playbook" - -**Skill Response**: - -1. **Validate AAP Prerequisites**: - - Invoke mcp-aap-validator skill → PASSED - -2. **List Job Templates**: - - Call `job_templates_list()` → Found 2 templates - - Filter compatible templates → 1 matches requirements - -3. **User Selects Template**: - ``` - Found 1 compatible job template: - 1. "CVE Remediation Template" (ID: 10) - - Select template (1) or "create" for new: 1 - ``` - -4. **Playbook Preparation**: - ``` - Guide user to add playbook to Git: - - Commands provided - - User syncs AAP project - - Verification: Playbook available ✓ - ``` - -5. **Offer Dry-Run**: - ``` - Run dry-run first? yes - ``` - -6. **Execute Dry-Run**: - - Launch with `job_type="check"` - - Monitor progress → COMPLETED - - Display dry-run results: - ``` - Would change 3 tasks on 3 hosts - No errors detected - ``` - -7. **Proceed to Actual Execution**: - ``` - Proceed with actual execution? yes - ``` - -8. **Execute Playbook**: - - Launch with `job_type="run"` - - Monitor progress → COMPLETED - - Generate report: - ``` - ✅ 3 of 3 hosts successfully remediated - Duration: 5m 23s - ``` - -9. **Suggest Verification**: - ``` - Recommend: Run remediation-verifier skill to confirm CVE resolution - ``` - -## Example 2: Handle Execution Failure - -**User Request**: "Execute remediation playbook" - -**Skill Response**: - -1-7. [Same as Example 1 through execution] - -8. **Execute Playbook**: - - Launch job → FAILED on 1 of 3 hosts - - Generate error report: - ``` - ❌ prod-web-03 failed - Error: httpd service not found - Troubleshooting steps provided - ``` - -9. **Offer Relaunch**: - ``` - Relaunch for failed host only? yes - ``` - -10. **Relaunch Job**: - - Call `jobs_relaunch_retrieve` with hosts="failed" - - Monitor → COMPLETED - - Final report: - ``` - ✅ All 3 hosts successfully remediated (1 after retry) - ``` - -## Example 3: Skip Dry-Run - -**User Request**: "Execute playbook directly, skip dry-run" - -**Skill Response**: - -1-4. [Same as Example 1 through template selection] - -5. **Offer Dry-Run**: - ``` - Run dry-run first? no - ``` - -6. **Final Confirmation**: - ``` - ⚠️ Execute on production without dry-run? - This will make changes immediately. - Confirm: yes - ``` - -7. **Execute Playbook**: - - Launch with `job_type="run"` - - Monitor and report as in Example 1 diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/04-dry-run-display-templates.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/04-dry-run-display-templates.md deleted file mode 100644 index d9b6f0dc..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/04-dry-run-display-templates.md +++ /dev/null @@ -1,93 +0,0 @@ -# Step 04: Dry-Run Display Templates - -Read this reference when displaying Phase 3 dry-run content. - -## Playbook Preview - -```markdown -# Playbook Preview - -**Playbook**: remediation-CVE-2025-49794.yml -**Target Systems**: 5 systems - -## Tasks Overview: -1. **Gather Facts** - Collect system information -2. **Check Disk Space** - Ensure sufficient space for updates (>500MB) -3. **Backup Configuration** - Snapshot critical configs -4. **Update Package: httpd** - Upgrade to version 2.4.57-8.el9 -5. **Restart Service: httpd** - Apply changes -6. **Verify Service Status** - Confirm httpd is running -7. **Update Audit Log** - Record remediation event - -**Estimated Duration**: 3-5 minutes per system -**Requires Reboot**: No -**Downtime**: Brief (~10 seconds during service restart) -``` - -## Dry-Run Offer - -``` -⚠️ Recommended: Run dry-run first - -Dry-run mode (--check) simulates changes without applying them. -This helps identify: -- Package availability issues -- Permission problems -- Configuration conflicts -- Unexpected side effects - -❓ Run dry-run before actual execution? -- "yes" - Run dry-run first (recommended) -- "no" - Skip to actual execution -- "abort" - Cancel execution - -Please respond with your choice. -``` - -## Dry-Run Results Display - -```markdown -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s -**Completed**: 2024-01-20 15:32:17 UTC - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| prod-web-01 | 3 | 8 | 0 | ✓ Ready | -| prod-web-02 | 3 | 8 | 0 | ✓ Ready | -| prod-web-03 | 3 | 8 | 0 | ✓ Ready | - -## Changes That Would Be Made: -1. **httpd package** - Would update from 2.4.53-7.el9 to 2.4.57-8.el9 -2. **httpd service** - Would restart -3. **audit log** - Would add remediation entry - -## Dry-Run Output: -
-Click to expand full output - -[Full stdout from jobs_stdout_retrieve] - -
- -✓ No errors detected in dry-run -✓ All systems passed pre-flight checks -``` - -## Proceed to Actual Execution Prompt - -``` -❓ Dry-run completed successfully. Proceed with actual execution? - -Options: -- "yes" or "execute" - Proceed with actual remediation -- "review" - Show dry-run output again -- "abort" - Cancel execution - -Please respond with your choice. -``` diff --git a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/05-git-flow-prompts.md b/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/05-git-flow-prompts.md deleted file mode 100644 index 41945d0e..00000000 --- a/evaluation/with_skills/rh-sre__job-template-creator/environment/skills/playbook-executor/references/05-git-flow-prompts.md +++ /dev/null @@ -1,97 +0,0 @@ -# Step 05: Git Flow Prompts - -Read this reference when executing Git Flow (Scenario 1 Override or Scenario 2). - -## Scenario 1 Prompt (Same path) - -The template already points to our playbook path. The project may need the latest content. - -``` -Found template [name] (ID: X) with matching playbook path. The project may need to be updated with the latest playbook. - -Options: -(A) Override: I'll add/update the playbook in the project via git. You sync the AAP project, then confirm. -(B) Manual: You add the playbook and sync. Confirm when done. - -❓ Choose (A) or (B): -``` - -- **If A**: Execute Git Flow (see Git Flow section below). Wait for user: "Sync complete" or "done". -- **If B**: Wait for user confirmation. - -## Scenario 2 Prompt (Different path) - -**CRITICAL**: The template points to a DIFFERENT playbook than our generated playbook. You MUST NOT launch the job without Git Flow. AAP executes from synced project content—there is no "override at launch". The playbook MUST be in the repo before any job launch. - -``` -Found template [name] (ID: X) pointing to [template.playbook]. Our generated playbook is [our_playbook_path]. - -⚠️ The template's playbook path does NOT match. We must update the playbook in the project before execution. - -Options: -- "yes" or "proceed" - I'll add our playbook to the project via git (you'll confirm commit/push, then sync AAP) -- "no" - Create a new template via `/job-template-creator` skill - -❓ Proceed with playbook update (git flow)? -``` - -- **If yes**: Execute Git Flow. **Do NOT proceed to Phase 3 until Git Flow completes.** -- **If no**: Fall through to Scenario 3 (job-template-creator). - -## Repo Path Question - -``` -What is the local path to the Git repository for project [Project Name] (scm_url)? -``` - -Use `projects_list` to get project name and `scm_url`; display to help user identify the repo. - -**Path format**: Ask for the **absolute path** (e.g. `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project`). When writing the playbook, the Write tool path MUST be `/playbooks/remediation/` — the full absolute path. Do NOT use a relative path like `test-aap-project/playbooks/...`; that causes "Error writing file". - -## Git Flow: Write Step (FAST) - -**CRITICAL**: The playbook is ALREADY generated. During Git Flow you must WRITE the existing content to disk—nothing more. - -- **DO**: Single file write of the playbook content already in context (from playbook-generator or remediation) -- **DO NOT**: Invoke playbook-generator again, call create_vulnerability_playbook, re-fetch from MCP, or validate/transform the content -- **Expected duration**: Seconds. If it takes minutes, you are doing unnecessary work. - -### Write Path (ABSOLUTE REQUIRED) - -**⚠️ WRITE PATH MUST START WITH `/`** — The Write tool path MUST be an absolute path. Relative paths cause "Error writing file" because the repo is often outside the workspace. - -**Formula**: `write_path = user_provided_path + "/" + target_path` - -- `user_provided_path` = exactly what the user typed (e.g. `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project`) -- `target_path` = e.g. `playbooks/remediation/cve-remediation.yml` - -**Correct**: `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project/playbooks/remediation/cve-remediation.yml` - -**WRONG** (will fail): -- `test-aap-project/playbooks/remediation/cve-remediation.yml` -- `playbooks/remediation/cve-remediation.yml` - -**Before calling Write**: Verify the path starts with `/`. If it does not, prepend the user's repo path. - -## Git Flow HITL Checkpoint - -**REQUIRED** before commit/push: - -``` -Ready to commit and push these changes? -- File: [target_path] -- CVE: [cve_id] -- This will update the playbook in the AAP project. - -Reply 'yes' or 'proceed' to continue, or 'abort' to cancel. -``` - -**Wait for user confirmation.** If "yes" or "proceed": `git commit -m "Add/update remediation playbook for CVE-YYYY-NNNNN"` then `git push origin main`. - -## After Push Message - -``` -I've pushed the playbook. Sync the AAP project: Automation Execution > Projects > [Project] > Sync. Reply 'sync complete' when done. -``` - -**Do NOT proceed to Phase 3 (Dry-Run) until user confirms sync complete.** diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 1092dd1d..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- pass rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/SKILL.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/SKILL.md deleted file mode 100644 index 510776ca..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/SKILL.md +++ /dev/null @@ -1,321 +0,0 @@ ---- -name: job-template-creator -description: | - Create AAP (Ansible Automation Platform) job templates for executing playbooks. Use when users request: - - "Create a job template for this playbook" - - "Set up a template to run remediation playbooks" - - "Configure AAP to execute this playbook" - - "Add a new job template for CVE remediation" - - This skill guides through adding playbooks to Git projects and creating job templates via AAP Web UI. -model: inherit -color: blue ---- - -# AAP Job Template Creator Skill - -This skill helps SREs create AAP job templates for executing Ansible playbooks, particularly for CVE remediation workflows. - -## Prerequisites - -**Required AAP Components**: -- AAP (Ansible Automation Platform) instance with API access -- Projects configured with playbooks -- Inventories with target hosts -- Credentials for authentication - -**Required MCP Servers**: `aap-mcp-job-management` ([setup guide](https://docs.redhat.com/)) - -**Currently Available MCP Tools** (read-only): -- `job_templates_list` - List existing templates -- `job_templates_retrieve` - Get template details -- `projects_list` - List available projects -- `inventories_list` - List available inventories - -**Missing MCP Tools** (needed for creation): -- ⚠️ `job_templates_create` - **NOT CURRENTLY AVAILABLE** -- ⚠️ `job_templates_update` - **NOT CURRENTLY AVAILABLE** - -**Required Environment Variables**: -- `AAP_MCP_SERVER` - Base URL for the MCP endpoint of the AAP server (must point to the AAP MCP gateway) -- `AAP_API_TOKEN` - AAP API authentication token - -### Prerequisite Validation - -**CRITICAL**: Before executing operations, execute the `/mcp-aap-validator` skill to verify AAP MCP server availability. - -**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-aap-validator/SKILL.md#validation-freshness-policy). - -**How to invoke**: Execute the `/mcp-aap-validator` skill - -**Handle validation result**: -- **If validation PASSED**: Continue with job template creation workflow -- **If validation PARTIAL**: Warn user and ask to proceed -- **If validation FAILED**: Stop execution, provide setup instructions from validator - -## Current Limitation: No Create Tools Available - -⚠️ **IMPORTANT**: The current AAP MCP implementation does **NOT** include tools to create job templates programmatically. The available MCP tools are read-only (list, retrieve, launch). - -**Current Approach**: -- Job templates must be created through the **AAP Web UI** -- This skill provides step-by-step instructions for Web UI creation -- Future MCP tool additions will enable programmatic template creation - -This skill documents both the **current manual workflow** and the **intended automated workflow** for when creation tools become available. - -## When to Use This Skill - -**Use this skill when you need**: -- Create a new job template for a remediation playbook -- Configure AAP to execute dynamically generated playbooks -- Set up templates for CVE remediation workflows -- Automate job template creation as part of remediation setup - -**Do NOT use this skill when**: -- Job templates already exist (use `/playbook-executor` skill instead) -- Only need to execute existing templates (use `job_templates_launch_retrieve`) -- Need to modify existing templates (requires AAP Web UI currently) - -## Invocation from playbook-executor - -When invoked from the [playbook-executor](../playbook-executor/SKILL.md) skill (Scenario 3 - No suitable template), this skill receives playbook content in context. The playbook-executor invokes with an instruction such as: - -``` -Create a job template for this remediation playbook. Playbook: [content]. Filename: [filename]. Path: [our_playbook_path]. CVE: [cve_id]. Target systems: [list]. -``` - -**When playbook content is provided**: -- Use the provided content for Phase 1 (Prepare Playbook in Git) instead of asking the user to supply it -- Write the playbook to the specified path in the user's Git repository (ask for repo path if not provided) -- Follow the Git flow: add, commit (with checkpoint for confirmation), push -- Then guide template creation via AAP Web UI (Phase 4) -- **Output**: Include the created template ID and name in the final report so playbook-executor can retrieve and validate it - -**Phase 0 - Check context**: If playbook content is provided by the invoking skill, execute the git flow (write, add, commit with confirmation checkpoint, push) before guiding template creation. Otherwise, use the existing manual flow where the user supplies the playbook. - -## Workflow - -### Phase 0: Validate AAP MCP Prerequisites - -**Action**: Execute the `/mcp-aap-validator` skill - -**Note**: Can skip if validation was performed earlier in this session and succeeded. See [Validation Freshness Policy](../mcp-aap-validator/SKILL.md#validation-freshness-policy). - -**How to invoke**: Execute the `/mcp-aap-validator` skill - -**Handle validation result**: -- **If validation PASSED**: Continue to Phase 1 -- **If validation PARTIAL**: Warn user and ask to proceed -- **If validation FAILED**: Stop execution, user must set up AAP MCP servers - -### Phase 1: Prepare Playbook in Git Project - -**Goal**: Add playbook to a Git repository AAP can access. - -**Read [references/01-git-setup.md](references/01-git-setup.md)** for Option A (existing repo) and Option B (new repo). - -**Verification**: Playbook committed, pushed, AAP synced, playbook path noted. - -### Phase 2: Gather Required Information - -Before creating a job template, collect: - -1. **Playbook Information**: - - Playbook name/path (e.g., `remediation-CVE-2025-49794.yml`) - - Project where playbook is stored - - Required variables/parameters - -2. **Target Information**: - - Inventory containing target hosts - - Host groups or specific hosts to target - - Any host limits or filters - -3. **Credentials**: - - SSH credentials for host access - - Vault passwords (if playbook uses Ansible Vault) - - Cloud credentials (if targeting cloud resources) - -4. **Execution Settings**: - - Job type (run/check) - - Verbosity level - - Concurrent execution limits - - Timeout settings - -### Phase 3: Verify Prerequisites - -**Step 1: List Available Projects** - -**MCP Tool**: `projects_list` (from aap-mcp-job-management) - -**Parameters**: -- `page_size`: 50 (retrieve up to 50 projects) -- `search`: "remediation" (optional - filter by keyword) - -**Expected Output**: -```json -{ - "count": 1, - "results": [ - { - "id": 6, - "name": "Remediation Playbooks", - "scm_type": "git", - "scm_url": "https://github.com/org/playbooks.git", - "status": "successful" - } - ] -} -``` - -**Action**: Identify the project ID where your playbook is stored. - -**Step 2: List Available Inventories** - -**MCP Tool**: `inventories_list` (from aap-mcp-inventory-management) - -**Parameters**: -- `page_size`: 50 -- `search`: "production" (optional - filter by keyword) - -**Expected Output**: -```json -{ - "count": 1, - "results": [ - { - "id": 1, - "name": "Production Inventory", - "total_hosts": 150, - "has_active_failures": false - } - ] -} -``` - -**Action**: Identify the inventory ID containing your target hosts. - -**Step 3: Verify Credentials** - -**Note**: The current AAP MCP doesn't expose credential listing tools. You'll need credential IDs from AAP Web UI or administrator. - -### Phase 4: Create Job Template via AAP Web UI - -⚠️ **CURRENT LIMITATION**: AAP MCP has no create tools. Template creation must be done via AAP Web UI. - -**Read [references/02-web-ui-form.md](references/02-web-ui-form.md)** for form fields and steps. - -**Required**: Name, Inventory, Project, Playbook, Credentials. Enable Privilege Escalation. Prompt on Launch: Job Type (REQUIRED), Variables, Limit. - -### Phase 5: Verify Template Creation - -**MCP Tool**: `job_templates_list` (from aap-mcp-job-management) - -**Parameters**: -- `search`: "CVE-2025-49794" (search for your template) -- `page_size`: 10 - -**Expected Output**: -```json -{ - "results": [ - { - "id": 42, - "name": "Remediate CVE-2025-49794", - "playbook": "remediation-CVE-2025-49794.yml", - "project": 6, - "inventory": 1, - "status": "never updated" - } - ] -} -``` - -**Success Criteria**: -- ✓ Template appears in search results -- ✓ Playbook path matches your playbook -- ✓ Project and inventory IDs are correct -- ✓ Template status is valid - -### Phase 6: Test Template Execution (Optional) - -**MCP Tool**: `job_templates_launch_retrieve` (from aap-mcp-job-management) - -**Parameters**: -- `id`: "42" (template ID from Phase 5) - -**Expected Output**: -```json -{ - "job": 1234, - "status": "pending", - "url": "/api/controller/v2/jobs/1234/" -} -``` - -**Follow-up**: Use `playbook-executor` skill to track job execution. - -## Output and Examples - -**Read [references/03-output-template.md](references/03-output-template.md)** for report format. -**Read [references/04-examples.md](references/04-examples.md)** for CVE remediation and dynamic variable examples. - -## Dependencies - -### Required MCP Servers -- `aap-mcp-job-management` - AAP job management API access - -### Required MCP Tools (Current) -- `job_templates_list` - List existing templates (verification) -- `job_templates_retrieve` - Get template details (verification) -- `projects_list` - List available projects (prerequisite) -- `inventories_list` (from aap-mcp-inventory-management) - List inventories (prerequisite) - -### Missing MCP Tools (Needed for Full Automation) -- `job_templates_create` - Create new job templates -- `job_templates_update` - Modify existing templates -- `credentials_list` - List available credentials - -### Related Skills -- `mcp-aap-validator` - **PREREQUISITE** - Validates AAP MCP server before creation (invoke in Phase 0 if not validated in session) -- `job-template-remediation-validator` - Validates created template meets remediation requirements -- `playbook-executor` - Execute templates after creation -- `playbook-generator` - Generate remediation playbooks for templates -- `system-context` - Identify target systems for inventory selection - -### Reference Documentation -- [AAP 2.6 Job Templates Documentation](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates) -- [AAP 2.6 Creating Projects](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects) - -## Best Practices - -1. **Use descriptive template names** - Include CVE ID or purpose: "Remediate CVE-2025-49794" -2. **Enable variable prompts for flexibility** - Check "Variables" in the "Prompt on Launch" section for dynamic values -3. **Set appropriate timeouts** - CVE remediation can take time; set generous timeouts -4. **Use privilege escalation** - Most remediation requires sudo/root access -5. **Document template purpose** - Use description field to explain usage -6. **Version playbooks** - Keep playbooks in Git for change tracking -7. **Test templates first** - Use check mode or test inventory before production -8. **Set concurrent limits** - Prevent overwhelming infrastructure with simultaneous jobs -9. **Enable notifications** - Configure email/webhook alerts for job completion -10. **Regular template audits** - Review and update templates as playbooks evolve - -## Human-in-the-Loop Requirements - -This skill requires user confirmation for: - -1. **Git Operations** (adding playbook to repository): - - Display: "I'll help you add the playbook to your Git repository" - - Ask: "Proceed with Git operations (clone, commit, push)?" - - Wait for confirmation - -2. **Manual Template Creation** (AAP Web UI): - - Display: "Template creation requires using the AAP Web UI" - - Ask: "I'll provide step-by-step instructions. Ready to proceed?" - - Wait for confirmation - -3. **Test Execution** (optional verification): - - Ask: "Should I test the template by launching a job?" - - Wait for confirmation before launching - -**Never assume approval** - always wait for explicit user confirmation. diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/01-git-setup.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/01-git-setup.md deleted file mode 100644 index 1a367a2e..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/01-git-setup.md +++ /dev/null @@ -1,25 +0,0 @@ -# Git Setup for Playbooks - -Read when guiding user through Phase 1 (Prepare Playbook in Git). - -## Option A: Add to Existing Project - -1. Ask: repo URL, local path, or "I don't have one" -2. Clone or `cd` to repo -3. `mkdir -p playbooks/remediation`; copy playbook; `git add`; `git commit`; `git push` -4. Sync AAP project (Automation Execution → Projects → Sync) -5. Note playbook path: `playbooks/remediation/remediation-CVE-YYYY-NNNNN.yml` - -## Option B: Create New Repository - -1. `mkdir ansible-remediation-playbooks`; `git init`; `mkdir -p playbooks/remediation` -2. Copy playbook; create README, .gitignore; `git add .`; `git commit` -3. Create remote repo; `git remote add origin `; `git push -u origin main` -4. Add project in AAP Web UI (Automation Execution → Projects → Add) -5. Note playbook path - -## Verification Checklist - -- Playbook committed and pushed -- AAP project synced -- Playbook path noted for template creation diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/02-web-ui-form.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/02-web-ui-form.md deleted file mode 100644 index 690d63ec..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/02-web-ui-form.md +++ /dev/null @@ -1,24 +0,0 @@ -# AAP Web UI Job Template Form - -Read when guiding Phase 4 (Create Template via Web UI). AAP MCP has no create tools—use Web UI. - -## Form Fields - -**Required**: Name, Inventory, Project, Playbook, Credentials (Machine/SSH) -**Job Type**: Run (or Check for dry-run) -**Options**: Enable Privilege Escalation: Yes -**Prompt on Launch** (check): Job Type (REQUIRED), Variables, Limit - -**Extra Variables** (optional): -```yaml -target_cve: "CVE-YYYY-NNNNN" -remediation_mode: "automated" -verify_after: true -``` - -## Steps - -1. Automation Execution → Templates → Add → Job Template -2. Fill form; Save -3. Note template ID from URL or details -4. Verify via `job_templates_list(search="CVE-ID")` diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/03-output-template.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/03-output-template.md deleted file mode 100644 index 496d2c45..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/03-output-template.md +++ /dev/null @@ -1,20 +0,0 @@ -# Job Template Creation Output - -Read when completing template creation. - -## Report Format - -```markdown -# AAP Job Template Created - -**Name**: Remediate CVE-YYYY-NNNNN -**ID**: [template_id] -**Project**: [name] (ID: [id]) -**Playbook**: playbooks/remediation/remediation-CVE-YYYY-NNNNN.yml -**Inventory**: [name] (ID: [id]) - -## Next Steps -1. Execute via AAP Web UI or job_templates_launch_retrieve -2. Monitor via jobs_retrieve, jobs_stdout_retrieve -3. Verify via remediation-verifier skill -``` diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/04-examples.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/04-examples.md deleted file mode 100644 index d19c66c6..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-creator/references/04-examples.md +++ /dev/null @@ -1,19 +0,0 @@ -# Job Template Creator Examples - -Read when handling specific request types. - -## Example 1: CVE Remediation Template - -**Request**: "Create a job template for CVE-2025-49794 playbook" -- Phase 1: Git setup (see 01-git-setup.md)—add playbook, commit, push, sync AAP -- Phase 2: Gather playbook path, project, inventory -- Phase 3: projects_list, inventories_list -- Phase 4: Web UI instructions (see 02-web-ui-form.md) -- Phase 5: job_templates_list to verify - -## Example 2: Dynamic CVE Template - -**Request**: "Template with variable CVE ID" -- Enable "Prompt on Launch" → Variables -- Extra vars: cve_id, remediation_mode, verify_after -- Override at launch for different CVEs diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-remediation-validator/SKILL.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-remediation-validator/SKILL.md deleted file mode 100644 index c86141d4..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/job-template-remediation-validator/SKILL.md +++ /dev/null @@ -1,414 +0,0 @@ ---- -name: job-template-remediation-validator -description: | - Verify an AAP job template meets requirements for executing CVE remediation playbooks. - - Use when: - - "Does this job template support remediation playbooks?" - - "Validate job template X for CVE remediation" - - "Check if template is ready for playbook-executor" - - Before playbook-executor selects a template - - NOT for: AAP MCP connectivity (use `/mcp-aap-validator`), creating templates (use `/job-template-creator`). -model: inherit -color: blue ---- - -# AAP Job Template Remediation Validator - -This skill verifies that an AAP (Ansible Automation Platform) job template meets the requirements for executing CVE remediation playbooks as defined by the remediation skill and playbook-executor workflow. - -## Prerequisites - -**Required MCP Servers**: `aap-mcp-job-management`, `aap-mcp-inventory-management` ([setup guide](https://docs.redhat.com/)) - -**Required MCP Tools**: -- `job_templates_list` (from aap-mcp-job-management) - List job templates -- `job_templates_retrieve` (from aap-mcp-job-management) - Get template details -- `projects_list` (from aap-mcp-job-management) - Verify project exists and status -- `inventories_list` (from aap-mcp-inventory-management) - Verify inventory exists - -**Required Environment Variables**: -- `AAP_MCP_SERVER` - Base URL for the MCP endpoint of the AAP server (must point to the AAP MCP gateway) -- `AAP_API_TOKEN` - AAP API authentication token - -### Prerequisite Validation - -**CRITICAL**: Before executing, execute the `/mcp-aap-validator` skill to verify AAP MCP server availability. - -**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-aap-validator/SKILL.md#validation-freshness-policy). - -**How to invoke**: Execute the `/mcp-aap-validator` skill - -**Handle validation result**: -- **If validation PASSED**: Continue with template validation -- **If validation PARTIAL**: Warn user and ask to proceed -- **If validation FAILED**: Stop execution, provide setup instructions from validator - -**Human Notification on Failure**: -If prerequisites are not met: -- ❌ "Cannot proceed: AAP MCP servers are not available" -- 📋 "Setup required: Configure AAP_MCP_SERVER and AAP_API_TOKEN environment variables" -- ❓ "How would you like to proceed? (setup now / skip / abort)" -- ⏸️ Wait for user decision - -## When to Use This Skill - -**Use this skill when**: -- Verifying a job template before playbook execution -- Checking if a template meets remediation requirements -- Auditing existing templates for remediation readiness -- Troubleshooting "template not compatible" in playbook-executor - -**Do NOT use when**: -- Validating AAP MCP connectivity → Use `/mcp-aap-validator` skill -- Creating new job templates → Use `/job-template-creator` skill -- Executing playbooks → Use `/playbook-executor` skill - -## Remediation Template Requirements - -This skill validates against the requirements documented in [playbook-executor](../playbook-executor/SKILL.md) and [job-template-creator](../job-template-creator/SKILL.md). - -### Required (Must Pass) - -| Requirement | Description | Validation | -|-------------|-------------|------------| -| **Inventory** | Template has inventory configured | `inventory` field present and non-null | -| **Project** | Template has project configured | `project` field present and non-null | -| **Playbook** | Template has playbook path | `playbook` field present, non-empty | -| **Credentials** | Machine credential (SSH) configured | `summary_fields.credentials` or `credentials` has at least one credential | -| **Privilege Escalation** | Required for package updates | `become_enabled` is true | -| **Ask Job Type on Launch** | Required for dry-run and run modes | `ask_job_type_on_launch` is true | - -**Why Ask Job Type on Launch**: playbook-executor uses the same template for dry-run (`job_type: "check"`) and actual execution (`job_type: "run"`). Without `ask_job_type_on_launch: true`, the template is locked to one mode and you would need separate templates for check vs run. - -**Example**: Template with `job_type: "check"` (default) and `ask_job_type_on_launch: true` allows launching as check for dry-run or run for execution. - -### Recommended (Warnings if Missing) - -| Requirement | Description | Validation | -|-------------|-------------|------------| -| **Ask Variables on Launch** | Enables dynamic CVE targeting | `ask_variables_on_launch` is true | -| **Ask Limit on Launch** | Enables host targeting at launch | `ask_limit_on_launch` is true | -| **Ask Inventory on Launch** | Enables inventory override at launch | `ask_inventory_on_launch` is true | - -### Optional Context Checks - -| Check | Description | -|-------|-------------| -| **Project Status** | Project exists and is synced (status "successful") | -| **Inventory Exists** | Inventory exists in AAP | -| **Playbook Path** | Path suggests remediation playbook (e.g., contains "remediation") | -| **Playbook Path Matching** | When used by playbook-executor (Scenario 3), the template's playbook path is trusted to match the playbook just created via job-template-creator | - -## Workflow - -### Phase 0: Validate AAP MCP Prerequisites - -**Action**: Execute the `/mcp-aap-validator` skill - -**Note**: Can skip if validation was performed earlier in this session and succeeded. - -**Handle validation result**: -- **If validation PASSED**: Continue to Phase 1 -- **If validation PARTIAL**: Warn user and ask to proceed -- **If validation FAILED**: Stop execution - -### Phase 1: Obtain Job Template - -**Goal**: Get the job template to validate. User may provide template ID or name. - -#### Option A: User Provides Template ID - -If user specifies a template ID (e.g., "42" or "template 42"): - -**MCP Tool**: `job_templates_retrieve` (from aap-mcp-job-management) - -**Parameters**: -- `id`: Template ID as string (e.g., `"42"`) - -**Expected Output**: Full job template object with fields: `id`, `name`, `inventory`, `project`, `playbook`, `become_enabled`, `ask_variables_on_launch`, `ask_limit_on_launch`, `summary_fields` (may include `credentials`), `credentials` (array of credential IDs) - -**Error Handling**: -- If 404 or template not found: Report "Template ID X not found. Verify the ID exists in AAP." -- If connection error: Report per mcp-aap-validator troubleshooting - -#### Option B: User Provides Template Name or No ID - -If user says "validate my remediation template" or provides a name: - -**MCP Tool**: `job_templates_list` (from aap-mcp-job-management) - -**Parameters**: -- `page_size`: 50 -- `search`: User-provided name or "remediation" (optional) - -**Action**: List templates, let user select by number or ID. If exactly one match, use it. If multiple, present list and ask user to choose. - -### Phase 2: Validate Required Fields - -**Goal**: Check each required field against the template response. - -**Input**: Template object from `job_templates_retrieve` - -**Validation Logic**: - -``` -required_checks = [] -required_checks.append(("Inventory", template.get("inventory") is not None and template.get("inventory") != "")) -required_checks.append(("Project", template.get("project") is not None and template.get("project") != "")) -required_checks.append(("Playbook", template.get("playbook") is not None and len(str(template.get("playbook", "")).strip()) > 0)) -required_checks.append(("Privilege Escalation", template.get("become_enabled") == True)) - -# Credentials: AAP API may return credentials in summary_fields.credentials or credentials array -creds = template.get("summary_fields", {}).get("credentials") or template.get("credentials") or [] -has_creds = (isinstance(creds, list) and len(creds) > 0) or (isinstance(creds, dict) and creds) -required_checks.append(("Credentials", has_creds)) -required_checks.append(("Ask Job Type on Launch", template.get("ask_job_type_on_launch") == True)) -``` - -**Note**: If the AAP MCP response structure differs, adapt the field paths. Common AAP API response structures: -- `inventory`: number (ID) -- `project`: number (ID) -- `playbook`: string (path) -- `become_enabled`: boolean -- `credentials`: array of credential IDs, or `summary_fields.credentials` array of objects with `id`, `name` - -### Phase 3: Validate Recommended Fields - -**Validation Logic**: - -``` -recommended_checks = [] -recommended_checks.append(("Ask Variables on Launch", template.get("ask_variables_on_launch") == True)) -recommended_checks.append(("Ask Limit on Launch", template.get("ask_limit_on_launch") == True)) -recommended_checks.append(("Ask Inventory on Launch", template.get("ask_inventory_on_launch") == True)) -``` - -### Phase 4: Optional Context Verification - -**Goal**: Verify referenced project and inventory exist and are usable. - -**Step 4.1: Verify Project Exists and Status** - -**MCP Tool**: `projects_list` (from aap-mcp-job-management) - -**Parameters**: -- `page_size`: 100 -- `search`: Optional - filter by project ID if API supports it - -**Action**: Search results for `id == template["project"]`. If found, check `status`: -- `"successful"`: ✓ Project synced, playbooks available -- `"failed"` or `"error"`: ⚠ Project sync failed - playbooks may be stale -- `"pending"` or `"running"`: ⚠ Project syncing - wait before use - -**Step 4.2: Verify Inventory Exists** - -**MCP Tool**: `inventories_list` (from aap-mcp-inventory-management) - -**Parameters**: -- `page_size`: 100 - -**Action**: Search results for `id == template["inventory"]`. If found: ✓ Inventory exists. If not found: ⚠ Inventory ID not found (may be permission issue). - -### Phase 5: Generate Validation Report - -**Output Format**: - -```markdown -# Job Template Remediation Validation Report - -**Template**: {name} (ID: {id}) -**Validated**: {timestamp} - -## Required Checks -| Requirement | Status | Details | -|-------------|--------|---------| -| Inventory | ✓/✗ | {inventory_id} - {inventory_name or "configured"} | -| Project | ✓/✗ | {project_id} - {project_name or "configured"} | -| Playbook | ✓/✗ | {playbook_path} | -| Credentials | ✓/✗ | {count} credential(s) configured | -| Privilege Escalation | ✓/✗ | become_enabled: {value} | -| Ask Job Type on Launch | ✓/✗ | Required for dry-run + run modes | - -## Recommended Checks -| Requirement | Status | Details | -|-------------|--------|---------| -| Ask Variables on Launch | ✓/⚠ | {value} | -| Ask Limit on Launch | ✓/⚠ | {value} | -| Ask Inventory on Launch | ✓/⚠ | {value} | - -## Context Verification -| Check | Status | Details | -|-------|--------|---------| -| Project Exists | ✓/⚠/✗ | {status} | -| Inventory Exists | ✓/⚠/✗ | {details} | - -## Overall Result -{✓ PASSED / ⚠ PASSED WITH WARNINGS / ✗ FAILED} - -{If PASSED}: Template is ready for remediation playbook execution. -{If WARNINGS}: Template works but consider enabling ask_variables_on_launch and ask_limit_on_launch for flexibility. -{If FAILED}: Fix required checks before using with playbook-executor. See job-template-creator for setup guidance. If Ask Job Type on Launch fails: Enable "Prompt on Launch" for Job Type in AAP Web UI → Templates → [Template] → Edit → Options. -``` - -### Pass/Fail Determination - -- **PASSED**: All 6 required checks pass -- **PASSED WITH WARNINGS**: All required pass, one or more recommended fail -- **FAILED**: One or more required checks fail - -## Dependencies - -### Required MCP Servers -- `aap-mcp-job-management` - AAP job template and execution management -- `aap-mcp-inventory-management` - AAP inventory management - -### Required MCP Tools -- `job_templates_list` (from aap-mcp-job-management) - List templates -- `job_templates_retrieve` (from aap-mcp-job-management) - Get template details -- `projects_list` (from aap-mcp-job-management) - Verify project -- `inventories_list` (from aap-mcp-inventory-management) - Verify inventory - -### Related Skills -- `mcp-aap-validator` - **PREREQUISITE** - Validates AAP MCP before this skill -- `playbook-executor` - **PRIMARY USER** - Uses compatible templates for execution -- `job-template-creator` - Creates templates that this skill validates - -### Reference Documentation -- [playbook-executor/SKILL.md](../playbook-executor/SKILL.md) - Template compatibility requirements (Phase 1 Step 1.2, Scenario 3 validation) -- [job-template-creator/SKILL.md](../job-template-creator/SKILL.md) - Template configuration for remediation -- [AAP Job Templates](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates) - -## Example Usage - -### Example 1: Validate Template by ID - -**User Request**: "Validate job template 42 for remediation" - -**Skill Response**: - -1. Invoke mcp-aap-validator → PASSED -2. Call `job_templates_retrieve(id="42")` → Get template -3. Run validation checks -4. Call `projects_list`, `inventories_list` for context -5. Output report: - -``` -# Job Template Remediation Validation Report - -**Template**: CVE Remediation Template (ID: 42) -**Validated**: 2026-02-24 - -## Required Checks -| Requirement | Status | Details | -|-------------|--------|---------| -| Inventory | ✓ | 1 - Production Inventory | -| Project | ✓ | 6 - Remediation Playbooks | -| Playbook | ✓ | playbooks/remediation/remediation-template.yml | -| Credentials | ✓ | 1 credential(s) configured | -| Privilege Escalation | ✓ | become_enabled: true | - -## Recommended Checks -| Requirement | Status | Details | -|-------------|--------|---------| -| Ask Variables on Launch | ✓ | true | -| Ask Limit on Launch | ✓ | true | -| Ask Inventory on Launch | ✓ | true | - -## Overall Result -✓ PASSED - -Template is ready for remediation playbook execution. -``` - -### Example 2: Template Fails - Missing Ask Job Type on Launch - -**User Request**: "Validate template 20 for remediation" - -**Skill Response**: - -``` -# Job Template Remediation Validation Report - -**Template**: CVE Remediation (ID: 20) -**Validated**: 2026-02-24 - -## Required Checks -| Requirement | Status | Details | -|-------------|--------|---------| -| Inventory | ✓ | 1 - Production Inventory | -| Project | ✓ | 6 - Remediation Playbooks | -| Playbook | ✓ | playbooks/remediation/remediation-template.yml | -| Credentials | ✓ | 1 credential(s) configured | -| Privilege Escalation | ✓ | become_enabled: true | -| Ask Job Type on Launch | ✗ | ask_job_type_on_launch: false | - -## Overall Result -✗ FAILED - -Fix required: Enable "Prompt on Launch" for Job Type. -playbook-executor needs the same template for dry-run (job_type: "check") and actual execution (job_type: "run"). Without this, you would need separate templates for each mode. - -To fix: AAP Web UI → Templates → [Template] → Edit → Options → ✓ Prompt on Launch → Job Type -``` - -### Example 3: Template Fails - Missing Privilege Escalation - -**User Request**: "Check if template 15 works for remediation" - -**Skill Response**: - -``` -# Job Template Remediation Validation Report - -**Template**: General Playbook Runner (ID: 15) -**Validated**: 2026-02-24 - -## Required Checks -| Requirement | Status | Details | -|-------------|--------|---------| -| Inventory | ✓ | 1 - Production Inventory | -| Project | ✓ | 6 - Remediation Playbooks | -| Playbook | ✓ | playbooks/example.yml | -| Credentials | ✓ | 1 credential(s) configured | -| Privilege Escalation | ✗ | become_enabled: false | - -## Overall Result -✗ FAILED - -Fix required: Enable privilege escalation (become) on this template. -Remediation playbooks require sudo/root for package updates and system changes. - -To fix: AAP Web UI → Templates → [Template] → Edit → Options → ✓ Enable Privilege Escalation -``` - -### Example 4: Invoked by Playbook-Executor - -**Context**: playbook-executor filters templates and may invoke this skill to validate user-selected template before execution. - -**Workflow**: -``` -[playbook-executor] → User selects template ID 10 -[playbook-executor] → Invoke job-template-remediation-validator with template 10 -[job-template-remediation-validator] → Returns PASSED -[playbook-executor] → Proceeds with execution -``` - -## Critical: Human-in-the-Loop Requirements - -This skill performs **read-only validation** only. It does not modify AAP resources or execute playbooks. - -**When user input is needed**: -- **Template selection**: If multiple templates match a search, present the list and ask user to select by number or ID before proceeding -- **Template not found**: If template ID invalid, report error and ask user for correct ID or "list" to see available templates - -**No confirmation required** for validation execution - the skill only reads and reports. - -## Best Practices - -1. **Validate before execution** - Run this skill before playbook-executor when using a new or unfamiliar template -2. **Enable recommended options** - ask_variables_on_launch and ask_limit_on_launch improve flexibility -3. **Project sync** - Ensure project status is "successful" before execution -4. **Credential types** - Template should have Machine (SSH) credential; Vault optional for encrypted playbooks -5. **Naming convention** - Use descriptive names like "Remediate CVE-YYYY-NNNNN" for auditability diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/mcp-aap-validator/SKILL.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/mcp-aap-validator/SKILL.md deleted file mode 100644 index a1c4f708..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/mcp-aap-validator/SKILL.md +++ /dev/null @@ -1,66 +0,0 @@ ---- -name: mcp-aap-validator -description: | - Validate AAP (Ansible Automation Platform) MCP server connectivity. Use when the user asks to "validate AAP MCP", "check AAP connection", or when other skills need to verify AAP MCP availability before job management or inventory operations. -model: haiku -color: yellow ---- - -# MCP AAP Validator - -Validates connectivity to AAP MCP servers by running lightweight tool calls. - -## When to Use This Skill - -Use when validating AAP MCP before job template operations, troubleshooting connection issues, or when other skills (e.g. playbook-executor) need to verify availability. Do NOT use for creating templates—use job-template-creator. - -## Workflow - -1. **Test connectivity**: Call these tools to verify each server responds: - - `job_templates_list` (page_size: 10) from aap-mcp-job-management - - `inventories_list` (page_size: 10) from aap-mcp-inventory-management -2. **If any fails**: Provide a comprehensive message with possible root causes (see below). -3. **Report**: Output a table with validated servers and outcome (emojis). - -## Failure Message (Root Causes) - -When a tool call fails, include: - -``` -❌ AAP MCP connection failed - -**Possible root causes:** -- **Credentials**: AAP_MCP_SERVER or AAP_API_TOKEN not set or invalid -- **401 Unauthorized**: Token expired or invalid → regenerate in AAP Web UI -- **403 Forbidden**: Token lacks RBAC permissions (need Job Templates, Inventories) -- **404 Not Found**: Wrong AAP_MCP_SERVER URL (must point to MCP gateway, not main AAP UI) -- **Connection timeout**: Server unreachable, firewall, or network issue -- **SSL/TLS error**: Certificate verification problem - -**Troubleshooting:** -1. Verify env vars: AAP_MCP_SERVER, AAP_API_TOKEN (never echo values) -2. Get token: AAP Web UI → Users → [Your User] → Tokens → Create -3. Ensure AAP_MCP_SERVER points to MCP gateway endpoint -4. Restart host after config changes -``` - -## Report Format - -Always end with a table: - -| Server | Outcome | -|--------|---------| -| aap-mcp-job-management | ✅ PASSED | -| aap-mcp-inventory-management | ✅ PASSED | - -Use ✅ for success, ❌ for failure, ⚠️ for partial (e.g. one server OK, one failed). - -## Dependencies - -### Required MCP Tools -- `job_templates_list` (from aap-mcp-job-management) - Connectivity test -- `inventories_list` (from aap-mcp-inventory-management) - Connectivity test - -### Required MCP Servers -- `aap-mcp-job-management` - AAP job template and execution -- `aap-mcp-inventory-management` - AAP inventory management diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/SKILL.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/SKILL.md deleted file mode 100644 index a29c9443..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/SKILL.md +++ /dev/null @@ -1,499 +0,0 @@ ---- -name: playbook-executor -description: | - **CRITICAL**: Use for Ansible playbook execution via AAP. DO NOT call AAP MCP tools directly. - - Execute remediation playbooks with job management, dry-run, and reporting. Use after playbook-generator. - - **Git Flow**: If template playbook path ≠ generated playbook, perform Git Flow (commit, push, sync) BEFORE launch. ---- - -# AAP Playbook Executor Skill - -This skill executes Ansible remediation playbooks through AAP (Ansible Automation Platform) with full job management capabilities. - -**Integration with Remediation Skill**: The `/remediation` skill orchestrates this skill as part of its Step 5 (Execute Playbook) workflow. For standalone playbook execution, you can invoke this skill directly. - -## Prerequisites - -**Required MCP Servers**: `aap-mcp-job-management`, `aap-mcp-inventory-management` ([setup guide](https://docs.redhat.com/)) - -**Required MCP Tools**: -- `job_templates_list` (from aap-mcp-job-management) - List job templates -- `job_templates_retrieve` (from aap-mcp-job-management) - Get template details -- `projects_list` (from aap-mcp-job-management) - Get project name and scm_url for Git Flow -- `job_templates_launch_retrieve` (from aap-mcp-job-management) - Launch jobs -- `jobs_retrieve` (from aap-mcp-job-management) - Get job status -- `jobs_stdout_retrieve` (from aap-mcp-job-management) - Get console output -- `jobs_job_events_list` (from aap-mcp-job-management) - Get task events -- `jobs_job_host_summaries_list` (from aap-mcp-job-management) - Get host statistics -- `inventories_list` (from aap-mcp-inventory-management) - List inventories -- `hosts_list` (from aap-mcp-inventory-management) - List inventory hosts - -**Required Environment Variables**: -- `AAP_MCP_SERVER` - Base URL for the MCP endpoint of the AAP server (must point to the AAP MCP gateway) -- `AAP_API_TOKEN` - AAP API authentication token - -### Prerequisite Validation - -**CRITICAL**: Before executing operations, execute the `/mcp-aap-validator` skill to verify AAP MCP server availability. - -**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-aap-validator/SKILL.md#validation-freshness-policy). - -**How to invoke**: Execute the `/mcp-aap-validator` skill - -**Handle validation result**: -- **If validation PASSED**: Continue with playbook execution workflow -- **If validation PARTIAL**: Warn user and ask to proceed -- **If validation FAILED**: Stop execution, provide setup instructions from validator - -**Human Notification on Failure**: -If prerequisites are not met: -- ❌ "Cannot proceed: AAP MCP servers are not available" -- 📋 "Setup required: Configure AAP_MCP_SERVER and AAP_API_TOKEN environment variables" -- ❓ "How would you like to proceed? (setup now / skip / abort)" -- ⏸️ Wait for user decision - -## When to Use This Skill - -**Use this skill directly when you need**: -- Execute a previously generated Ansible playbook via AAP -- Track the status of a running AAP job -- Monitor playbook job completion -- Run dry-run (check mode) before production execution -- Verify playbook execution succeeded - -**Use the `/remediation` skill when you need**: -- Full remediation workflow including playbook execution -- Integrated CVE analysis → playbook generation → execution → verification -- End-to-end remediation orchestration - -**How they work together**: The `/remediation` skill invokes this skill after generating a remediation playbook, managing the full workflow from analysis to verification. - -## Workflow - -**Git Flow is MANDATORY**: When the job template's playbook path differs from the generated playbook (or content must be updated), you MUST perform Git Flow (write, commit, push, sync) and receive "sync complete" from the user BEFORE launching any job. Do NOT skip this—launching without it executes the wrong playbook. - -### Phase 0: Validate AAP MCP Prerequisites - -**Action**: Execute the `/mcp-aap-validator` skill - -**Note**: Can skip if validation was performed earlier in this session and succeeded. - -**How to invoke**: Execute the `/mcp-aap-validator` skill - -**Handle validation result**: -- **If validation PASSED**: Continue to Phase 1 -- **If validation PARTIAL**: Warn user and ask to proceed -- **If validation FAILED**: Stop execution, user must set up AAP MCP servers - -### Phase 1: Job Template Selection and Playbook Preparation - -**Goal**: Identify an AAP job template suitable for executing the remediation playbook. **Git Flow is MANDATORY** before Phase 3 when the template points to a different playbook or when content must be updated. - -**Input**: Playbook content and metadata from playbook-generator (filename, CVE ID, target systems). The playbook YAML is already in context—do NOT regenerate it during Git Flow. Playbook path is derived from metadata: `playbooks/remediation/` (e.g., `playbooks/remediation/remediation-CVE-2025-49794.yml`). - -**BLOCKING**: You MUST NOT launch any job (dry-run or production) until the playbook is in the Git repo and the user has confirmed "sync complete". AAP executes from the synced project—there is no "override at launch". Launching without Git Flow executes the WRONG playbook. - -#### Step 1.1: Derive Playbook Path - -From playbook metadata (filename from playbook-generator): -- Use convention `playbooks/remediation/` -- Support both `remediation-CVE-*.yml` and `remediation-CVE-*-playbook.yml` patterns. -- Example: CVE-2026-26103 → `playbooks/remediation/remediation-CVE-2026-26103.yml` - -#### Step 1.2: List Templates and Validate Each Candidate - -**MCP Tool**: `job_templates_list` (from aap-mcp-job-management) - -**Parameters**: -- `page_size`: 50 (retrieve up to 50 templates) -- `search`: "" (search for all templates) - -**REQUIRED**: For each template in results: -1. Call `job_templates_retrieve(id)` to get full details -2. **Invoke the `/job-template-remediation-validator` skill** with the template ID to verify it meets remediation requirements (inventory, project, playbook, credentials, become_enabled) -3. Only include templates that PASS validation in the lists below - -Build two lists: -- **exact_match**: `template.playbook` equals `our_playbook_path` (normalize slashes; match if equal or basenames match) -- **compatible_other**: Passes job-template-remediation-validator but **different playbook path** (template points to e.g. `cve-remediation.yml` while we have `remediation-CVE-2026-26103.yml`) - -**Path normalization**: Normalize slashes, handle `playbooks/remediation/` prefix. Match if `template.playbook` equals `our_playbook_path` or if basenames match. **Different filenames = different path = Scenario 2.** - -#### Step 1.3: Scenario Selection (MANDATORY - Do Not Skip) - -**Scenario 1 - Same playbook path** (exact_match not empty): - -The template already points to our playbook path. The project may need the latest content. **Read [references/05-git-flow-prompts.md](references/05-git-flow-prompts.md)** for Scenario 1 prompt, options (A/B), and Git Flow steps. - -- **If A**: Execute Git Flow (see Git Flow section below). **BLOCK Phase 3** until user confirms "sync complete" or "done". -- **If B**: Wait for user confirmation. **BLOCK Phase 3** until user confirms. - -**Scenario 2 - Different playbook path** (compatible_other not empty, exact_match empty): - -**CRITICAL**: The template points to a DIFFERENT playbook than our generated playbook. You MUST NOT launch the job without Git Flow—AAP executes from synced content; there is no override at launch. **Read [references/05-git-flow-prompts.md](references/05-git-flow-prompts.md)** for Scenario 2 prompt and Git Flow steps. - -- **If yes**: Execute Git Flow. **BLOCK Phase 3** until Git Flow completes and user confirms "sync complete". -- **If no**: Fall through to Scenario 3. - -**Anti-pattern**: Do NOT say "I'll override with our playbook" and then launch—that is impossible. The playbook MUST be in the repo before launch. - -**Scenario 3 - No suitable template** (exact_match and compatible_other both empty, or user chose "no" in Scenario 2): - -Execute the `/job-template-creator` skill with instruction: -``` -"Create a job template for this remediation playbook. Playbook: [content]. Filename: [filename]. Path: [our_playbook_path]. CVE: [cve_id]. Target systems: [list]." -``` - -The job-template-creator skill guides the user through: (1) Adding playbook to Git repository, (2) Syncing AAP project, (3) Creating job template via AAP Web UI with correct path, inventory, credentials, privilege escalation. - -After `/job-template-creator` completes, retrieve the template ID (from skill output or user confirmation). Execute `/job-template-remediation-validator` to validate the newly created template. If passed, proceed to Phase 3 (Dry-Run). If failed, report issues and ask user to fix in AAP Web UI. - -**Multiple matches**: If multiple exact matches, present list and ask user to choose by number. If multiple different-path matches, prefer by project name containing "remediation" or "CVE", else first. - -**Phase 1 Checkpoint** (BLOCKING - must pass before Phase 3): -- **Git Flow required**: If Scenario 1 or 2, you MUST complete Git Flow and receive "sync complete" from the user before proceeding. Do NOT skip. -- **No override**: There is no way to "override" the playbook at launch. AAP runs whatever is in the synced project. -- **Never launch** if the playbook has not been committed, pushed, and synced - -#### Git Flow (for Scenario 1 Override and Scenario 2) - MANDATORY HITL - -**When**: Scenario 1 (same path, update content) or Scenario 2 (different path, replace playbook). **Do not skip**—execution with wrong playbook content will remediate the wrong CVE. - -**Target path**: -- Scenario 1: `our_playbook_path` (e.g. `playbooks/remediation/remediation-CVE-2026-26103.yml`) -- Scenario 2: `template.playbook` (e.g. `playbooks/remediation/cve-remediation.yml`)—we replace the template's playbook with our generated content - -**Prerequisite**: Ask user for the local path to the Git repository. Use `projects_list` for project name and `scm_url`. **Read [references/05-git-flow-prompts.md](references/05-git-flow-prompts.md)** for repo path question, HITL checkpoint text, and after-push message. - -**Steps** (execute in order; HITL at checkpoint): -1. **Write playbook to file** (FAST—do NOT regenerate): - - The playbook content is ALREADY in context from playbook-generator (or remediation skill). Use it directly. - - **⚠️ ABSOLUTE PATH REQUIRED**: The Write path MUST start with `/`. Use: `/`. Example: `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project/playbooks/remediation/cve-remediation.yml` - - **WRONG** (causes "Error writing file"): `test-aap-project/playbooks/...` or `playbooks/remediation/...` — these are relative and fail when repo is outside workspace. - - **Before Write**: Confirm path starts with `/`. If not, prepend the user's repo path. - - Do NOT invoke playbook-generator, do NOT call MCP tools, do NOT re-fetch. This should take seconds, not minutes. -2. Use Run tool: `git add ` (from repo root, e.g. `git add playbooks/remediation/cve-remediation.yml`) -3. **HITL Checkpoint** (REQUIRED): Display summary per reference file. Wait for "yes" or "proceed" -4. If confirmed: `git commit -m "Add/update remediation playbook for CVE-YYYY-NNNNN"` -5. `git push origin main` (or branch from project's scm_branch if available) - -**Note**: Git must be configured. Use Run tool for git commands. - -**Do NOT proceed to Phase 3 (Dry-Run) until user confirms sync complete.** - -### Phase 2: Git Flow (MANDATORY before Phase 3) - -**BLOCKING**: You MUST NOT proceed to Phase 3 (Dry-Run) until Git Flow is complete. - -**When**: Scenario 1 (same path, update content) or Scenario 2 (different path). See Phase 1 Step 1.3. - -**Checkpoint**: Before Phase 3, confirm: -- [ ] Playbook written to repo at target path -- [ ] Git commit and push completed (with user confirmation) -- [ ] User confirmed "sync complete" after AAP project sync - -**If any unchecked**: STOP. Do Git Flow. Do NOT launch the job. - -### Phase 3: Dry-Run Execution (Recommended) - -**Prerequisite**: Phase 2 (Git Flow) MUST be complete. User must have confirmed "sync complete". - -**Goal**: Test playbook in check mode before actual execution to simulate changes. - -**Read [references/04-dry-run-display-templates.md](references/04-dry-run-display-templates.md)** for: Playbook Preview, Dry-Run Offer, Dry-Run Results Display, Proceed prompt. - -#### Step 3.1–3.2: Display Preview and Offer Dry-Run - -Show playbook structure per reference. Offer dry-run with options: yes / no / abort. **ONLY if user confirms**, proceed. - -#### Step 3.3: Launch Dry-Run Job - -**Pre-launch check** (BLOCKING): If Scenario 1 or 2 applied, you MUST have completed Git Flow and received "sync complete" from the user. If not, STOP—do not launch. Return to Phase 2 / Git Flow. - -**MCP Tool**: `job_templates_launch_retrieve` (from aap-mcp-job-management) - -**Parameters**: `id`, `requestBody` with `job_type: "check"`, `extra_vars`, `limit` - -**Key**: `job_type: "check"` - Runs Ansible in check mode (dry-run) - -#### Step 3.4: Monitor Dry-Run Progress - -Poll `jobs_retrieve` every 2 seconds. Use `jobs_job_events_list` for live task updates. - -#### Step 3.5: Display Dry-Run Results - -**MCP Tools**: `jobs_stdout_retrieve` (id, format: "txt"), `jobs_job_host_summaries_list` (id). Use display format from reference. - -#### Step 3.6: Proceed to Actual Execution? - -Ask per reference. Wait for "yes" or "execute". - -### Phase 4: Actual Execution - -**ONLY execute if user explicitly confirms** (either after dry-run or directly if they skipped dry-run). - -#### Step 4.1: Final Confirmation - -``` -⚠️ CRITICAL: Playbook Execution Confirmation Required - -This playbook will: -- Execute on: 3 production systems -- Update packages: httpd (2.4.53-7.el9 → 2.4.57-8.el9) -- Restart services: httpd -- Estimated downtime: ~10 seconds per system -- Requires reboot: No - -Job Template: CVE Remediation Template (ID: 10) -AAP URL: https://aap.example.com/jobs/ - -❓ Execute this playbook now? - -Options: -- "yes" or "execute" - Proceed with execution -- "abort" - Cancel execution - -Please respond with your choice. -``` - -Wait for explicit "yes" or "execute" response. - -#### Step 4.2: Launch Production Job - -**Pre-launch check** (BLOCKING): Same as Phase 3—if Scenario 1 or 2 applied, Git Flow must be complete and user must have confirmed "sync complete". Do NOT launch without it. - -**MCP Tool**: `job_templates_launch_retrieve` (from aap-mcp-job-management) - -**Parameters**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run", - "extra_vars": { - "target_cve": "CVE-2025-49794", - "remediation_mode": "automated", - "verify_after": true - }, - "limit": "prod-web-01,prod-web-02,prod-web-03" - } -} -``` - -**Key Parameter**: `job_type: "run"` - Runs Ansible in execution mode (actual changes) - -**Expected Output**: -```json -{ - "job": 1235, - "status": "pending", - "url": "/api/controller/v2/jobs/1235/" -} -``` - -#### Step 4.3: Monitor Execution Progress - -**Polling Strategy**: -1. Call `jobs_retrieve(id=job_id)` every 2 seconds -2. Get task events with `jobs_job_events_list(id=job_id)` for progress updates -3. Display real-time task completion status -4. Continue until status is "successful", "failed", or "error" - -**Progress Display**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running -Elapsed: 1m 23s -AAP URL: https://aap.example.com/#/jobs/playbook/1235 - -Recent Events: -- ✓ Gathering Facts (completed - all hosts) -- ✓ Check Disk Space (completed - all hosts) -- ✓ Backup Configuration (completed - all hosts) -- ⏳ Update Package: httpd (running - prod-web-01, prod-web-02) - └─ prod-web-01: Installing httpd-2.4.57-8.el9... - └─ prod-web-02: Installing httpd-2.4.57-8.el9... -- ⏸ Restart Service: httpd (pending) -``` - -**Update every 2 seconds** until completion. - -### Phase 5: Execution Report - -**Goal**: Generate comprehensive report with job details, per-host results, and full output. - -**Read [references/01-execution-report-templates.md](references/01-execution-report-templates.md)** for JSON examples, comprehensive report template, and Success/Partial Success/Failure output templates. - -#### Step 5.1–5.4: Gather Data - -**MCP Tools** (all from aap-mcp-job-management): -- `jobs_retrieve` (id) - Job details -- `jobs_job_host_summaries_list` (id) - Per-host stats -- `jobs_job_events_list` (id) - Task timeline -- `jobs_stdout_retrieve` (id, format: "txt") - Full console output - -#### Step 5.5: Generate Report - -Format all gathered data per reference. Use Success / Partial Success / Failure template based on job status. - -#### Step 5.6: Validate Job Log for CVE Handling (MANDATORY) - -**Goal**: Confirm from the job stdout that the playbook actually addressed the target CVE(s). - -**Input**: Target CVE ID(s) from invocation (e.g. CVE-2025-49794). Job stdout from `jobs_stdout_retrieve` (already gathered in Step 5.4). - -**Parse stdout for**: -- Target CVE ID(s) in output (vars, task names, audit logs, playbook metadata) -- Package update tasks for affected packages (dnf/yum install/update, package module) -- Remediation-related task names (e.g. "Update package", "Restart service", "remediation") - -**Report** (add to execution report): -- **✓ Job log confirms CVE-XXXX-YYYY was addressed** — CVE ID or package updates found in stdout -- **⚠️ Job log did not show clear evidence of CVE handling** — No CVE ID or package updates found; recommend manual verification or `/remediation-verifier` - -**Batch**: For multiple CVEs, validate each. Report per-CVE confirmation or warning. - -### Phase 6: Error Handling - -**If job status is "failed" or "error"**, provide detailed troubleshooting. - -**Read [references/02-error-handling-guide.md](references/02-error-handling-guide.md)** for: Error categories, error report template, troubleshooting steps, relaunch parameters. - -#### Step 6.1: Parse Error Output - -**MCP Tool**: `jobs_stdout_retrieve`. Analyze output for error categories per reference. - -#### Step 6.2: Generate Error Report - -Use error report template from reference. Include per-host results, failed task details, troubleshooting steps, relaunch options. - -#### Step 6.3: Offer Relaunch - -If user chooses relaunch: **MCP Tool** `jobs_relaunch_retrieve` with `hosts: "failed"`, `job_type: "run"` per reference. - -## Reference Files - -| File | Use When | -|------|----------| -| [01-execution-report-templates.md](references/01-execution-report-templates.md) | Phase 5 reports, Success/Partial/Failure output | -| [02-error-handling-guide.md](references/02-error-handling-guide.md) | Phase 6 error reports, relaunch | -| [03-workflow-examples.md](references/03-workflow-examples.md) | Demo full workflow, failure handling, skip dry-run | -| [04-dry-run-display-templates.md](references/04-dry-run-display-templates.md) | Phase 3 preview, offer, results, proceed prompt | -| [05-git-flow-prompts.md](references/05-git-flow-prompts.md) | Scenario 1/2 prompts, Git Flow HITL, after-push | - -## Dependencies - -### Required MCP Servers -- `aap-mcp-job-management` - AAP job management and execution -- `aap-mcp-inventory-management` - AAP inventory management - -### Required MCP Tools -- `job_templates_list` (from aap-mcp-job-management) - List templates -- `job_templates_retrieve` (from aap-mcp-job-management) - Get template details -- `projects_list` (from aap-mcp-job-management) - Get project name and scm_url for Git Flow -- `job_templates_launch_retrieve` (from aap-mcp-job-management) - Launch jobs -- `jobs_retrieve` (from aap-mcp-job-management) - Get job status -- `jobs_stdout_retrieve` (from aap-mcp-job-management) - Get console output -- `jobs_job_events_list` (from aap-mcp-job-management) - Get task events -- `jobs_job_host_summaries_list` (from aap-mcp-job-management) - Get host statistics -- `inventories_list` (from aap-mcp-inventory-management) - List inventories -- `hosts_list` (from aap-mcp-inventory-management) - List hosts - -### Related Skills -- `mcp-aap-validator` - **PREREQUISITE** - Validates AAP MCP servers (invoke in Phase 0) -- `job-template-remediation-validator` - **REQUIRED** - Invoke for each candidate template in Phase 1 Step 1.2 to verify remediation requirements -- `job-template-creator` - Creates/guides AAP job template setup -- `playbook-generator` - Generates playbooks for execution -- `remediation-verifier` - Verifies success after execution - -### Reference Documentation -- [references/](references/) - Step-numbered reference files (01–05) for templates and examples -- [AAP Job Execution Guide](../../docs/ansible/aap-job-execution.md) - AAP job execution best practices -- [Playbook Integration with AAP](../../docs/ansible/playbook-integration-aap.md) - Playbook-to-AAP workflow - -## Critical: Human-in-the-Loop Requirements - -This skill executes code on production systems. **Explicit user confirmation is REQUIRED** at multiple stages. - -**Before Git commit/push** (Scenario 1 Override, Scenario 2): -1. **Display change summary**: File path, diff or file size -2. **Ask for confirmation**: "Ready to commit and push these changes? Reply 'yes' or 'proceed' to continue, or 'abort' to cancel." -3. **Wait for explicit "yes" or "proceed"**: Do not commit/push without confirmation - -**Before Dry-Run Execution** (if user chooses dry-run): -1. **Display Playbook Preview**: Show tasks and explain changes -2. **Ask for Dry-Run Confirmation**: - ``` - ❓ Run dry-run to simulate changes? - - Options: - - "yes" - Run dry-run (recommended) - - "no" - Skip to actual execution - - "abort" - Cancel - - Please respond with your choice. - ``` -3. **Wait for Explicit Response**: Do not proceed without confirmation - -**Before Actual Execution** (REQUIRED): -1. **Display Execution Summary**: Show systems, changes, downtime estimate -2. **Ask for Final Confirmation**: - ``` - ⚠️ CRITICAL: Execute playbook on production systems? - - This will make real changes to N systems. - - Options: - - "yes" or "execute" - Proceed - - "abort" - Cancel - - Please respond with your choice. - ``` -3. **Wait for Explicit "yes" or "execute"**: Do not proceed without confirmation - -**Never assume approval** - always wait for explicit user confirmation before executing playbooks. - -## Best Practices - -1. **Write path must be absolute** - When Git Flow writes the playbook to the user's repo, use `/playbooks/remediation/`. The path MUST start with `/`. Relative paths cause "Error writing file". -2. **Always validate AAP prerequisites** - Invoke mcp-aap-validator in Phase 0 -3. **Validate each template** - Invoke job-template-remediation-validator for each candidate before selection -4. **Never skip Git Flow** - If template playbook path ≠ generated playbook path (Scenario 2) or content must be updated (Scenario 1), you MUST complete Git Flow and receive "sync complete" before Phase 3. Do NOT launch without it. -5. **Recommend dry-run** - Offer check mode before production execution -6. **Filter compatible templates** - Check inventory, project, and credentials match -7. **Monitor in real-time** - Display task progress during execution -8. **Comprehensive reporting** - Include per-host stats, task timeline, full output -9. **Error categorization** - Parse errors and provide specific troubleshooting -10. **Relaunch capability** - Offer to retry failed hosts -11. **Link to AAP** - Provide direct URL to job in AAP Web UI -12. **Suggest verification** - Always recommend remediation-verifier after success -13. **Document job details** - Save job ID and template info for audit trail - -## Integration with Other Skills - -- **playbook-generator**: Generates playbooks that this skill executes -- **job-template-creator**: Creates AAP job templates when needed -- **remediation-verifier**: Verifies success after this skill completes execution -- **`/remediation` skill**: Orchestrates full workflow including playbook execution - -**Orchestration Example** (from `/remediation` skill): -1. Agent invokes playbook-generator skill → Creates playbook YAML -2. playbook-generator asks for confirmation → User approves playbook content -3. Agent invokes playbook-executor skill (this skill) → Execution workflow -4. Skill validates templates via job-template-remediation-validator → Filters valid candidates -5. Skill checks path match → If different path, offers Git Flow (HITL: commit/push, sync AAP) -6. Skill waits for "sync complete" before proceeding (if Git Flow was used) -7. Skill offers dry-run → User runs check mode -8. Skill asks for execution confirmation → User approves -9. Skill executes and monitors → Reports completion -10. Agent invokes remediation-verifier skill → Confirms CVE resolved - -**Note**: Both playbook-generator and playbook-executor require separate confirmations for different purposes: -- playbook-generator: Confirms playbook content is acceptable -- playbook-executor: Confirms execution on production systems is approved - -This two-step approval ensures user control over both what to run and when to run it. diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/01-execution-report-templates.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/01-execution-report-templates.md deleted file mode 100644 index a6773c5f..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/01-execution-report-templates.md +++ /dev/null @@ -1,168 +0,0 @@ -# Step 01: Execution Report Templates - -Read this reference when generating Phase 5 execution reports or output templates. - -## Phase 5: Job Details (JSON Examples) - -### jobs_retrieve Expected Output - -```json -{ - "id": 1235, - "name": "CVE Remediation Template", - "status": "successful", - "started": "2026-02-24T15:35:02Z", - "finished": "2026-02-24T15:40:25Z", - "elapsed": 323.45, - "job_template": 10, - "inventory": 1, - "limit": "prod-web-01,prod-web-02,prod-web-03", - "playbook": "playbooks/remediation/remediation-CVE-2025-49794.yml" -} -``` - -### jobs_job_host_summaries_list Expected Output - -```json -{ - "results": [ - { - "host_name": "prod-web-01", - "ok": 8, - "changed": 3, - "failed": 0, - "unreachable": 0 - }, - { - "host_name": "prod-web-02", - "ok": 8, - "changed": 3, - "failed": 0, - "unreachable": 0 - }, - { - "host_name": "prod-web-03", - "ok": 5, - "changed": 0, - "failed": 1, - "unreachable": 0 - } - ] -} -``` - -## Comprehensive Report Template - -```markdown -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 5m 23s -**Started**: 2026-02-24 15:35:02 UTC -**Completed**: 2026-02-24 15:40:25 UTC -**Job Template**: CVE Remediation Template -**Playbook**: playbooks/remediation/remediation-CVE-2025-49794.yml -**AAP URL**: [View in AAP](https://aap.example.com/#/jobs/playbook/1235) - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| prod-web-01 | 8 | 3 | 0 | 0 | ✅ Success | -| prod-web-02 | 8 | 3 | 0 | 0 | ✅ Success | -| prod-web-03 | 8 | 3 | 0 | 0 | ✅ Success | - -**Summary**: 3 of 3 hosts successfully remediated - -## Task Timeline -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) - - prod-web-01: 2.4.53-7.el9 → 2.4.57-8.el9 - - prod-web-02: 2.4.53-7.el9 → 2.4.57-8.el9 - - prod-web-03: 2.4.53-7.el9 → 2.4.57-8.el9 -5. ✅ Restart httpd service (15s) -6. ✅ Verify service status (2s) -7. ✅ Update audit log (1s) - -## Full Console Output -
-Click to expand (187 lines) - -[Full stdout from jobs_stdout_retrieve] - -
- -## Job Log CVE Validation (Step 5.6) -✓ Job log confirms CVE-XXXX-YYYY was addressed - -*(Or: ⚠️ Job log did not show clear evidence of CVE handling—verify manually or use remediation-verifier)* - -## Next Steps -1. ✅ All systems successfully remediated -2. ☐ Verify remediation with remediation-verifier skill -3. ☐ Update vulnerability tracking system -4. ☐ Schedule follow-up verification in 24-48 hours - ---- - -**Recommendation**: Run remediation-verifier skill to confirm CVE status has been updated in Red Hat Lightspeed. -``` - -## Output Templates - -### Success Template - -```markdown -✅ Playbook Execution Successful - -Job ID: 1235 -Duration: 5m 23s -Systems Remediated: 3 of 3 - -View full report above for details. - -Next Steps: -- Run remediation-verifier skill to confirm CVE resolution -- Update vulnerability tracking system -- Monitor systems for 24-48 hours - -AAP URL: https://aap.example.com/#/jobs/playbook/1235 -``` - -### Partial Success Template - -```markdown -⚠️ Playbook Execution Completed with Failures - -Job ID: 1235 -Duration: 2m 45s -Systems Remediated: 2 of 3 -Failed Systems: prod-web-03 - -See error details above for troubleshooting steps. - -Options: -- Relaunch for failed hosts -- Manual remediation -- Skip failed hosts - -AAP URL: https://aap.example.com/#/jobs/playbook/1235 -``` - -### Failure Template - -```markdown -❌ Playbook Execution Failed - -Job ID: 1235 -Duration: 1m 15s -Systems Remediated: 0 of 3 - -Critical errors prevented execution. -See error details above for troubleshooting. - -AAP URL: https://aap.example.com/#/jobs/playbook/1235 -``` diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/02-error-handling-guide.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/02-error-handling-guide.md deleted file mode 100644 index 90492f00..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/02-error-handling-guide.md +++ /dev/null @@ -1,108 +0,0 @@ -# Step 02: Error Handling Guide - -Read this reference when generating Phase 6 error reports or troubleshooting. - -## Error Categories - -**Parse error output** from `jobs_stdout_retrieve` for these common patterns: - -1. **Connection Failures**: SSH timeout, host unreachable, authentication failed -2. **Permission Errors**: sudo required, insufficient privileges, SELinux denials -3. **Package Manager Issues**: repo unavailable, package not found, dependency conflicts -4. **Service Failures**: service not found, restart failed, timeout -5. **Disk Space**: insufficient space for updates -6. **General Failures**: playbook syntax errors, task failures - -## Error Report Template - -```markdown -# Playbook Execution Failed - -## Job Summary -**Job ID**: 1235 -**Status**: ❌ Failed -**Duration**: 2m 45s -**Started**: 2026-02-24 15:35:02 UTC -**Failed At**: 2026-02-24 15:37:47 UTC -**Job Template**: CVE Remediation Template -**AAP URL**: [View in AAP](https://aap.example.com/#/jobs/playbook/1235) - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| prod-web-01 | 8 | 3 | 0 | 0 | ✅ Success | -| prod-web-02 | 8 | 3 | 0 | 0 | ✅ Success | -| prod-web-03 | 5 | 0 | 1 | 0 | ❌ Failed | - -**Summary**: 2 of 3 hosts succeeded, 1 failed - -## Failed Tasks Details - -### Host: prod-web-03 - -**Task**: Restart httpd service -**Error**: "Failed to restart httpd.service: Unit httpd.service not found." - -**Error Category**: Service Failure - -**Root Cause**: The httpd service is not installed or not recognized by systemd. - -**Troubleshooting Steps**: -1. Check if httpd is installed: - ```bash - ssh prod-web-03 'rpm -q httpd' - ``` -2. If not installed, the package update may have failed: - ```bash - ssh prod-web-03 'dnf info httpd' - ``` -3. Check systemd service status: - ```bash - ssh prod-web-03 'systemctl status httpd' - ``` -4. Review package manager logs: - ```bash - ssh prod-web-03 'tail -50 /var/log/dnf.log' - ``` - -**Recommended Action**: -- Verify httpd package installation on prod-web-03 -- Check if package update completed successfully -- Manually install httpd if needed: `dnf install httpd` -- Relaunch job for failed host only - -## Console Output (Last 50 Lines) -
-Click to expand error context - -[Relevant error output from jobs_stdout_retrieve] - -
- -## Relaunch Options - -Would you like to: -1. **Relaunch for failed hosts only** - Run job again with limit="prod-web-03" -2. **Fix issues manually and relaunch** - Resolve problems first, then relaunch -3. **View full job output** - See complete execution logs -4. **Abort** - Stop remediation workflow - -Please choose an option (1-4): -``` - -## Relaunch Parameters - -**MCP Tool**: `jobs_relaunch_retrieve` (from aap-mcp-job-management) - -**Parameters**: -```json -{ - "id": "1235", - "requestBody": { - "hosts": "failed", - "job_type": "run" - } -} -``` - -This relaunches the job for only the failed hosts. diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/03-workflow-examples.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/03-workflow-examples.md deleted file mode 100644 index f5caa0d1..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/03-workflow-examples.md +++ /dev/null @@ -1,119 +0,0 @@ -# Step 03: Workflow Examples - -Read this reference when demonstrating end-to-end workflow patterns. - -## Example 1: Full Workflow with Dry-Run - -**User Request**: "Execute the CVE-2025-49794 remediation playbook" - -**Skill Response**: - -1. **Validate AAP Prerequisites**: - - Invoke mcp-aap-validator skill → PASSED - -2. **List Job Templates**: - - Call `job_templates_list()` → Found 2 templates - - Filter compatible templates → 1 matches requirements - -3. **User Selects Template**: - ``` - Found 1 compatible job template: - 1. "CVE Remediation Template" (ID: 10) - - Select template (1) or "create" for new: 1 - ``` - -4. **Playbook Preparation**: - ``` - Guide user to add playbook to Git: - - Commands provided - - User syncs AAP project - - Verification: Playbook available ✓ - ``` - -5. **Offer Dry-Run**: - ``` - Run dry-run first? yes - ``` - -6. **Execute Dry-Run**: - - Launch with `job_type="check"` - - Monitor progress → COMPLETED - - Display dry-run results: - ``` - Would change 3 tasks on 3 hosts - No errors detected - ``` - -7. **Proceed to Actual Execution**: - ``` - Proceed with actual execution? yes - ``` - -8. **Execute Playbook**: - - Launch with `job_type="run"` - - Monitor progress → COMPLETED - - Generate report: - ``` - ✅ 3 of 3 hosts successfully remediated - Duration: 5m 23s - ``` - -9. **Suggest Verification**: - ``` - Recommend: Run remediation-verifier skill to confirm CVE resolution - ``` - -## Example 2: Handle Execution Failure - -**User Request**: "Execute remediation playbook" - -**Skill Response**: - -1-7. [Same as Example 1 through execution] - -8. **Execute Playbook**: - - Launch job → FAILED on 1 of 3 hosts - - Generate error report: - ``` - ❌ prod-web-03 failed - Error: httpd service not found - Troubleshooting steps provided - ``` - -9. **Offer Relaunch**: - ``` - Relaunch for failed host only? yes - ``` - -10. **Relaunch Job**: - - Call `jobs_relaunch_retrieve` with hosts="failed" - - Monitor → COMPLETED - - Final report: - ``` - ✅ All 3 hosts successfully remediated (1 after retry) - ``` - -## Example 3: Skip Dry-Run - -**User Request**: "Execute playbook directly, skip dry-run" - -**Skill Response**: - -1-4. [Same as Example 1 through template selection] - -5. **Offer Dry-Run**: - ``` - Run dry-run first? no - ``` - -6. **Final Confirmation**: - ``` - ⚠️ Execute on production without dry-run? - This will make changes immediately. - Confirm: yes - ``` - -7. **Execute Playbook**: - - Launch with `job_type="run"` - - Monitor and report as in Example 1 diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/04-dry-run-display-templates.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/04-dry-run-display-templates.md deleted file mode 100644 index d9b6f0dc..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/04-dry-run-display-templates.md +++ /dev/null @@ -1,93 +0,0 @@ -# Step 04: Dry-Run Display Templates - -Read this reference when displaying Phase 3 dry-run content. - -## Playbook Preview - -```markdown -# Playbook Preview - -**Playbook**: remediation-CVE-2025-49794.yml -**Target Systems**: 5 systems - -## Tasks Overview: -1. **Gather Facts** - Collect system information -2. **Check Disk Space** - Ensure sufficient space for updates (>500MB) -3. **Backup Configuration** - Snapshot critical configs -4. **Update Package: httpd** - Upgrade to version 2.4.57-8.el9 -5. **Restart Service: httpd** - Apply changes -6. **Verify Service Status** - Confirm httpd is running -7. **Update Audit Log** - Record remediation event - -**Estimated Duration**: 3-5 minutes per system -**Requires Reboot**: No -**Downtime**: Brief (~10 seconds during service restart) -``` - -## Dry-Run Offer - -``` -⚠️ Recommended: Run dry-run first - -Dry-run mode (--check) simulates changes without applying them. -This helps identify: -- Package availability issues -- Permission problems -- Configuration conflicts -- Unexpected side effects - -❓ Run dry-run before actual execution? -- "yes" - Run dry-run first (recommended) -- "no" - Skip to actual execution -- "abort" - Cancel execution - -Please respond with your choice. -``` - -## Dry-Run Results Display - -```markdown -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s -**Completed**: 2024-01-20 15:32:17 UTC - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| prod-web-01 | 3 | 8 | 0 | ✓ Ready | -| prod-web-02 | 3 | 8 | 0 | ✓ Ready | -| prod-web-03 | 3 | 8 | 0 | ✓ Ready | - -## Changes That Would Be Made: -1. **httpd package** - Would update from 2.4.53-7.el9 to 2.4.57-8.el9 -2. **httpd service** - Would restart -3. **audit log** - Would add remediation entry - -## Dry-Run Output: -
-Click to expand full output - -[Full stdout from jobs_stdout_retrieve] - -
- -✓ No errors detected in dry-run -✓ All systems passed pre-flight checks -``` - -## Proceed to Actual Execution Prompt - -``` -❓ Dry-run completed successfully. Proceed with actual execution? - -Options: -- "yes" or "execute" - Proceed with actual remediation -- "review" - Show dry-run output again -- "abort" - Cancel execution - -Please respond with your choice. -``` diff --git a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/05-git-flow-prompts.md b/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/05-git-flow-prompts.md deleted file mode 100644 index 41945d0e..00000000 --- a/evaluation/with_skills/rh-sre__job-template-remediation-validator/environment/skills/playbook-executor/references/05-git-flow-prompts.md +++ /dev/null @@ -1,97 +0,0 @@ -# Step 05: Git Flow Prompts - -Read this reference when executing Git Flow (Scenario 1 Override or Scenario 2). - -## Scenario 1 Prompt (Same path) - -The template already points to our playbook path. The project may need the latest content. - -``` -Found template [name] (ID: X) with matching playbook path. The project may need to be updated with the latest playbook. - -Options: -(A) Override: I'll add/update the playbook in the project via git. You sync the AAP project, then confirm. -(B) Manual: You add the playbook and sync. Confirm when done. - -❓ Choose (A) or (B): -``` - -- **If A**: Execute Git Flow (see Git Flow section below). Wait for user: "Sync complete" or "done". -- **If B**: Wait for user confirmation. - -## Scenario 2 Prompt (Different path) - -**CRITICAL**: The template points to a DIFFERENT playbook than our generated playbook. You MUST NOT launch the job without Git Flow. AAP executes from synced project content—there is no "override at launch". The playbook MUST be in the repo before any job launch. - -``` -Found template [name] (ID: X) pointing to [template.playbook]. Our generated playbook is [our_playbook_path]. - -⚠️ The template's playbook path does NOT match. We must update the playbook in the project before execution. - -Options: -- "yes" or "proceed" - I'll add our playbook to the project via git (you'll confirm commit/push, then sync AAP) -- "no" - Create a new template via `/job-template-creator` skill - -❓ Proceed with playbook update (git flow)? -``` - -- **If yes**: Execute Git Flow. **Do NOT proceed to Phase 3 until Git Flow completes.** -- **If no**: Fall through to Scenario 3 (job-template-creator). - -## Repo Path Question - -``` -What is the local path to the Git repository for project [Project Name] (scm_url)? -``` - -Use `projects_list` to get project name and `scm_url`; display to help user identify the repo. - -**Path format**: Ask for the **absolute path** (e.g. `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project`). When writing the playbook, the Write tool path MUST be `/playbooks/remediation/` — the full absolute path. Do NOT use a relative path like `test-aap-project/playbooks/...`; that causes "Error writing file". - -## Git Flow: Write Step (FAST) - -**CRITICAL**: The playbook is ALREADY generated. During Git Flow you must WRITE the existing content to disk—nothing more. - -- **DO**: Single file write of the playbook content already in context (from playbook-generator or remediation) -- **DO NOT**: Invoke playbook-generator again, call create_vulnerability_playbook, re-fetch from MCP, or validate/transform the content -- **Expected duration**: Seconds. If it takes minutes, you are doing unnecessary work. - -### Write Path (ABSOLUTE REQUIRED) - -**⚠️ WRITE PATH MUST START WITH `/`** — The Write tool path MUST be an absolute path. Relative paths cause "Error writing file" because the repo is often outside the workspace. - -**Formula**: `write_path = user_provided_path + "/" + target_path` - -- `user_provided_path` = exactly what the user typed (e.g. `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project`) -- `target_path` = e.g. `playbooks/remediation/cve-remediation.yml` - -**Correct**: `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project/playbooks/remediation/cve-remediation.yml` - -**WRONG** (will fail): -- `test-aap-project/playbooks/remediation/cve-remediation.yml` -- `playbooks/remediation/cve-remediation.yml` - -**Before calling Write**: Verify the path starts with `/`. If it does not, prepend the user's repo path. - -## Git Flow HITL Checkpoint - -**REQUIRED** before commit/push: - -``` -Ready to commit and push these changes? -- File: [target_path] -- CVE: [cve_id] -- This will update the playbook in the AAP project. - -Reply 'yes' or 'proceed' to continue, or 'abort' to cancel. -``` - -**Wait for user confirmation.** If "yes" or "proceed": `git commit -m "Add/update remediation playbook for CVE-YYYY-NNNNN"` then `git push origin main`. - -## After Push Message - -``` -I've pushed the playbook. Sync the AAP project: Automation Execution > Projects > [Project] > Sync. Reply 'sync complete' when done. -``` - -**Do NOT proceed to Phase 3 (Dry-Run) until user confirms sync complete.** diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 1092dd1d..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- pass rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/skills/mcp-aap-validator/SKILL.md b/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/skills/mcp-aap-validator/SKILL.md deleted file mode 100644 index a1c4f708..00000000 --- a/evaluation/with_skills/rh-sre__mcp-aap-validator/environment/skills/mcp-aap-validator/SKILL.md +++ /dev/null @@ -1,66 +0,0 @@ ---- -name: mcp-aap-validator -description: | - Validate AAP (Ansible Automation Platform) MCP server connectivity. Use when the user asks to "validate AAP MCP", "check AAP connection", or when other skills need to verify AAP MCP availability before job management or inventory operations. -model: haiku -color: yellow ---- - -# MCP AAP Validator - -Validates connectivity to AAP MCP servers by running lightweight tool calls. - -## When to Use This Skill - -Use when validating AAP MCP before job template operations, troubleshooting connection issues, or when other skills (e.g. playbook-executor) need to verify availability. Do NOT use for creating templates—use job-template-creator. - -## Workflow - -1. **Test connectivity**: Call these tools to verify each server responds: - - `job_templates_list` (page_size: 10) from aap-mcp-job-management - - `inventories_list` (page_size: 10) from aap-mcp-inventory-management -2. **If any fails**: Provide a comprehensive message with possible root causes (see below). -3. **Report**: Output a table with validated servers and outcome (emojis). - -## Failure Message (Root Causes) - -When a tool call fails, include: - -``` -❌ AAP MCP connection failed - -**Possible root causes:** -- **Credentials**: AAP_MCP_SERVER or AAP_API_TOKEN not set or invalid -- **401 Unauthorized**: Token expired or invalid → regenerate in AAP Web UI -- **403 Forbidden**: Token lacks RBAC permissions (need Job Templates, Inventories) -- **404 Not Found**: Wrong AAP_MCP_SERVER URL (must point to MCP gateway, not main AAP UI) -- **Connection timeout**: Server unreachable, firewall, or network issue -- **SSL/TLS error**: Certificate verification problem - -**Troubleshooting:** -1. Verify env vars: AAP_MCP_SERVER, AAP_API_TOKEN (never echo values) -2. Get token: AAP Web UI → Users → [Your User] → Tokens → Create -3. Ensure AAP_MCP_SERVER points to MCP gateway endpoint -4. Restart host after config changes -``` - -## Report Format - -Always end with a table: - -| Server | Outcome | -|--------|---------| -| aap-mcp-job-management | ✅ PASSED | -| aap-mcp-inventory-management | ✅ PASSED | - -Use ✅ for success, ❌ for failure, ⚠️ for partial (e.g. one server OK, one failed). - -## Dependencies - -### Required MCP Tools -- `job_templates_list` (from aap-mcp-job-management) - Connectivity test -- `inventories_list` (from aap-mcp-inventory-management) - Connectivity test - -### Required MCP Servers -- `aap-mcp-job-management` - AAP job template and execution -- `aap-mcp-inventory-management` - AAP inventory management diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 1092dd1d..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- pass rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/skills/mcp-lightspeed-validator/SKILL.md b/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/skills/mcp-lightspeed-validator/SKILL.md deleted file mode 100644 index e1f1528e..00000000 --- a/evaluation/with_skills/rh-sre__mcp-lightspeed-validator/environment/skills/mcp-lightspeed-validator/SKILL.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -name: mcp-lightspeed-validator -description: | - Validate Red Hat Lightspeed MCP server connectivity. Use when the user asks to "validate Lightspeed MCP", "check Lightspeed connection", or when other skills need to verify lightspeed-mcp availability before CVE operations. -model: haiku -color: yellow ---- - -# MCP Lightspeed Validator - -Validates connectivity to the Red Hat Lightspeed MCP server by running a lightweight tool call. - -## When to Use This Skill - -Use when validating Lightspeed MCP before CVE operations, troubleshooting connection issues, or when other skills (e.g. remediation) need to verify availability. Do NOT use for actual CVE queries—use cve-impact or cve-validation. - -## Workflow - -1. **Test connectivity**: Call `vulnerability__get_cves` with **no parameters** (uses default limit=10). Do NOT pass `limit`—some MCP clients incorrectly serialize it as `limit_`, causing validation errors. -2. **If it fails**: Provide a comprehensive message with possible root causes (see below). -3. **Report**: Output a table with validated servers and outcome (emojis). - -## Failure Message (Root Causes) - -When the tool call fails, include: - -``` -❌ Lightspeed MCP connection failed - -**Possible root causes:** -- **Credentials**: LIGHTSPEED_CLIENT_ID or LIGHTSPEED_CLIENT_SECRET not set or invalid -- **Expired credentials**: Red Hat Console tokens may have expired -- **Server not running**: MCP server/container may be stopped -- **Network**: Firewall or proxy blocking console.redhat.com -- **Configuration**: .mcp.json misconfigured or server not registered - -**Troubleshooting:** -1. Verify env vars: LIGHTSPEED_CLIENT_ID, LIGHTSPEED_CLIENT_SECRET (never echo values) -2. Check credentials at: https://console.redhat.com/settings/integrations -3. Restart MCP server or host after config changes -4. Check container logs if using podman/docker -``` - -## Report Format - -Always end with a table: - -| Server | Outcome | -|--------|---------| -| lightspeed-mcp | ✅ PASSED | -| lightspeed-mcp | ❌ FAILED | - -Use ✅ for success, ❌ for failure, ⚠️ for partial (e.g. connected but error on tool). - -## Dependencies - -### Required MCP Tools -- `vulnerability__get_cves` or `get_cves` (from lightspeed-mcp) - Connectivity test - -### Required MCP Servers -- `lightspeed-mcp` - Red Hat Lightspeed vulnerability and inventory data diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 1092dd1d..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- pass rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/mcp-aap-validator/SKILL.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/mcp-aap-validator/SKILL.md deleted file mode 100644 index a1c4f708..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/mcp-aap-validator/SKILL.md +++ /dev/null @@ -1,66 +0,0 @@ ---- -name: mcp-aap-validator -description: | - Validate AAP (Ansible Automation Platform) MCP server connectivity. Use when the user asks to "validate AAP MCP", "check AAP connection", or when other skills need to verify AAP MCP availability before job management or inventory operations. -model: haiku -color: yellow ---- - -# MCP AAP Validator - -Validates connectivity to AAP MCP servers by running lightweight tool calls. - -## When to Use This Skill - -Use when validating AAP MCP before job template operations, troubleshooting connection issues, or when other skills (e.g. playbook-executor) need to verify availability. Do NOT use for creating templates—use job-template-creator. - -## Workflow - -1. **Test connectivity**: Call these tools to verify each server responds: - - `job_templates_list` (page_size: 10) from aap-mcp-job-management - - `inventories_list` (page_size: 10) from aap-mcp-inventory-management -2. **If any fails**: Provide a comprehensive message with possible root causes (see below). -3. **Report**: Output a table with validated servers and outcome (emojis). - -## Failure Message (Root Causes) - -When a tool call fails, include: - -``` -❌ AAP MCP connection failed - -**Possible root causes:** -- **Credentials**: AAP_MCP_SERVER or AAP_API_TOKEN not set or invalid -- **401 Unauthorized**: Token expired or invalid → regenerate in AAP Web UI -- **403 Forbidden**: Token lacks RBAC permissions (need Job Templates, Inventories) -- **404 Not Found**: Wrong AAP_MCP_SERVER URL (must point to MCP gateway, not main AAP UI) -- **Connection timeout**: Server unreachable, firewall, or network issue -- **SSL/TLS error**: Certificate verification problem - -**Troubleshooting:** -1. Verify env vars: AAP_MCP_SERVER, AAP_API_TOKEN (never echo values) -2. Get token: AAP Web UI → Users → [Your User] → Tokens → Create -3. Ensure AAP_MCP_SERVER points to MCP gateway endpoint -4. Restart host after config changes -``` - -## Report Format - -Always end with a table: - -| Server | Outcome | -|--------|---------| -| aap-mcp-job-management | ✅ PASSED | -| aap-mcp-inventory-management | ✅ PASSED | - -Use ✅ for success, ❌ for failure, ⚠️ for partial (e.g. one server OK, one failed). - -## Dependencies - -### Required MCP Tools -- `job_templates_list` (from aap-mcp-job-management) - Connectivity test -- `inventories_list` (from aap-mcp-inventory-management) - Connectivity test - -### Required MCP Servers -- `aap-mcp-job-management` - AAP job template and execution -- `aap-mcp-inventory-management` - AAP inventory management diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/SKILL.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/SKILL.md deleted file mode 100644 index a29c9443..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/SKILL.md +++ /dev/null @@ -1,499 +0,0 @@ ---- -name: playbook-executor -description: | - **CRITICAL**: Use for Ansible playbook execution via AAP. DO NOT call AAP MCP tools directly. - - Execute remediation playbooks with job management, dry-run, and reporting. Use after playbook-generator. - - **Git Flow**: If template playbook path ≠ generated playbook, perform Git Flow (commit, push, sync) BEFORE launch. ---- - -# AAP Playbook Executor Skill - -This skill executes Ansible remediation playbooks through AAP (Ansible Automation Platform) with full job management capabilities. - -**Integration with Remediation Skill**: The `/remediation` skill orchestrates this skill as part of its Step 5 (Execute Playbook) workflow. For standalone playbook execution, you can invoke this skill directly. - -## Prerequisites - -**Required MCP Servers**: `aap-mcp-job-management`, `aap-mcp-inventory-management` ([setup guide](https://docs.redhat.com/)) - -**Required MCP Tools**: -- `job_templates_list` (from aap-mcp-job-management) - List job templates -- `job_templates_retrieve` (from aap-mcp-job-management) - Get template details -- `projects_list` (from aap-mcp-job-management) - Get project name and scm_url for Git Flow -- `job_templates_launch_retrieve` (from aap-mcp-job-management) - Launch jobs -- `jobs_retrieve` (from aap-mcp-job-management) - Get job status -- `jobs_stdout_retrieve` (from aap-mcp-job-management) - Get console output -- `jobs_job_events_list` (from aap-mcp-job-management) - Get task events -- `jobs_job_host_summaries_list` (from aap-mcp-job-management) - Get host statistics -- `inventories_list` (from aap-mcp-inventory-management) - List inventories -- `hosts_list` (from aap-mcp-inventory-management) - List inventory hosts - -**Required Environment Variables**: -- `AAP_MCP_SERVER` - Base URL for the MCP endpoint of the AAP server (must point to the AAP MCP gateway) -- `AAP_API_TOKEN` - AAP API authentication token - -### Prerequisite Validation - -**CRITICAL**: Before executing operations, execute the `/mcp-aap-validator` skill to verify AAP MCP server availability. - -**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-aap-validator/SKILL.md#validation-freshness-policy). - -**How to invoke**: Execute the `/mcp-aap-validator` skill - -**Handle validation result**: -- **If validation PASSED**: Continue with playbook execution workflow -- **If validation PARTIAL**: Warn user and ask to proceed -- **If validation FAILED**: Stop execution, provide setup instructions from validator - -**Human Notification on Failure**: -If prerequisites are not met: -- ❌ "Cannot proceed: AAP MCP servers are not available" -- 📋 "Setup required: Configure AAP_MCP_SERVER and AAP_API_TOKEN environment variables" -- ❓ "How would you like to proceed? (setup now / skip / abort)" -- ⏸️ Wait for user decision - -## When to Use This Skill - -**Use this skill directly when you need**: -- Execute a previously generated Ansible playbook via AAP -- Track the status of a running AAP job -- Monitor playbook job completion -- Run dry-run (check mode) before production execution -- Verify playbook execution succeeded - -**Use the `/remediation` skill when you need**: -- Full remediation workflow including playbook execution -- Integrated CVE analysis → playbook generation → execution → verification -- End-to-end remediation orchestration - -**How they work together**: The `/remediation` skill invokes this skill after generating a remediation playbook, managing the full workflow from analysis to verification. - -## Workflow - -**Git Flow is MANDATORY**: When the job template's playbook path differs from the generated playbook (or content must be updated), you MUST perform Git Flow (write, commit, push, sync) and receive "sync complete" from the user BEFORE launching any job. Do NOT skip this—launching without it executes the wrong playbook. - -### Phase 0: Validate AAP MCP Prerequisites - -**Action**: Execute the `/mcp-aap-validator` skill - -**Note**: Can skip if validation was performed earlier in this session and succeeded. - -**How to invoke**: Execute the `/mcp-aap-validator` skill - -**Handle validation result**: -- **If validation PASSED**: Continue to Phase 1 -- **If validation PARTIAL**: Warn user and ask to proceed -- **If validation FAILED**: Stop execution, user must set up AAP MCP servers - -### Phase 1: Job Template Selection and Playbook Preparation - -**Goal**: Identify an AAP job template suitable for executing the remediation playbook. **Git Flow is MANDATORY** before Phase 3 when the template points to a different playbook or when content must be updated. - -**Input**: Playbook content and metadata from playbook-generator (filename, CVE ID, target systems). The playbook YAML is already in context—do NOT regenerate it during Git Flow. Playbook path is derived from metadata: `playbooks/remediation/` (e.g., `playbooks/remediation/remediation-CVE-2025-49794.yml`). - -**BLOCKING**: You MUST NOT launch any job (dry-run or production) until the playbook is in the Git repo and the user has confirmed "sync complete". AAP executes from the synced project—there is no "override at launch". Launching without Git Flow executes the WRONG playbook. - -#### Step 1.1: Derive Playbook Path - -From playbook metadata (filename from playbook-generator): -- Use convention `playbooks/remediation/` -- Support both `remediation-CVE-*.yml` and `remediation-CVE-*-playbook.yml` patterns. -- Example: CVE-2026-26103 → `playbooks/remediation/remediation-CVE-2026-26103.yml` - -#### Step 1.2: List Templates and Validate Each Candidate - -**MCP Tool**: `job_templates_list` (from aap-mcp-job-management) - -**Parameters**: -- `page_size`: 50 (retrieve up to 50 templates) -- `search`: "" (search for all templates) - -**REQUIRED**: For each template in results: -1. Call `job_templates_retrieve(id)` to get full details -2. **Invoke the `/job-template-remediation-validator` skill** with the template ID to verify it meets remediation requirements (inventory, project, playbook, credentials, become_enabled) -3. Only include templates that PASS validation in the lists below - -Build two lists: -- **exact_match**: `template.playbook` equals `our_playbook_path` (normalize slashes; match if equal or basenames match) -- **compatible_other**: Passes job-template-remediation-validator but **different playbook path** (template points to e.g. `cve-remediation.yml` while we have `remediation-CVE-2026-26103.yml`) - -**Path normalization**: Normalize slashes, handle `playbooks/remediation/` prefix. Match if `template.playbook` equals `our_playbook_path` or if basenames match. **Different filenames = different path = Scenario 2.** - -#### Step 1.3: Scenario Selection (MANDATORY - Do Not Skip) - -**Scenario 1 - Same playbook path** (exact_match not empty): - -The template already points to our playbook path. The project may need the latest content. **Read [references/05-git-flow-prompts.md](references/05-git-flow-prompts.md)** for Scenario 1 prompt, options (A/B), and Git Flow steps. - -- **If A**: Execute Git Flow (see Git Flow section below). **BLOCK Phase 3** until user confirms "sync complete" or "done". -- **If B**: Wait for user confirmation. **BLOCK Phase 3** until user confirms. - -**Scenario 2 - Different playbook path** (compatible_other not empty, exact_match empty): - -**CRITICAL**: The template points to a DIFFERENT playbook than our generated playbook. You MUST NOT launch the job without Git Flow—AAP executes from synced content; there is no override at launch. **Read [references/05-git-flow-prompts.md](references/05-git-flow-prompts.md)** for Scenario 2 prompt and Git Flow steps. - -- **If yes**: Execute Git Flow. **BLOCK Phase 3** until Git Flow completes and user confirms "sync complete". -- **If no**: Fall through to Scenario 3. - -**Anti-pattern**: Do NOT say "I'll override with our playbook" and then launch—that is impossible. The playbook MUST be in the repo before launch. - -**Scenario 3 - No suitable template** (exact_match and compatible_other both empty, or user chose "no" in Scenario 2): - -Execute the `/job-template-creator` skill with instruction: -``` -"Create a job template for this remediation playbook. Playbook: [content]. Filename: [filename]. Path: [our_playbook_path]. CVE: [cve_id]. Target systems: [list]." -``` - -The job-template-creator skill guides the user through: (1) Adding playbook to Git repository, (2) Syncing AAP project, (3) Creating job template via AAP Web UI with correct path, inventory, credentials, privilege escalation. - -After `/job-template-creator` completes, retrieve the template ID (from skill output or user confirmation). Execute `/job-template-remediation-validator` to validate the newly created template. If passed, proceed to Phase 3 (Dry-Run). If failed, report issues and ask user to fix in AAP Web UI. - -**Multiple matches**: If multiple exact matches, present list and ask user to choose by number. If multiple different-path matches, prefer by project name containing "remediation" or "CVE", else first. - -**Phase 1 Checkpoint** (BLOCKING - must pass before Phase 3): -- **Git Flow required**: If Scenario 1 or 2, you MUST complete Git Flow and receive "sync complete" from the user before proceeding. Do NOT skip. -- **No override**: There is no way to "override" the playbook at launch. AAP runs whatever is in the synced project. -- **Never launch** if the playbook has not been committed, pushed, and synced - -#### Git Flow (for Scenario 1 Override and Scenario 2) - MANDATORY HITL - -**When**: Scenario 1 (same path, update content) or Scenario 2 (different path, replace playbook). **Do not skip**—execution with wrong playbook content will remediate the wrong CVE. - -**Target path**: -- Scenario 1: `our_playbook_path` (e.g. `playbooks/remediation/remediation-CVE-2026-26103.yml`) -- Scenario 2: `template.playbook` (e.g. `playbooks/remediation/cve-remediation.yml`)—we replace the template's playbook with our generated content - -**Prerequisite**: Ask user for the local path to the Git repository. Use `projects_list` for project name and `scm_url`. **Read [references/05-git-flow-prompts.md](references/05-git-flow-prompts.md)** for repo path question, HITL checkpoint text, and after-push message. - -**Steps** (execute in order; HITL at checkpoint): -1. **Write playbook to file** (FAST—do NOT regenerate): - - The playbook content is ALREADY in context from playbook-generator (or remediation skill). Use it directly. - - **⚠️ ABSOLUTE PATH REQUIRED**: The Write path MUST start with `/`. Use: `/`. Example: `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project/playbooks/remediation/cve-remediation.yml` - - **WRONG** (causes "Error writing file"): `test-aap-project/playbooks/...` or `playbooks/remediation/...` — these are relative and fail when repo is outside workspace. - - **Before Write**: Confirm path starts with `/`. If not, prepend the user's repo path. - - Do NOT invoke playbook-generator, do NOT call MCP tools, do NOT re-fetch. This should take seconds, not minutes. -2. Use Run tool: `git add ` (from repo root, e.g. `git add playbooks/remediation/cve-remediation.yml`) -3. **HITL Checkpoint** (REQUIRED): Display summary per reference file. Wait for "yes" or "proceed" -4. If confirmed: `git commit -m "Add/update remediation playbook for CVE-YYYY-NNNNN"` -5. `git push origin main` (or branch from project's scm_branch if available) - -**Note**: Git must be configured. Use Run tool for git commands. - -**Do NOT proceed to Phase 3 (Dry-Run) until user confirms sync complete.** - -### Phase 2: Git Flow (MANDATORY before Phase 3) - -**BLOCKING**: You MUST NOT proceed to Phase 3 (Dry-Run) until Git Flow is complete. - -**When**: Scenario 1 (same path, update content) or Scenario 2 (different path). See Phase 1 Step 1.3. - -**Checkpoint**: Before Phase 3, confirm: -- [ ] Playbook written to repo at target path -- [ ] Git commit and push completed (with user confirmation) -- [ ] User confirmed "sync complete" after AAP project sync - -**If any unchecked**: STOP. Do Git Flow. Do NOT launch the job. - -### Phase 3: Dry-Run Execution (Recommended) - -**Prerequisite**: Phase 2 (Git Flow) MUST be complete. User must have confirmed "sync complete". - -**Goal**: Test playbook in check mode before actual execution to simulate changes. - -**Read [references/04-dry-run-display-templates.md](references/04-dry-run-display-templates.md)** for: Playbook Preview, Dry-Run Offer, Dry-Run Results Display, Proceed prompt. - -#### Step 3.1–3.2: Display Preview and Offer Dry-Run - -Show playbook structure per reference. Offer dry-run with options: yes / no / abort. **ONLY if user confirms**, proceed. - -#### Step 3.3: Launch Dry-Run Job - -**Pre-launch check** (BLOCKING): If Scenario 1 or 2 applied, you MUST have completed Git Flow and received "sync complete" from the user. If not, STOP—do not launch. Return to Phase 2 / Git Flow. - -**MCP Tool**: `job_templates_launch_retrieve` (from aap-mcp-job-management) - -**Parameters**: `id`, `requestBody` with `job_type: "check"`, `extra_vars`, `limit` - -**Key**: `job_type: "check"` - Runs Ansible in check mode (dry-run) - -#### Step 3.4: Monitor Dry-Run Progress - -Poll `jobs_retrieve` every 2 seconds. Use `jobs_job_events_list` for live task updates. - -#### Step 3.5: Display Dry-Run Results - -**MCP Tools**: `jobs_stdout_retrieve` (id, format: "txt"), `jobs_job_host_summaries_list` (id). Use display format from reference. - -#### Step 3.6: Proceed to Actual Execution? - -Ask per reference. Wait for "yes" or "execute". - -### Phase 4: Actual Execution - -**ONLY execute if user explicitly confirms** (either after dry-run or directly if they skipped dry-run). - -#### Step 4.1: Final Confirmation - -``` -⚠️ CRITICAL: Playbook Execution Confirmation Required - -This playbook will: -- Execute on: 3 production systems -- Update packages: httpd (2.4.53-7.el9 → 2.4.57-8.el9) -- Restart services: httpd -- Estimated downtime: ~10 seconds per system -- Requires reboot: No - -Job Template: CVE Remediation Template (ID: 10) -AAP URL: https://aap.example.com/jobs/ - -❓ Execute this playbook now? - -Options: -- "yes" or "execute" - Proceed with execution -- "abort" - Cancel execution - -Please respond with your choice. -``` - -Wait for explicit "yes" or "execute" response. - -#### Step 4.2: Launch Production Job - -**Pre-launch check** (BLOCKING): Same as Phase 3—if Scenario 1 or 2 applied, Git Flow must be complete and user must have confirmed "sync complete". Do NOT launch without it. - -**MCP Tool**: `job_templates_launch_retrieve` (from aap-mcp-job-management) - -**Parameters**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run", - "extra_vars": { - "target_cve": "CVE-2025-49794", - "remediation_mode": "automated", - "verify_after": true - }, - "limit": "prod-web-01,prod-web-02,prod-web-03" - } -} -``` - -**Key Parameter**: `job_type: "run"` - Runs Ansible in execution mode (actual changes) - -**Expected Output**: -```json -{ - "job": 1235, - "status": "pending", - "url": "/api/controller/v2/jobs/1235/" -} -``` - -#### Step 4.3: Monitor Execution Progress - -**Polling Strategy**: -1. Call `jobs_retrieve(id=job_id)` every 2 seconds -2. Get task events with `jobs_job_events_list(id=job_id)` for progress updates -3. Display real-time task completion status -4. Continue until status is "successful", "failed", or "error" - -**Progress Display**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running -Elapsed: 1m 23s -AAP URL: https://aap.example.com/#/jobs/playbook/1235 - -Recent Events: -- ✓ Gathering Facts (completed - all hosts) -- ✓ Check Disk Space (completed - all hosts) -- ✓ Backup Configuration (completed - all hosts) -- ⏳ Update Package: httpd (running - prod-web-01, prod-web-02) - └─ prod-web-01: Installing httpd-2.4.57-8.el9... - └─ prod-web-02: Installing httpd-2.4.57-8.el9... -- ⏸ Restart Service: httpd (pending) -``` - -**Update every 2 seconds** until completion. - -### Phase 5: Execution Report - -**Goal**: Generate comprehensive report with job details, per-host results, and full output. - -**Read [references/01-execution-report-templates.md](references/01-execution-report-templates.md)** for JSON examples, comprehensive report template, and Success/Partial Success/Failure output templates. - -#### Step 5.1–5.4: Gather Data - -**MCP Tools** (all from aap-mcp-job-management): -- `jobs_retrieve` (id) - Job details -- `jobs_job_host_summaries_list` (id) - Per-host stats -- `jobs_job_events_list` (id) - Task timeline -- `jobs_stdout_retrieve` (id, format: "txt") - Full console output - -#### Step 5.5: Generate Report - -Format all gathered data per reference. Use Success / Partial Success / Failure template based on job status. - -#### Step 5.6: Validate Job Log for CVE Handling (MANDATORY) - -**Goal**: Confirm from the job stdout that the playbook actually addressed the target CVE(s). - -**Input**: Target CVE ID(s) from invocation (e.g. CVE-2025-49794). Job stdout from `jobs_stdout_retrieve` (already gathered in Step 5.4). - -**Parse stdout for**: -- Target CVE ID(s) in output (vars, task names, audit logs, playbook metadata) -- Package update tasks for affected packages (dnf/yum install/update, package module) -- Remediation-related task names (e.g. "Update package", "Restart service", "remediation") - -**Report** (add to execution report): -- **✓ Job log confirms CVE-XXXX-YYYY was addressed** — CVE ID or package updates found in stdout -- **⚠️ Job log did not show clear evidence of CVE handling** — No CVE ID or package updates found; recommend manual verification or `/remediation-verifier` - -**Batch**: For multiple CVEs, validate each. Report per-CVE confirmation or warning. - -### Phase 6: Error Handling - -**If job status is "failed" or "error"**, provide detailed troubleshooting. - -**Read [references/02-error-handling-guide.md](references/02-error-handling-guide.md)** for: Error categories, error report template, troubleshooting steps, relaunch parameters. - -#### Step 6.1: Parse Error Output - -**MCP Tool**: `jobs_stdout_retrieve`. Analyze output for error categories per reference. - -#### Step 6.2: Generate Error Report - -Use error report template from reference. Include per-host results, failed task details, troubleshooting steps, relaunch options. - -#### Step 6.3: Offer Relaunch - -If user chooses relaunch: **MCP Tool** `jobs_relaunch_retrieve` with `hosts: "failed"`, `job_type: "run"` per reference. - -## Reference Files - -| File | Use When | -|------|----------| -| [01-execution-report-templates.md](references/01-execution-report-templates.md) | Phase 5 reports, Success/Partial/Failure output | -| [02-error-handling-guide.md](references/02-error-handling-guide.md) | Phase 6 error reports, relaunch | -| [03-workflow-examples.md](references/03-workflow-examples.md) | Demo full workflow, failure handling, skip dry-run | -| [04-dry-run-display-templates.md](references/04-dry-run-display-templates.md) | Phase 3 preview, offer, results, proceed prompt | -| [05-git-flow-prompts.md](references/05-git-flow-prompts.md) | Scenario 1/2 prompts, Git Flow HITL, after-push | - -## Dependencies - -### Required MCP Servers -- `aap-mcp-job-management` - AAP job management and execution -- `aap-mcp-inventory-management` - AAP inventory management - -### Required MCP Tools -- `job_templates_list` (from aap-mcp-job-management) - List templates -- `job_templates_retrieve` (from aap-mcp-job-management) - Get template details -- `projects_list` (from aap-mcp-job-management) - Get project name and scm_url for Git Flow -- `job_templates_launch_retrieve` (from aap-mcp-job-management) - Launch jobs -- `jobs_retrieve` (from aap-mcp-job-management) - Get job status -- `jobs_stdout_retrieve` (from aap-mcp-job-management) - Get console output -- `jobs_job_events_list` (from aap-mcp-job-management) - Get task events -- `jobs_job_host_summaries_list` (from aap-mcp-job-management) - Get host statistics -- `inventories_list` (from aap-mcp-inventory-management) - List inventories -- `hosts_list` (from aap-mcp-inventory-management) - List hosts - -### Related Skills -- `mcp-aap-validator` - **PREREQUISITE** - Validates AAP MCP servers (invoke in Phase 0) -- `job-template-remediation-validator` - **REQUIRED** - Invoke for each candidate template in Phase 1 Step 1.2 to verify remediation requirements -- `job-template-creator` - Creates/guides AAP job template setup -- `playbook-generator` - Generates playbooks for execution -- `remediation-verifier` - Verifies success after execution - -### Reference Documentation -- [references/](references/) - Step-numbered reference files (01–05) for templates and examples -- [AAP Job Execution Guide](../../docs/ansible/aap-job-execution.md) - AAP job execution best practices -- [Playbook Integration with AAP](../../docs/ansible/playbook-integration-aap.md) - Playbook-to-AAP workflow - -## Critical: Human-in-the-Loop Requirements - -This skill executes code on production systems. **Explicit user confirmation is REQUIRED** at multiple stages. - -**Before Git commit/push** (Scenario 1 Override, Scenario 2): -1. **Display change summary**: File path, diff or file size -2. **Ask for confirmation**: "Ready to commit and push these changes? Reply 'yes' or 'proceed' to continue, or 'abort' to cancel." -3. **Wait for explicit "yes" or "proceed"**: Do not commit/push without confirmation - -**Before Dry-Run Execution** (if user chooses dry-run): -1. **Display Playbook Preview**: Show tasks and explain changes -2. **Ask for Dry-Run Confirmation**: - ``` - ❓ Run dry-run to simulate changes? - - Options: - - "yes" - Run dry-run (recommended) - - "no" - Skip to actual execution - - "abort" - Cancel - - Please respond with your choice. - ``` -3. **Wait for Explicit Response**: Do not proceed without confirmation - -**Before Actual Execution** (REQUIRED): -1. **Display Execution Summary**: Show systems, changes, downtime estimate -2. **Ask for Final Confirmation**: - ``` - ⚠️ CRITICAL: Execute playbook on production systems? - - This will make real changes to N systems. - - Options: - - "yes" or "execute" - Proceed - - "abort" - Cancel - - Please respond with your choice. - ``` -3. **Wait for Explicit "yes" or "execute"**: Do not proceed without confirmation - -**Never assume approval** - always wait for explicit user confirmation before executing playbooks. - -## Best Practices - -1. **Write path must be absolute** - When Git Flow writes the playbook to the user's repo, use `/playbooks/remediation/`. The path MUST start with `/`. Relative paths cause "Error writing file". -2. **Always validate AAP prerequisites** - Invoke mcp-aap-validator in Phase 0 -3. **Validate each template** - Invoke job-template-remediation-validator for each candidate before selection -4. **Never skip Git Flow** - If template playbook path ≠ generated playbook path (Scenario 2) or content must be updated (Scenario 1), you MUST complete Git Flow and receive "sync complete" before Phase 3. Do NOT launch without it. -5. **Recommend dry-run** - Offer check mode before production execution -6. **Filter compatible templates** - Check inventory, project, and credentials match -7. **Monitor in real-time** - Display task progress during execution -8. **Comprehensive reporting** - Include per-host stats, task timeline, full output -9. **Error categorization** - Parse errors and provide specific troubleshooting -10. **Relaunch capability** - Offer to retry failed hosts -11. **Link to AAP** - Provide direct URL to job in AAP Web UI -12. **Suggest verification** - Always recommend remediation-verifier after success -13. **Document job details** - Save job ID and template info for audit trail - -## Integration with Other Skills - -- **playbook-generator**: Generates playbooks that this skill executes -- **job-template-creator**: Creates AAP job templates when needed -- **remediation-verifier**: Verifies success after this skill completes execution -- **`/remediation` skill**: Orchestrates full workflow including playbook execution - -**Orchestration Example** (from `/remediation` skill): -1. Agent invokes playbook-generator skill → Creates playbook YAML -2. playbook-generator asks for confirmation → User approves playbook content -3. Agent invokes playbook-executor skill (this skill) → Execution workflow -4. Skill validates templates via job-template-remediation-validator → Filters valid candidates -5. Skill checks path match → If different path, offers Git Flow (HITL: commit/push, sync AAP) -6. Skill waits for "sync complete" before proceeding (if Git Flow was used) -7. Skill offers dry-run → User runs check mode -8. Skill asks for execution confirmation → User approves -9. Skill executes and monitors → Reports completion -10. Agent invokes remediation-verifier skill → Confirms CVE resolved - -**Note**: Both playbook-generator and playbook-executor require separate confirmations for different purposes: -- playbook-generator: Confirms playbook content is acceptable -- playbook-executor: Confirms execution on production systems is approved - -This two-step approval ensures user control over both what to run and when to run it. diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/01-execution-report-templates.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/01-execution-report-templates.md deleted file mode 100644 index a6773c5f..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/01-execution-report-templates.md +++ /dev/null @@ -1,168 +0,0 @@ -# Step 01: Execution Report Templates - -Read this reference when generating Phase 5 execution reports or output templates. - -## Phase 5: Job Details (JSON Examples) - -### jobs_retrieve Expected Output - -```json -{ - "id": 1235, - "name": "CVE Remediation Template", - "status": "successful", - "started": "2026-02-24T15:35:02Z", - "finished": "2026-02-24T15:40:25Z", - "elapsed": 323.45, - "job_template": 10, - "inventory": 1, - "limit": "prod-web-01,prod-web-02,prod-web-03", - "playbook": "playbooks/remediation/remediation-CVE-2025-49794.yml" -} -``` - -### jobs_job_host_summaries_list Expected Output - -```json -{ - "results": [ - { - "host_name": "prod-web-01", - "ok": 8, - "changed": 3, - "failed": 0, - "unreachable": 0 - }, - { - "host_name": "prod-web-02", - "ok": 8, - "changed": 3, - "failed": 0, - "unreachable": 0 - }, - { - "host_name": "prod-web-03", - "ok": 5, - "changed": 0, - "failed": 1, - "unreachable": 0 - } - ] -} -``` - -## Comprehensive Report Template - -```markdown -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 5m 23s -**Started**: 2026-02-24 15:35:02 UTC -**Completed**: 2026-02-24 15:40:25 UTC -**Job Template**: CVE Remediation Template -**Playbook**: playbooks/remediation/remediation-CVE-2025-49794.yml -**AAP URL**: [View in AAP](https://aap.example.com/#/jobs/playbook/1235) - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| prod-web-01 | 8 | 3 | 0 | 0 | ✅ Success | -| prod-web-02 | 8 | 3 | 0 | 0 | ✅ Success | -| prod-web-03 | 8 | 3 | 0 | 0 | ✅ Success | - -**Summary**: 3 of 3 hosts successfully remediated - -## Task Timeline -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) - - prod-web-01: 2.4.53-7.el9 → 2.4.57-8.el9 - - prod-web-02: 2.4.53-7.el9 → 2.4.57-8.el9 - - prod-web-03: 2.4.53-7.el9 → 2.4.57-8.el9 -5. ✅ Restart httpd service (15s) -6. ✅ Verify service status (2s) -7. ✅ Update audit log (1s) - -## Full Console Output -
-Click to expand (187 lines) - -[Full stdout from jobs_stdout_retrieve] - -
- -## Job Log CVE Validation (Step 5.6) -✓ Job log confirms CVE-XXXX-YYYY was addressed - -*(Or: ⚠️ Job log did not show clear evidence of CVE handling—verify manually or use remediation-verifier)* - -## Next Steps -1. ✅ All systems successfully remediated -2. ☐ Verify remediation with remediation-verifier skill -3. ☐ Update vulnerability tracking system -4. ☐ Schedule follow-up verification in 24-48 hours - ---- - -**Recommendation**: Run remediation-verifier skill to confirm CVE status has been updated in Red Hat Lightspeed. -``` - -## Output Templates - -### Success Template - -```markdown -✅ Playbook Execution Successful - -Job ID: 1235 -Duration: 5m 23s -Systems Remediated: 3 of 3 - -View full report above for details. - -Next Steps: -- Run remediation-verifier skill to confirm CVE resolution -- Update vulnerability tracking system -- Monitor systems for 24-48 hours - -AAP URL: https://aap.example.com/#/jobs/playbook/1235 -``` - -### Partial Success Template - -```markdown -⚠️ Playbook Execution Completed with Failures - -Job ID: 1235 -Duration: 2m 45s -Systems Remediated: 2 of 3 -Failed Systems: prod-web-03 - -See error details above for troubleshooting steps. - -Options: -- Relaunch for failed hosts -- Manual remediation -- Skip failed hosts - -AAP URL: https://aap.example.com/#/jobs/playbook/1235 -``` - -### Failure Template - -```markdown -❌ Playbook Execution Failed - -Job ID: 1235 -Duration: 1m 15s -Systems Remediated: 0 of 3 - -Critical errors prevented execution. -See error details above for troubleshooting. - -AAP URL: https://aap.example.com/#/jobs/playbook/1235 -``` diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/02-error-handling-guide.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/02-error-handling-guide.md deleted file mode 100644 index 90492f00..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/02-error-handling-guide.md +++ /dev/null @@ -1,108 +0,0 @@ -# Step 02: Error Handling Guide - -Read this reference when generating Phase 6 error reports or troubleshooting. - -## Error Categories - -**Parse error output** from `jobs_stdout_retrieve` for these common patterns: - -1. **Connection Failures**: SSH timeout, host unreachable, authentication failed -2. **Permission Errors**: sudo required, insufficient privileges, SELinux denials -3. **Package Manager Issues**: repo unavailable, package not found, dependency conflicts -4. **Service Failures**: service not found, restart failed, timeout -5. **Disk Space**: insufficient space for updates -6. **General Failures**: playbook syntax errors, task failures - -## Error Report Template - -```markdown -# Playbook Execution Failed - -## Job Summary -**Job ID**: 1235 -**Status**: ❌ Failed -**Duration**: 2m 45s -**Started**: 2026-02-24 15:35:02 UTC -**Failed At**: 2026-02-24 15:37:47 UTC -**Job Template**: CVE Remediation Template -**AAP URL**: [View in AAP](https://aap.example.com/#/jobs/playbook/1235) - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| prod-web-01 | 8 | 3 | 0 | 0 | ✅ Success | -| prod-web-02 | 8 | 3 | 0 | 0 | ✅ Success | -| prod-web-03 | 5 | 0 | 1 | 0 | ❌ Failed | - -**Summary**: 2 of 3 hosts succeeded, 1 failed - -## Failed Tasks Details - -### Host: prod-web-03 - -**Task**: Restart httpd service -**Error**: "Failed to restart httpd.service: Unit httpd.service not found." - -**Error Category**: Service Failure - -**Root Cause**: The httpd service is not installed or not recognized by systemd. - -**Troubleshooting Steps**: -1. Check if httpd is installed: - ```bash - ssh prod-web-03 'rpm -q httpd' - ``` -2. If not installed, the package update may have failed: - ```bash - ssh prod-web-03 'dnf info httpd' - ``` -3. Check systemd service status: - ```bash - ssh prod-web-03 'systemctl status httpd' - ``` -4. Review package manager logs: - ```bash - ssh prod-web-03 'tail -50 /var/log/dnf.log' - ``` - -**Recommended Action**: -- Verify httpd package installation on prod-web-03 -- Check if package update completed successfully -- Manually install httpd if needed: `dnf install httpd` -- Relaunch job for failed host only - -## Console Output (Last 50 Lines) -
-Click to expand error context - -[Relevant error output from jobs_stdout_retrieve] - -
- -## Relaunch Options - -Would you like to: -1. **Relaunch for failed hosts only** - Run job again with limit="prod-web-03" -2. **Fix issues manually and relaunch** - Resolve problems first, then relaunch -3. **View full job output** - See complete execution logs -4. **Abort** - Stop remediation workflow - -Please choose an option (1-4): -``` - -## Relaunch Parameters - -**MCP Tool**: `jobs_relaunch_retrieve` (from aap-mcp-job-management) - -**Parameters**: -```json -{ - "id": "1235", - "requestBody": { - "hosts": "failed", - "job_type": "run" - } -} -``` - -This relaunches the job for only the failed hosts. diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/03-workflow-examples.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/03-workflow-examples.md deleted file mode 100644 index f5caa0d1..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/03-workflow-examples.md +++ /dev/null @@ -1,119 +0,0 @@ -# Step 03: Workflow Examples - -Read this reference when demonstrating end-to-end workflow patterns. - -## Example 1: Full Workflow with Dry-Run - -**User Request**: "Execute the CVE-2025-49794 remediation playbook" - -**Skill Response**: - -1. **Validate AAP Prerequisites**: - - Invoke mcp-aap-validator skill → PASSED - -2. **List Job Templates**: - - Call `job_templates_list()` → Found 2 templates - - Filter compatible templates → 1 matches requirements - -3. **User Selects Template**: - ``` - Found 1 compatible job template: - 1. "CVE Remediation Template" (ID: 10) - - Select template (1) or "create" for new: 1 - ``` - -4. **Playbook Preparation**: - ``` - Guide user to add playbook to Git: - - Commands provided - - User syncs AAP project - - Verification: Playbook available ✓ - ``` - -5. **Offer Dry-Run**: - ``` - Run dry-run first? yes - ``` - -6. **Execute Dry-Run**: - - Launch with `job_type="check"` - - Monitor progress → COMPLETED - - Display dry-run results: - ``` - Would change 3 tasks on 3 hosts - No errors detected - ``` - -7. **Proceed to Actual Execution**: - ``` - Proceed with actual execution? yes - ``` - -8. **Execute Playbook**: - - Launch with `job_type="run"` - - Monitor progress → COMPLETED - - Generate report: - ``` - ✅ 3 of 3 hosts successfully remediated - Duration: 5m 23s - ``` - -9. **Suggest Verification**: - ``` - Recommend: Run remediation-verifier skill to confirm CVE resolution - ``` - -## Example 2: Handle Execution Failure - -**User Request**: "Execute remediation playbook" - -**Skill Response**: - -1-7. [Same as Example 1 through execution] - -8. **Execute Playbook**: - - Launch job → FAILED on 1 of 3 hosts - - Generate error report: - ``` - ❌ prod-web-03 failed - Error: httpd service not found - Troubleshooting steps provided - ``` - -9. **Offer Relaunch**: - ``` - Relaunch for failed host only? yes - ``` - -10. **Relaunch Job**: - - Call `jobs_relaunch_retrieve` with hosts="failed" - - Monitor → COMPLETED - - Final report: - ``` - ✅ All 3 hosts successfully remediated (1 after retry) - ``` - -## Example 3: Skip Dry-Run - -**User Request**: "Execute playbook directly, skip dry-run" - -**Skill Response**: - -1-4. [Same as Example 1 through template selection] - -5. **Offer Dry-Run**: - ``` - Run dry-run first? no - ``` - -6. **Final Confirmation**: - ``` - ⚠️ Execute on production without dry-run? - This will make changes immediately. - Confirm: yes - ``` - -7. **Execute Playbook**: - - Launch with `job_type="run"` - - Monitor and report as in Example 1 diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/04-dry-run-display-templates.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/04-dry-run-display-templates.md deleted file mode 100644 index d9b6f0dc..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/04-dry-run-display-templates.md +++ /dev/null @@ -1,93 +0,0 @@ -# Step 04: Dry-Run Display Templates - -Read this reference when displaying Phase 3 dry-run content. - -## Playbook Preview - -```markdown -# Playbook Preview - -**Playbook**: remediation-CVE-2025-49794.yml -**Target Systems**: 5 systems - -## Tasks Overview: -1. **Gather Facts** - Collect system information -2. **Check Disk Space** - Ensure sufficient space for updates (>500MB) -3. **Backup Configuration** - Snapshot critical configs -4. **Update Package: httpd** - Upgrade to version 2.4.57-8.el9 -5. **Restart Service: httpd** - Apply changes -6. **Verify Service Status** - Confirm httpd is running -7. **Update Audit Log** - Record remediation event - -**Estimated Duration**: 3-5 minutes per system -**Requires Reboot**: No -**Downtime**: Brief (~10 seconds during service restart) -``` - -## Dry-Run Offer - -``` -⚠️ Recommended: Run dry-run first - -Dry-run mode (--check) simulates changes without applying them. -This helps identify: -- Package availability issues -- Permission problems -- Configuration conflicts -- Unexpected side effects - -❓ Run dry-run before actual execution? -- "yes" - Run dry-run first (recommended) -- "no" - Skip to actual execution -- "abort" - Cancel execution - -Please respond with your choice. -``` - -## Dry-Run Results Display - -```markdown -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s -**Completed**: 2024-01-20 15:32:17 UTC - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| prod-web-01 | 3 | 8 | 0 | ✓ Ready | -| prod-web-02 | 3 | 8 | 0 | ✓ Ready | -| prod-web-03 | 3 | 8 | 0 | ✓ Ready | - -## Changes That Would Be Made: -1. **httpd package** - Would update from 2.4.53-7.el9 to 2.4.57-8.el9 -2. **httpd service** - Would restart -3. **audit log** - Would add remediation entry - -## Dry-Run Output: -
-Click to expand full output - -[Full stdout from jobs_stdout_retrieve] - -
- -✓ No errors detected in dry-run -✓ All systems passed pre-flight checks -``` - -## Proceed to Actual Execution Prompt - -``` -❓ Dry-run completed successfully. Proceed with actual execution? - -Options: -- "yes" or "execute" - Proceed with actual remediation -- "review" - Show dry-run output again -- "abort" - Cancel execution - -Please respond with your choice. -``` diff --git a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/05-git-flow-prompts.md b/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/05-git-flow-prompts.md deleted file mode 100644 index 41945d0e..00000000 --- a/evaluation/with_skills/rh-sre__playbook-executor/environment/skills/playbook-executor/references/05-git-flow-prompts.md +++ /dev/null @@ -1,97 +0,0 @@ -# Step 05: Git Flow Prompts - -Read this reference when executing Git Flow (Scenario 1 Override or Scenario 2). - -## Scenario 1 Prompt (Same path) - -The template already points to our playbook path. The project may need the latest content. - -``` -Found template [name] (ID: X) with matching playbook path. The project may need to be updated with the latest playbook. - -Options: -(A) Override: I'll add/update the playbook in the project via git. You sync the AAP project, then confirm. -(B) Manual: You add the playbook and sync. Confirm when done. - -❓ Choose (A) or (B): -``` - -- **If A**: Execute Git Flow (see Git Flow section below). Wait for user: "Sync complete" or "done". -- **If B**: Wait for user confirmation. - -## Scenario 2 Prompt (Different path) - -**CRITICAL**: The template points to a DIFFERENT playbook than our generated playbook. You MUST NOT launch the job without Git Flow. AAP executes from synced project content—there is no "override at launch". The playbook MUST be in the repo before any job launch. - -``` -Found template [name] (ID: X) pointing to [template.playbook]. Our generated playbook is [our_playbook_path]. - -⚠️ The template's playbook path does NOT match. We must update the playbook in the project before execution. - -Options: -- "yes" or "proceed" - I'll add our playbook to the project via git (you'll confirm commit/push, then sync AAP) -- "no" - Create a new template via `/job-template-creator` skill - -❓ Proceed with playbook update (git flow)? -``` - -- **If yes**: Execute Git Flow. **Do NOT proceed to Phase 3 until Git Flow completes.** -- **If no**: Fall through to Scenario 3 (job-template-creator). - -## Repo Path Question - -``` -What is the local path to the Git repository for project [Project Name] (scm_url)? -``` - -Use `projects_list` to get project name and `scm_url`; display to help user identify the repo. - -**Path format**: Ask for the **absolute path** (e.g. `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project`). When writing the playbook, the Write tool path MUST be `/playbooks/remediation/` — the full absolute path. Do NOT use a relative path like `test-aap-project/playbooks/...`; that causes "Error writing file". - -## Git Flow: Write Step (FAST) - -**CRITICAL**: The playbook is ALREADY generated. During Git Flow you must WRITE the existing content to disk—nothing more. - -- **DO**: Single file write of the playbook content already in context (from playbook-generator or remediation) -- **DO NOT**: Invoke playbook-generator again, call create_vulnerability_playbook, re-fetch from MCP, or validate/transform the content -- **Expected duration**: Seconds. If it takes minutes, you are doing unnecessary work. - -### Write Path (ABSOLUTE REQUIRED) - -**⚠️ WRITE PATH MUST START WITH `/`** — The Write tool path MUST be an absolute path. Relative paths cause "Error writing file" because the repo is often outside the workspace. - -**Formula**: `write_path = user_provided_path + "/" + target_path` - -- `user_provided_path` = exactly what the user typed (e.g. `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project`) -- `target_path` = e.g. `playbooks/remediation/cve-remediation.yml` - -**Correct**: `/Users/dmartino/projects/AI/ai5/ai5-demo/test-aap-project/playbooks/remediation/cve-remediation.yml` - -**WRONG** (will fail): -- `test-aap-project/playbooks/remediation/cve-remediation.yml` -- `playbooks/remediation/cve-remediation.yml` - -**Before calling Write**: Verify the path starts with `/`. If it does not, prepend the user's repo path. - -## Git Flow HITL Checkpoint - -**REQUIRED** before commit/push: - -``` -Ready to commit and push these changes? -- File: [target_path] -- CVE: [cve_id] -- This will update the playbook in the AAP project. - -Reply 'yes' or 'proceed' to continue, or 'abort' to cancel. -``` - -**Wait for user confirmation.** If "yes" or "proceed": `git commit -m "Add/update remediation playbook for CVE-YYYY-NNNNN"` then `git push origin main`. - -## After Push Message - -``` -I've pushed the playbook. Sync the AAP project: Automation Execution > Projects > [Project] > Sync. Reply 'sync complete' when done. -``` - -**Do NOT proceed to Phase 3 (Dry-Run) until user confirms sync complete.** diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 1092dd1d..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- pass rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__playbook-generator/environment/skills/playbook-generator/SKILL.md b/evaluation/with_skills/rh-sre__playbook-generator/environment/skills/playbook-generator/SKILL.md deleted file mode 100644 index a9234cdd..00000000 --- a/evaluation/with_skills/rh-sre__playbook-generator/environment/skills/playbook-generator/SKILL.md +++ /dev/null @@ -1,377 +0,0 @@ ---- -name: playbook-generator -description: | - **CRITICAL**: This skill ONLY GENERATES playbooks. It does NOT EXECUTE them. For execution, use /playbook-executor skill. - - Generate production-ready Ansible remediation playbooks for CVE vulnerabilities with Red Hat best practices, error handling, and Kubernetes safety patterns. Use this skill when you need to create remediation playbooks that follow Red Hat Lightspeed patterns and incorporate RHEL-specific considerations. - - This skill calls the MCP tool (remediations__create_vuln_playbook) and returns the playbook **AS IS**. Do NOT modify, enhance, or add to the generated playbook. Any change requires explicit user validation first. - - **IMPORTANT**: - - ALWAYS use this skill instead of calling create_vulnerability_playbook directly - - NEVER execute playbooks using ansible-playbook CLI - - ALWAYS delegate execution to /playbook-executor skill ---- - -# Ansible Playbook Generator Skill - -This skill generates Ansible remediation playbooks for CVE vulnerabilities, applying Red Hat best practices, RHEL-specific patterns, and Kubernetes safety considerations. - -**Integration with Remediation Skill**: The `/remediation` skill orchestrates this skill as part of its Step 4 (Generate Playbook) workflow. For standalone playbook generation, you can invoke this skill directly. - -## When to Use This Skill - -**🚨 CRITICAL SCOPE LIMITATION**: This skill **ONLY GENERATES** playbooks. It does **NOT EXECUTE** them. - -**Use this skill directly when you need**: -- Generate a remediation playbook for a specific CVE -- Create batch remediation playbooks for multiple CVEs -- Get a remediation playbook from Red Hat Lightspeed (returned unmodified) -- Standalone playbook generation without full remediation workflow - -**Do NOT use this skill when you need**: -- "Create playbook and execute it" → Use `/remediation` skill (orchestrates this skill + playbook-executor) -- "Remediate CVE-X" (full workflow) → Use `/remediation` skill -- Execute playbooks → Use `/playbook-executor` skill instead -- Run ansible-playbook CLI → Use `/playbook-executor` skill via AAP MCP -- Monitor job execution → Use `/playbook-executor` skill instead - -**Use the `/remediation` skill when you need**: -- End-to-end CVE remediation (analysis → validation → playbook → execution → verification) -- Integrated impact analysis before playbook generation -- System context gathering and remediation strategy determination -- Execution guidance and verification workflows - -**How they work together**: -1. The `/remediation` skill orchestrates this skill after gathering system context -2. This skill generates the optimized playbook -3. The remediation skill then invokes `/playbook-executor` for execution via AAP MCP -4. Finally, `/remediation-verifier` confirms success - -## Workflow - -**🚨 Tool Failure Rule**: If `create_vulnerability_playbook` (or `create_vuln_playbook`) fails, STOP. Present options (retry / generate from knowledge with user confirmation / exit). Never auto-generate from your knowledge. - -### 1. Playbook Generation (MCP Tool) - -**MCP Tool**: `create_vulnerability_playbook` or `remediations__create_vulnerability_playbook` (from lightspeed-mcp) - -**Parameters** (tool may use `cve_ids`/`cves` and `system_ids`/`uuids`—check tool schema): -- `cves` or `cve_ids`: Array of CVE identifiers - - Example: `["CVE-2024-1234"]` - - Format: CVE-YYYY-NNNNN strings -- `uuids` or `system_ids`: Array of system UUIDs from Red Hat Lightspeed inventory - - Example: `["uuid-1", "uuid-2"]` - - Format: UUID strings (get from system-context skill) -- `playbook_name`: Name for the playbook (if required by tool) - -**Expected Output**: Ansible playbook YAML from Red Hat Lightspeed. - -**CRITICAL — Return AS IS**: You MUST return the playbook exactly as the MCP tool provides it. Do NOT add pre-flight checks, backups, service restarts, audit logging, or any other modifications. The MCP tool description states: "Don't process the playbook. You MUST return the YAML as is." Any enhancement requires explicit user approval—offer modifications only after user requests them. - -#### When MCP Tool Fails (REQUIRED Error Handling) - -**🚨 CRITICAL**: When `create_vulnerability_playbook` (or `remediations__create_vulnerability_playbook`) returns an error, you MUST NOT generate a playbook from your own knowledge. Stop and present options to the user. - -**If the tool returns error** (e.g., "Unhandled error", timeout, 500, connection failure): - -1. **Report the failure** to the user with the error message -2. **Present these options** and wait for explicit user choice: - -``` -❌ Red Hat Lightspeed playbook generation failed: [error message] - -**Next steps** (choose one): - -(A) **Retry** - Try the MCP tool again (may succeed if transient) -(B) **Generate from knowledge** - Create a playbook using documentation templates (⚠️ NOT from Red Hat Lightspeed; requires your explicit approval) -(C) **Exit** - Stop playbook generation; user can retry later or use manual remediation - -❓ Reply with A, B, or C: -``` - -3. **Execute based on user choice**: - - **A (Retry)**: Call the MCP tool again. If it fails again, present options again (limit retries to 2; after 2 failures, present B and C only) - - **B (Generate from knowledge)**: ONLY proceed if user explicitly chose B. Use documentation (cve-remediation-templates.md, package-management.md) to build a playbook. Add disclaimer: "Generated from documentation templates—Red Hat Lightspeed API was unavailable. Review carefully before execution." - - **C (Exit)**: Stop. Do not generate any playbook. Suggest: "You can retry later when Lightspeed MCP is available, or create a manual remediation playbook." - -**NEVER** auto-generate a playbook from your knowledge when the tool fails without explicit user confirmation for option B. - -### 5. Return Playbook AS IS (No Modifications) - -**CRITICAL**: Return the playbook exactly as the MCP tool provides it. Do NOT add, remove, or modify any content. - -**Do NOT**: -- Add pre-flight checks (RHEL validation, subscription check) -- Add backup/snapshot creation -- Add service restart logic -- Add audit logging -- Add Kubernetes pod eviction -- Replace or wrap the MCP output with documentation templates - -**If the user requests enhancements** (e.g. "add pre-flight checks", "add backup step"): -1. Show the original playbook first -2. Ask: "The playbook above is from Red Hat Lightspeed. You requested [enhancement]. Should I create a modified version with these additions? (yes/no)" -3. Only if user confirms "yes", create a modified version and show the diff -4. Require explicit approval before any modified playbook is used - -### 6. Playbook Validation (Minimal) - -Before returning, verify only: -- YAML is returned (the MCP tool output) -- No modifications were applied - -Do NOT validate for "best practices" or add missing elements—return AS IS. - -## Critical: Human-in-the-Loop Requirements - -This skill generates code that will execute on production systems. **Explicit user confirmation is REQUIRED** before returning the playbook. - -**When MCP Tool Fails** (REQUIRED): -- Do NOT generate a playbook from your own knowledge without explicit user confirmation -- Present options: (A) Retry, (B) Generate from knowledge (requires user approval), (C) Exit -- Wait for user to choose A, B, or C before proceeding -- If user chooses B: Add disclaimer that playbook was generated from documentation, not Red Hat Lightspeed - -**Before Playbook Return** (REQUIRED): -1. **Display Playbook Preview**: Show complete playbook YAML to user -2. **Display Metadata**: Show CVE IDs, target systems, reboot requirements, Kubernetes considerations -3. **Ask for Confirmation**: - ``` - ❓ Review the playbook above. This playbook will: - - Update packages on N systems - - Require reboot: [Yes/No] - - Affect Kubernetes pods: [Yes/No] - - Should I provide this playbook for execution? - - Options: - - "yes" or "proceed" - Provide playbook for execution - - "modify" - Request changes to playbook - - "abort" - Cancel playbook generation - - Please respond with your choice. - ``` -4. **Wait for Explicit Confirmation**: Do not provide playbook without "yes" or "proceed" - -**Never assume approval** - always wait for explicit user confirmation before providing executable playbooks. - -### 7. Return Playbook - -**🚨 CRITICAL**: This skill **ONLY GENERATES** playbooks. It does **NOT EXECUTE** them. - -**ONLY after receiving explicit user confirmation**, return the production-ready playbook with metadata: - -```yaml -# Playbook metadata to return: -playbook: - file: remediation-CVE-YYYY-NNNNN.yml - path: playbooks/remediation/remediation-CVE-YYYY-NNNNN.yml # Full path for playbook-executor template matching - content: | - [Complete YAML playbook] - - metadata: - cve_ids: ["CVE-YYYY-NNNNN"] - target_systems: ["uuid-1", "uuid-2"] - rhel_versions_supported: ["7", "8", "9"] - requires_reboot: true/false - kubernetes_safe: true/false - estimated_duration_minutes: 15 - risk_level: "medium" # based on reboot requirement - - execution_notes: - - "Test in staging environment first" - - "Schedule maintenance window if reboot required" - - "Ensure kubectl access if Kubernetes systems" - - "Back up critical data before execution" -``` - -## Critical: Execution Handoff - -**🚨 THIS SKILL DOES NOT EXECUTE PLAYBOOKS** - -After generating the playbook, if the user requests execution: - -❌ **WRONG** - Do NOT use `ansible-playbook` CLI: -```bash -ansible-playbook remediation.yml --check # ❌ This skill cannot do this -``` - -✅ **CORRECT** - Delegate to the `/playbook-executor` skill: -```markdown -I've generated the remediation playbook. To execute it in dry-run mode, I'll invoke the playbook-executor skill: - -[Invoke /playbook-executor skill with the playbook content] -``` - -**When user asks to execute**: -1. Save the playbook to a file (if needed for reference) -2. Invoke `/playbook-executor` skill with instruction: - ``` - "Execute this playbook for CVE-XXXX-YYYY in dry-run mode using AAP job template [ID]. Monitor job status and report results." - ``` -3. The playbook-executor skill handles all execution via AAP MCP tools - -**Never attempt to**: -- Run `ansible-playbook` command directly -- Execute playbooks via Shell/Bash tool -- Use any local Ansible execution method - -**Always delegate execution to** `/playbook-executor` skill. - -## Output Template - -When completing playbook generation, provide output in this format: - -```markdown -# Remediation Playbook Generated - -## CVE Information -**CVE ID**: CVE-YYYY-NNNNN -**Target Systems**: N systems -**RHEL Versions**: 7, 8, 9 -**Requires Reboot**: Yes/No -**Kubernetes Safe**: Yes/No - -## Playbook Features -✓ Generated by Red Hat Lightspeed (returned AS IS, no modifications) - -## Playbook File: remediation-CVE-YYYY-NNNNN.yml - -```yaml -[Complete playbook YAML] -``` - -## Next Steps: Execution - -**🔴 IMPORTANT**: Do NOT execute this playbook using `ansible-playbook` CLI. - -**✅ To execute this playbook**, invoke the `/playbook-executor` skill: - -```markdown -Ready to execute? The playbook-executor skill will: -1. Add this playbook to your AAP Git project -2. Create/use an AAP job template -3. Execute in dry-run mode first (if requested) -4. Launch actual execution (with your approval) -5. Monitor job status and report results - -Would you like me to invoke the playbook-executor skill now? -Options: -- "yes" or "execute" - Invoke playbook-executor skill -- "dry-run first" - Execute in check mode first -- "save only" - Just save the playbook file for later -``` - -**Execution Flow**: -1. **This skill** → Generates playbook (DONE ✓) -2. **playbook-executor skill** → Executes via AAP MCP tools -3. **remediation-verifier skill** → Verifies success after execution - -**Safety Notes**: -- Playbook is from Red Hat Lightspeed—review before execution -- No modifications were applied; user may request enhancements separately -``` - -## Examples - -### Example 1: Simple CVE - -**User Request**: "Generate playbook for CVE-2024-1234 on 5 RHEL 8 systems" - -**Skill Response**: -1. Call `remediations__create_vuln_playbook` with cves, uuids, playbook_name -2. Return the playbook **exactly as received**—no modifications -3. Ask for user confirmation before handoff to playbook-executor - -### Example 2: Batch CVEs - -**User Request**: "Generate playbook for CVE-2024-1234, CVE-2024-5678 on 20 systems" - -**Skill Response**: -1. Call `remediations__create_vuln_playbook` with multiple CVE IDs and system UUIDs -2. Return the playbook **exactly as received**—no modifications -3. Ask for user confirmation before handoff to playbook-executor - -## Error Handling - -**CVE has no automated remediation**: -``` -CVE-YYYY-NNNNN does not have an automated remediation playbook available in Red Hat Lightspeed. - -Manual remediation required: -1. Affected packages: package-name-version -2. Recommended action: dnf update package-name -3. Verification: package-name --version - -Would you like me to create a manual playbook template based on Red Hat best practices? -``` - -**Unsupported RHEL version**: -``` -Target systems include RHEL 6, which is not supported by this skill. - -Supported RHEL versions: 7, 8, 9 - -Please filter target systems to supported versions or consult Red Hat documentation for RHEL 6 remediation guidance. -``` - -**Kubernetes context missing**: -``` -Target systems appear to be Kubernetes nodes but kubectl access is not configured. - -To generate Kubernetes-safe playbooks, ensure: -1. kubectl is installed and configured -2. Access to cluster is available -3. Appropriate RBAC permissions for node operations - -Proceeding with standard playbook (without pod eviction). Add pod eviction manually if needed. -``` - -## Dependencies - -### Required MCP Servers -- `lightspeed-mcp` - Red Hat Lightspeed platform access - -### Required MCP Tools -- `remediations__create_vuln_playbook` (from lightspeed-mcp) - Generate remediation playbook from Red Hat Lightspeed - - Parameters: playbook_name, cves (array), uuids (array of system UUIDs) - - Returns: Ansible playbook YAML—**return AS IS**, do not modify - -### Related Skills -- `cve-impact` - Provides CVE severity and risk assessment to inform playbook complexity -- `system-context` - Provides system inventory and deployment context for playbook targeting -- `remediation-verifier` - Verifies playbook execution success after deployment -- `playbook-executor` - Executes generated playbooks and tracks job status - -### Reference Documentation -- [cve-remediation-templates.md](../../docs/ansible/cve-remediation-templates.md) - Ansible playbook templates for different CVE types -- [package-management.md](../../docs/rhel/package-management.md) - RHEL package management best practices (DNF vs YUM, reboot detection) - -## Best Practices - -1. **🚨 NEVER EXECUTE PLAYBOOKS** - This skill generates only. Always delegate execution to `/playbook-executor` skill -2. **🚨 RETURN AS IS** - Do NOT modify the MCP-generated playbook. No enhancements without explicit user request and approval -3. **🚨 NEVER auto-generate on tool failure** - When the MCP tool fails, present options (retry / generate from knowledge with user confirmation / exit). Do NOT silently generate from your own knowledge -4. **Require user approval** - ALWAYS get explicit confirmation before providing playbooks for execution -5. **Clear handoff** - After generation, explicitly tell user to invoke `/playbook-executor` for execution - -## Tools Reference - -This skill uses: -- `remediations__create_vuln_playbook` (from lightspeed-mcp) - Generate playbook from Red Hat Lightspeed. Returns YAML **as is**—do not modify. - -All MCP tools are provided by the lightspeed-mcp server configured in `.mcp.json`. - -## Integration with Other Skills - -- **cve-impact**: Provides CVE severity and risk assessment to inform playbook complexity -- **system-context**: Provides system inventory and deployment context for playbook targeting -- **remediation-verifier**: Verifies playbook execution success after deployment - -**Orchestration Example** (from `/remediation` skill): -1. Agent invokes cve-impact skill → Gets risk assessment -2. Agent gathers context → Determines deployment requirements -3. Agent invokes playbook-generator skill → Generates production-ready playbook -4. Agent provides execution guidance → User deploys playbook -5. Agent invokes remediation-verifier skill → Confirms success diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 1092dd1d..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- pass rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__remediation-verifier/environment/skills/remediation-verifier/SKILL.md b/evaluation/with_skills/rh-sre__remediation-verifier/environment/skills/remediation-verifier/SKILL.md deleted file mode 100644 index e393f5ba..00000000 --- a/evaluation/with_skills/rh-sre__remediation-verifier/environment/skills/remediation-verifier/SKILL.md +++ /dev/null @@ -1,399 +0,0 @@ ---- -name: remediation-verifier -description: | - **CRITICAL**: This skill must be used for remediation verification. DO NOT use raw MCP tools like get_cve or get_host_details directly for verification. - - Verify CVE remediation success by checking Red Hat Lightspeed CVE status, validating package versions, and confirming service health. Use this skill after executing remediation playbooks to ensure vulnerabilities are properly fixed. - - This skill orchestrates MCP tools (get_cve, get_cve_systems, get_host_details) to provide comprehensive remediation verification including CVE status checking, package version validation, and service health confirmation. - - **IMPORTANT**: ALWAYS use this skill instead of calling verification MCP tools directly. ---- - -# Remediation Verification Skill - -This skill verifies CVE remediation success by validating that vulnerabilities have been properly fixed on target systems. - -**Integration with Remediation Skill**: The `/remediation` skill orchestrates this skill as part of its Step 6 (Verify Deployment) workflow. For standalone verification after manual remediation, you can invoke this skill directly. - -## When to Use This Skill - -**Use this skill directly when you need**: -- Verify CVE remediation after playbook execution -- Confirm package updates were applied successfully -- Check service health after remediation -- Validate Kubernetes pod recovery after node updates -- Generate verification reports for compliance - -**Use the `/remediation` skill when you need**: -- Full remediation workflow including verification -- Integrated remediation → execution → verification - -**How they work together**: The `/remediation` skill invokes this skill after the user executes the remediation playbook, providing final confirmation that the CVE is resolved. - -## Workflow - -### 1. CVE Status Verification - -**MCP Tool**: `get_cve` or `vulnerability__get_cve` (from lightspeed-mcp) - -**Parameters for get_cve**: -- `cve_id`: Exact CVE identifier (format: `"CVE-YYYY-NNNNN"`) - - Example: `"CVE-2024-1234"` -- `include_details`: `true` (retrieve complete metadata including remediation status) - -**Expected After Remediation**: -- CVE metadata still exists (CVE doesn't disappear from database) -- Remediation marked as applied -- Fixed version recorded - -**MCP Tool**: `get_cve_systems` or `vulnerability__get_cve_systems` (from lightspeed-mcp) - -**Parameters for get_cve_systems**: -- `cve_id`: Exact CVE identifier (format: `"CVE-YYYY-NNNNN"`) - - Example: `"CVE-2024-1234"` -- `limit`: Optional number of systems to return (default: all) - - Example: `100` -- `offset`: Optional pagination offset (default: 0) - - Example: `0` - -**Expected After Remediation**: -- Target systems removed from affected list OR -- Systems marked as "patched" status OR -- Systems show fixed package version - -**Verification Logic**: -``` -✓ System UUID not in affected systems list → PASS -✓ System status = "patched" → PASS -✗ System still in affected list with "vulnerable" status → FAIL -``` - -**Important**: Red Hat Lightspeed inventory updates may take time (up to 24 hours after remediation). Consider this timing when interpreting results. - -### 2. Package Version Verification - -**MCP Tool**: `get_host_details` or `inventory__get_host_details` (from lightspeed-mcp) - -**Parameters**: -- `system_id`: UUID of the system to verify (from Red Hat Lightspeed inventory) - - Example: `"uuid-1"` - - Format: UUID string (get from system-context skill or get_cve_systems result) -- `include_system_profile`: `true` (retrieve installed packages and service status) - - Example: `true` - -**Expected Output**: System details including: -- `system_profile.installed_packages` - List of installed RPM packages with versions -- `system_profile.enabled_services` - Services enabled at boot -- `system_profile.running_processes` - Currently running processes - -**Verification Workflow**: -``` -For each target system: - -1. Get current installed packages - Tool: get_host_details(system_id="uuid-1", include_system_profile=true) - Extract: system_profile.installed_packages - -2. Compare against expected fixed versions - CVE Fix Example: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 - - Installed Packages Check: - ✓ httpd-2.4.37-2.el8 (or newer) installed → PASS - ✗ httpd-2.4.37-1.el8 (old version) still present → FAIL - ✗ httpd not found → FAIL (package removed unexpectedly) - -3. Handle version comparison edge cases - - Epoch numbers (e.g., 1:httpd-2.4.37) - - Release suffixes (e.g., 2.4.37-2.el8_9.1) - - Architecture (x86_64, aarch64) -``` - -**Package Version Comparison Logic**: -```python -def verify_package_version(installed, expected_fixed): - """ - installed: "httpd-2.4.37-2.el8.x86_64" - expected_fixed: "httpd-2.4.37-2.el8" - - Returns: True if installed >= expected_fixed - """ - # Parse version components using RPM version comparison - # Account for epoch, version, release - # Use >= comparison (newer versions are acceptable) -``` - -### 3. Service Health Verification - -**MCP Tool**: `get_host_details` or `inventory__get_host_details` (from lightspeed-mcp) - -**Parameters**: Same as Step 2 - system_id with include_system_profile=true - -Verify affected services are running properly: - -``` -For each affected service (e.g., httpd): - -1. Check service status - Extract from: system_profile.enabled_services - Extract from: system_profile.running_processes - - Service Health Checks: - ✓ Service in enabled_services list → Service will start on boot - ✓ Service process in running_processes → Service currently running - ✗ Service not enabled → WARN (service won't start on reboot) - ✗ Service process not running → FAIL (service down) - -2. Check for service errors - Look for: system_profile.systemd_failed_units - ✓ Service not in failed units → PASS - ✗ Service in failed units → FAIL (service failed to start) - -3. Verify uptime (if applicable) - Check: Service started recently (after remediation) - ✓ Service uptime < remediation time + 10 minutes → Service restarted - ⚠ Service uptime > remediation time → Service may not have restarted -``` - -### 4. Remediation Summary Generation - -Generate comprehensive verification report: - -```json -{ - "verification_status": "success", # or "partial_success", "failed" - - "cve_id": "CVE-2024-1234", - "verification_date": "2024-01-20T15:30:00Z", - - "systems_remediated": { - "total": 10, - "successful": 10, - "failed": 0, - "partial": 0 - }, - - "verification_details": [ - { - "system_id": "uuid-1", - "hostname": "web-server-01", - "status": "verified", - - "checks": { - "cve_status": { - "result": "pass", - "details": "System removed from affected systems list" - }, - "package_version": { - "result": "pass", - "expected": "httpd-2.4.37-2.el8", - "installed": "httpd-2.4.37-2.el8", - "details": "Package updated successfully" - }, - "service_health": { - "result": "pass", - "service": "httpd", - "status": "running", - "details": "Service restarted and healthy" - } - } - } - ], - - "compliance": { - "all_systems_patched": true, - "services_healthy": true - }, - - "recommendations": [ - "Remediation verified successfully on all systems", - "Consider re-scanning with insights-client for updated inventory", - "Document remediation in change management system" - ] -} -``` - -### 6. Handle Verification Failures - -If verification fails, provide troubleshooting guidance: - -**Package Version Mismatch**: -``` -Verification Failed: Package Not Updated - -System: web-server-01 (uuid-1) -Expected: httpd-2.4.37-2.el8 -Found: httpd-2.4.37-1.el8 (OLD VERSION) - -Possible causes: -1. Playbook execution failed (check Ansible output) -2. Package repository doesn't have fixed version -3. Package update was skipped due to dependency conflict - -Troubleshooting: -1. Check Ansible playbook output for errors -2. Verify package availability: - sudo dnf info httpd-2.4.37-2.el8 -3. Manually update package: - sudo dnf update httpd -4. Check for package holds: - sudo dnf versionlock list -``` - -**Service Not Running**: -``` -Verification Failed: Service Not Running - -System: web-server-01 (uuid-1) -Service: httpd -Status: Failed - -Troubleshooting: -1. Check service status: - sudo systemctl status httpd -2. View service logs: - sudo journalctl -u httpd --since "10 minutes ago" -3. Check for configuration errors: - sudo httpd -t -4. Restart service manually: - sudo systemctl restart httpd -``` - -## Output Template - -When completing verification, provide output in this format: - -```markdown -# Remediation Verification Report - -## CVE: CVE-YYYY-NNNNN -**Verification Date**: 2024-01-20 15:30 UTC -**Overall Status**: ✓ SUCCESS - -## Summary -**Total Systems**: 10 -**Successfully Remediated**: 10 -**Failed**: 0 -**Partial Success**: 0 - -## Verification Results - -### System: web-server-01 (uuid-1) -**Status**: ✓ VERIFIED - -**Checks Performed**: -✓ CVE Status: System removed from affected list -✓ Package Version: httpd-2.4.37-2.el8 (updated from 2.4.37-1.el8) -✓ Service Health: httpd running and healthy - ---- - -### System: web-server-02 (uuid-2) -**Status**: ✓ VERIFIED - -**Checks Performed**: -✓ CVE Status: System marked as patched -✓ Package Version: httpd-2.4.37-2.el8 installed -✓ Service Health: httpd running - ---- - -[Additional systems...] - -## Compliance Status -✓ All systems successfully patched -✓ All services running and healthy - -## Recommendations -1. Remediation verified successfully on all 10 systems -2. Re-scan systems with Red Hat Lightspeed for updated inventory: - ```bash - sudo insights-client --check-results - ``` -3. Document remediation in change management system -4. Consider scheduling next vulnerability scan in 7 days - -## Next Steps -- Remediation complete, no further action required -- Monitor systems for 24 hours to ensure stability -- Update vulnerability tracking system -``` - -## Examples - -### Example 1: Successful Verification - -**User Request**: "Verify remediation for CVE-2024-1234 on 5 systems" - -**Skill Response**: -1. Call `get_cve_systems` → 0 systems affected (down from 5) -2. Call `get_host_details` for each → All have httpd-2.4.37-2.el8 -3. Check service status → All httpd services running -4. Return: "✓ All 5 systems verified, CVE remediated successfully" - -### Example 2: Partial Success - -**User Request**: "Verify batch remediation on 20 systems" - -**Skill Response**: -1. Call `get_cve_systems` → 2 systems still affected (18 fixed) -2. Call `get_host_details` → 2 systems have old package version -3. Identify failed systems: web-server-18, web-server-19 -4. Return: "⚠ 18/20 systems verified. 2 systems failed package update. Troubleshooting guidance provided." - -## Dependencies - -### Required MCP Servers -- `lightspeed-mcp` - Red Hat Lightspeed platform access - -### Required MCP Tools -- `get_cve` or `vulnerability__get_cve` (from lightspeed-mcp) - Get CVE metadata and remediation status - - Parameters: cve_id (string, format CVE-YYYY-NNNNN), include_details (boolean) - - Returns: CVE metadata including remediation status -- `get_cve_systems` or `vulnerability__get_cve_systems` (from lightspeed-mcp) - List systems affected by CVE - - Parameters: cve_id (string), limit (number), offset (number) - - Returns: List of systems with vulnerability status -- `get_host_details` or `inventory__get_host_details` (from lightspeed-mcp) - Get system details including packages and services - - Parameters: system_id (UUID string), include_system_profile (boolean) - - Returns: System profile with installed_packages, enabled_services, running_processes - -### Related Skills -- `playbook-generator` - Generates playbooks that this skill verifies -- `system-context` - Provides system context for verification scope -- `cve-impact` - Initial impact assessment to compare against verification results -- `playbook-executor` - Executes playbooks that this skill verifies - -### Reference Documentation -- None required (verification skill uses MCP tool data) - -## Best Practices - -1. **Wait before verification** - Allow 5-10 minutes after playbook execution for system updates to register -2. **Check multiple indicators** - CVE status + package version + service health (defense in depth) -3. **Re-scan with Lightspeed** - Recommend `insights-client --check-results` to update inventory -4. **Document failures** - Provide detailed troubleshooting for any verification failures -5. **Consider timing** - Lightspeed inventory updates may take up to 24 hours to propagate -6. **Verify at scale** - Use batch verification for large deployments (call get_host_details in parallel) - -## Integration with Other Skills - -- **playbook-generator**: Generates playbooks that this skill verifies -- **system-context**: Provides system context for verification scope -- **cve-impact**: Initial impact assessment to compare against verification results - -**Orchestration Example** (from `/remediation` skill): -1. User requests CVE remediation -2. Agent invokes playbook-generator → Creates playbook -3. User executes playbook manually -4. Agent invokes remediation-verifier skill → Confirms success -5. Agent reports: "✓ CVE remediated and verified on all systems" - -**Verification-First Principle**: -``` -Never assume remediation worked. Always verify: -1. CVE status in Lightspeed -2. Package versions updated -3. Services running - -Trust, but verify. -``` diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__remediation/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__remediation/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 1092dd1d..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- pass rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/SKILL.md b/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/SKILL.md deleted file mode 100644 index fde713b0..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/SKILL.md +++ /dev/null @@ -1,340 +0,0 @@ ---- -name: cve-validation -description: | - **CRITICAL**: This skill must be used for CVE validation queries. DO NOT use raw MCP tools like get_cve directly. - - Validate CVE identifiers and check remediation availability in Red Hat Lightspeed. Use this skill when you need to verify a CVE exists, check its severity, and confirm automated remediation is available before proceeding with remediation planning. - - **DO NOT use this skill when** user requests full remediation - use `/remediation` skill instead: - - "Create a remediation playbook for CVE-X" → `/remediation` skill - - "Create playbook and execute it" → `/remediation` skill - - "Remediate CVE-X" / "Patch CVE-X" → `/remediation` skill - - This skill orchestrates MCP tools (get_cve) to provide comprehensive CVE validation. The `/remediation` skill invokes this skill as Step 2 of its workflow. ---- - -# CVE Validation Skill - -This skill validates CVE identifiers and checks remediation availability in Red Hat Lightspeed, ensuring CVEs are valid and remediable before investing effort in remediation planning. - -**Integration with Remediation Skill**: The `/remediation` skill orchestrates this skill as part of its Step 2 (Validate CVE) workflow. For standalone CVE validation, you can invoke this skill directly. - -## Invocation Note (Host-Specific) - -When invoked by another skill (e.g. remediation), use the Skill tool—do NOT use "Task Output" with the skill name as task ID. That causes "No task found with ID: cve-validation". See [skill-invocation.md](../../../docs/references/skill-invocation.md). - -## Prerequisites - -**Required MCP Servers**: `lightspeed-mcp` ([setup guide](https://console.redhat.com/)) - -**Required MCP Tools**: -- `get_cve` (from lightspeed-mcp) - Get CVE metadata and validation - -**Required Environment Variables**: -- `LIGHTSPEED_CLIENT_ID` - Red Hat Lightspeed service account client ID -- `LIGHTSPEED_CLIENT_SECRET` - Red Hat Lightspeed service account secret - -### Prerequisite Validation - -**CRITICAL**: Before executing any operations, execute the `/mcp-lightspeed-validator` skill to verify MCP server availability. - -**Validation freshness**: Can skip if already validated in this session. See [Validation Freshness Policy](../mcp-lightspeed-validator/SKILL.md#validation-freshness-policy). - -**How to invoke**: Execute the `/mcp-lightspeed-validator` skill - -**Handle validation result**: -- **If validation PASSED**: Continue with CVE validation -- **If validation PARTIAL**: Warn user and ask to proceed -- **If validation FAILED**: Stop execution, provide setup instructions - -## When to Use This Skill - -**Use this skill directly when you need**: -- Quick validation of CVE identifier format and existence (standalone query) -- Check if automated remediation is available -- Verify CVE metadata before analysis -- Validate CVE lists for batch operations - -**DO NOT use this skill when** - use `/remediation` skill instead: -- User says "create a remediation playbook" or "remediate CVE-X" or "patch CVE-X" -- User says "create playbook and execute it" - agent orchestrates full workflow -- Any request that implies playbook generation or execution - -**Use the `/remediation` skill when you need**: -- Full remediation workflow (validation + analysis + playbook + execution) -- Integrated CVE validation as part of remediation planning - -**How they work together**: The `/remediation` skill invokes this skill early in the workflow to fail fast if a CVE is invalid or has no automated remediation, saving time and effort. - -**When invoked by remediation**: Return remediatable status prominently so the orchestrator can gate. Include `remediation_status.automated_remediation_available` (boolean) and `validation_status` ("valid" | "not_remediable" | "invalid" | "not_found") in the output. - -## Workflow - -### Step 0: Validate Lightspeed MCP Prerequisites - -**Action**: Execute the `/mcp-lightspeed-validator` skill - -**Note**: Can skip if validation was performed earlier in this session and succeeded. See [Validation Freshness Policy](../mcp-lightspeed-validator/SKILL.md#validation-freshness-policy). - -**How to invoke**: Execute the `/mcp-lightspeed-validator` skill - -**Handle validation result**: -- **If validation PASSED**: Continue to Step 1 -- **If validation PARTIAL**: Warn user and ask to proceed -- **If validation FAILED**: Stop execution, user must set up MCP server - -### Step 1: CVE Format Validation - -Validate CVE identifier format before calling MCP tools. **Format only**—do NOT reject based on year or sequence magnitude. - -```python -CVE Format: CVE-YYYY-NNNNN -Where: -- YYYY = 4-digit year (1999-2030; current and recent years are valid) -- NNNNN = 4-7 digit sequence number (e.g. 1234, 24882, 1234567) - -Valid Examples: -- CVE-2024-1234 -- CVE-2026-24882 # 2026 CVEs exist; 24882 is 5 digits (valid) -- CVE-2023-12345 -- CVE-2021-1234567 - -Invalid Examples (format only): -- CVE-24-1234 (year must be 4 digits) -- CVE-2024-ABC (sequence must be numeric) -- 2024-1234 (missing CVE- prefix) -- CVE-2024-123 (sequence must be 4-7 digits) -``` - -**Quick Regex Check**: -``` -Pattern: ^CVE-\d{4}-\d{4,7}$ - -If invalid format: -→ Return error immediately -→ Suggest format correction -→ Do not proceed to MCP tool calls -``` - -**CRITICAL - Do NOT add extra checks**: If the format matches the regex, you MUST call `get_cve`. Do NOT reject based on: -- "Future" or "current year" assumptions (e.g. "2026 CVE might not exist yet") -- Sequence number magnitude (e.g. "24882 seems high")—5 digits is valid -- Your training data about typical CVE ranges - -Let the API determine existence. A 404 from get_cve means "not found"; format validation only catches malformed IDs. - -### Step 2: CVE Metadata Retrieval - -**CRITICAL**: Document consultation MUST happen BEFORE tool invocation. - -**Document Consultation** (REQUIRED - Execute FIRST): -1. **Action**: Read [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) using the Read tool to understand CVE validation criteria -2. **Action**: Read [references/01-remediation-indicators.md](references/01-remediation-indicators.md) to interpret get_cve response—**CRITICAL** to avoid misinterpreting remediation availability -3. **Output to user**: "I consulted [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) and [01-remediation-indicators.md](references/01-remediation-indicators.md) for CVE validation and remediation indicator interpretation." - -**MCP Tool**: `get_cve` or `vulnerability__get_cve` (from lightspeed-mcp) - -**Do NOT use** `vulnerability__explain_cves` for validation. That tool requires `system_uuid` from inventory; at validation you may not have it. `get_cve` provides remediation availability. Never pass `system_uuid: "undefined"` or placeholders. - -**Parameters**: -- `cve`: Exact CVE identifier from user query (format: `"CVE-YYYY-NNNNN"`) - - Example: `"CVE-2024-1234"` -- `advisory_available`: `"true"` (retrieve CVE with advisory/remediation info) - -**Expected Output**: CVE metadata including CVSS score, severity, affected packages, remediation availability - -Retrieve CVE metadata from Red Hat Lightspeed: - -```json -{ - "cve_id": "CVE-2024-1234", - "cvss_score": 7.5, - "cvss_vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - "severity": "Important", # Red Hat severity rating - "description": "A vulnerability in Apache HTTPD...", - "published_date": "2024-01-15", - "modified_date": "2024-01-20", - - "affected_packages": [ - { - "name": "httpd", - "version": "2.4.37-1.el8", - "fixed_version": "2.4.37-2.el8" - } - ], - - "references": [ - "https://access.redhat.com/errata/RHSA-2024:1234", - "https://nvd.nist.gov/vuln/detail/CVE-2024-1234" - ], - - "cwe": "CWE-400: Uncontrolled Resource Consumption", - - "exploitability": "Proof of concept available", - "remediation_available": true, # KEY FIELD - "reboot_required": false -} -``` - -### Step 3: Validation Checks - -**CRITICAL**: Document consultation MUST happen BEFORE validation logic. - -**Document Consultation** (REQUIRED - Execute FIRST): -1. **Action**: Read [cvss-scoring.md](../../docs/references/cvss-scoring.md) using the Read tool to understand Red Hat severity classification and CVSS score ranges -2. **Output to user**: "I consulted [cvss-scoring.md](../../docs/references/cvss-scoring.md) to understand Red Hat severity classification and CVSS score ranges." - -Perform comprehensive validation: - -**A. Existence Check**: -``` -✓ CVE exists in Red Hat CVE database -✗ CVE not found → Return error with suggestions -``` - -**B. Red Hat Relevance Check**: -``` -✓ CVE affects RHEL systems -✗ CVE is Windows/macOS specific → Not applicable to RHEL -``` - -**C. Severity Classification**: -``` -Red Hat Severity Levels: -- Critical (CVSS 9.0-10.0): Immediate action required -- Important (CVSS 7.0-8.9): Urgent remediation needed -- Moderate (CVSS 4.0-6.9): Plan remediation -- Low (CVSS 0.1-3.9): Low priority -``` - -**D. Remediation Availability Check** (READ [references/01-remediation-indicators.md](references/01-remediation-indicators.md)): -``` -Key Question: Can Red Hat Lightspeed generate an automated playbook? - -✅ USE these fields: - - advisory_available === true → Remediation available - - remediation === 2 → Automated remediation available - - advisories_list non-empty → RHSA exists, remediation available - -❌ DO NOT use rules[] for remediation decision: - - rules: [] (empty) does NOT mean "no remediation" - - Advisor rules are separate from vulnerability remediation - - Remediation comes from Security Advisories (RHSA), not Advisor rules - -✓ If advisory_available=true OR remediation=2 OR advisories_list has entries - → Proceed with automated remediation - → Use create_vulnerability_playbook tool - -✗ If advisory_available=false AND (remediation=0 or advisories_list empty) - → Manual remediation required - → Provide manual steps based on affected packages -``` - -**E. Package Information Validation**: -``` -Check affected_packages array: -✓ Packages identified: httpd-2.4.37-1.el8 -✓ Fixed version available: httpd-2.4.37-2.el8 -✓ Package exists in RHEL repositories - -This information will be used by playbook-generator skill. -``` - -### Step 4: Return Validation Result - -Return structured validation result. **When invoked by remediation skill**: Ensure `validation_status` and `remediation_status.automated_remediation_available` are explicit—the orchestrator gates on these. - -```json -{ - "validation_status": "valid", # or "invalid", "not_found", "not_remediable" - - "cve_metadata": { - "cve_id": "CVE-2024-1234", - "cvss_score": 7.5, - "severity": "Important", - "description": "Brief description...", - "published_date": "2024-01-15" - }, - - "remediation_status": { - "automated_remediation_available": true, - "reboot_required": false, - "affected_packages": [ - { - "name": "httpd", - "current_version": "2.4.37-1.el8", - "fixed_version": "2.4.37-2.el8" - } - ] - }, - - "recommendations": [ - "Automated remediation available via Red Hat Lightspeed", - "No reboot required for this CVE", - "Severity: Important - Urgent remediation recommended", - "Test in staging environment before production deployment" - ], - - "next_steps": [ - "Analyze CVE impact (use cve-impact skill)", - "Gather system context (use system-context skill)", - "Generate remediation playbook (use playbook-generator skill)" - ] -} -``` - -## Output, Examples, Error Handling - -**Read [references/03-output-template.md](references/03-output-template.md)** for report format. -**Read [references/04-examples.md](references/04-examples.md)** for validation examples. -**Read [references/05-error-handling.md](references/05-error-handling.md)** for format, not-found, no-remediation, and API errors. - -## Best Practices - -Validate format first; if regex matches, ALWAYS call get_cve (do not reject on year/sequence). Check remediation availability; fail fast if none. Provide clear next steps and manual guidance when automated unavailable. Link to NVD and Red Hat Security. Cache results to avoid redundant calls. - -## Dependencies - -### Required MCP Servers -- `lightspeed-mcp` - Red Hat Lightspeed platform access - -### Required MCP Tools -- `get_cve` (from lightspeed-mcp) - Get CVE metadata and validation - - Parameters: cve_id (string, format CVE-YYYY-NNNNN), include_details (boolean), validate_format (boolean) - - Returns: CVE metadata with CVSS score, severity, affected packages, remediation availability - -### Related Skills -- `mcp-lightspeed-validator` - **PREREQUISITE** - Validates Lightspeed MCP server before operations - - Use before: ALL cve-validation operations (Step 0 in workflow) - - Purpose: Ensures MCP server is available before attempting tool calls - -- `cve-impact` - Analyze CVE impact after validation - - Use after: Step 4 when CVE is validated and user wants impact analysis - - Purpose: Assess risk and affected systems for validated CVE - -- `system-context` - Get system details after validating CVE affects systems - - Use after: Validation confirms CVE has affected systems - - Purpose: Understand deployment context before remediation - -- `playbook-generator` - Generate remediation playbooks for validated CVEs - - Use after: Validation confirms remediation_available = true - - Purpose: Create automated remediation for valid, remediable CVEs - -### Reference Documentation -- [vulnerability-logic.md](../../docs/insights/vulnerability-logic.md) - CVE validation criteria -- [references/01-remediation-indicators.md](references/01-remediation-indicators.md) - **REQUIRED** - Correct interpretation of get_cve response (advisory_available, remediation, advisories_list). Do NOT use rules[] for remediation decision. -- [cvss-scoring.md](../../docs/references/cvss-scoring.md) - Red Hat severity classification and CVSS score ranges -- [cve-remediation-templates.md](../../docs/ansible/cve-remediation-templates.md) - Manual remediation templates for CVEs without automated playbooks - -## Tools Reference - -This skill uses: -- `get_cve` (vulnerability toolset) - Get CVE metadata and remediation availability from Red Hat Lightspeed - -**Do NOT use** `vulnerability__explain_cves` in this skill—it requires `system_uuid` which may not be available at validation time. Use `get_cve` only. - -All tools are provided by the lightspeed-mcp MCP server configured in `.mcp.json`. - -## Integration with Other Skills - -cve-impact, playbook-generator, system-context, remediation-verifier all depend on validation first. The `/remediation` skill invokes cve-validation as Step 2. Validate → proceed if valid; stop and return error if invalid. diff --git a/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/01-remediation-indicators.md b/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/01-remediation-indicators.md deleted file mode 100644 index 17f9afe8..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/01-remediation-indicators.md +++ /dev/null @@ -1,66 +0,0 @@ -# Remediation Availability Indicators (get_cve Response) - -Read this reference when interpreting `vulnerability__get_cve` or `get_cve` response to determine if automated remediation is available. - -## CRITICAL: Correct vs Incorrect Indicators - -### ✅ USE These Fields for Remediation Availability - -| Field | Meaning | Remediation Available When | -|-------|---------|-----------------------------| -| `advisory_available` | Red Hat Security Advisory exists | `true` | -| `remediation` | Remediation status code | `2` = automated remediation available | -| `advisories_list` | List of RHSA/errata IDs | Non-empty (e.g. `["RHSA-2026:2719"]`) | - -**Decision rule**: Remediation IS available when `advisory_available === true` OR `remediation === 2` OR `advisories_list` is non-empty. - -### ❌ DO NOT Use These Fields for Remediation - -| Field | Meaning | Why NOT to Use | -|-------|---------|----------------| -| `rules` | Red Hat Insights **Advisor** rules | Advisor rules are separate from vulnerability remediation. Empty `rules: []` does NOT mean no remediation. Remediation comes from Security Advisories (RHSA), not Advisor rules. | - -**Common mistake**: Agent sees `rules: []` (empty) and incorrectly concludes "no remediation available." This is WRONG. Always check `advisory_available` and `remediation` first. - -## Interpretation Checklist - -When evaluating `get_cve` response: - -1. **Check `advisory_available`**: If `true` → remediation available ✓ -2. **Check `remediation`**: If `2` → automated remediation available ✓ -3. **Check `advisories_list`**: If non-empty (e.g. RHSA-*) → remediation available ✓ -4. **Ignore `rules`**: Do NOT use for remediation decision. Empty rules ≠ no remediation. - -## Example: Remediation Available (rules empty) - -```json -{ - "advisory_available": true, - "advisories_list": ["RHSA-2026:2719"], - "remediation": 2, - "rules": [] -} -``` - -**Correct interpretation**: Remediation IS available. `rules: []` only means no Advisor rule—remediation comes from RHSA-2026:2719. - -## Example: No Remediation - -```json -{ - "advisory_available": false, - "advisories_list": [], - "remediation": 0, - "rules": [] -} -``` - -**Correct interpretation**: No automated remediation. Manual steps required. - -## get_cve_systems Response (per-system) - -When using `get_cve_systems` for system-level check, each system entry may include: -- `attributes.advisory_available` — same meaning as get_cve -- `attributes.remediation` — same meaning as get_cve - -Use the same interpretation rules. Do NOT use `rules` for remediation decision. diff --git a/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/03-output-template.md b/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/03-output-template.md deleted file mode 100644 index 51bb3992..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/03-output-template.md +++ /dev/null @@ -1,36 +0,0 @@ -# CVE Validation Output Template - -Read when completing CVE validation to format the report. - -```markdown -# CVE Validation Result - -## CVE: CVE-YYYY-NNNNN -**Status**: ✓ Valid - -## CVE Information -**CVSS Score**: X.X (Severity) -**Published**: YYYY-MM-DD -**Description**: [Brief description] - -## Affected Packages -- package-current → package-fixed (fixed) - -## Remediation Status -✓ **Automated Remediation Available** (or ✗ Manual required) -✓ Package updates available -✗ Reboot NOT required - -## Severity Assessment -**Red Hat Severity**: Critical/Important/Moderate/Low -**Priority**: P0/P1/P2 -**Response Time**: [guidance] - -## Recommendations -1. [Automated/manual remediation guidance] -2. Test in staging first -3. Schedule deployment during change window - -## Next Steps -1. cve-impact → system-context → playbook-generator → remediation-verifier -``` diff --git a/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/04-examples.md b/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/04-examples.md deleted file mode 100644 index 2a16ce85..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/04-examples.md +++ /dev/null @@ -1,35 +0,0 @@ -# CVE Validation Examples - -## Example 1: Valid CVE with Automated Remediation - -**Request**: "Validate CVE-2024-1234" -1. Format check → Valid -2. get_cve → found -3. advisory_available/remediation/advisories_list → remediation available (ignore rules[]) -4. Return: "Valid, automated remediation available" - -## Example 2: Valid CVE, No Automated Remediation - -**Request**: "Validate CVE-2024-5678" -1. Format → Valid, get_cve → found -2. advisory_available/remediation/advisories_list → no remediation -3. Return: "Valid but manual remediation: yum update custom-app" - -## Example 3: Invalid Format - -**Request**: "Validate CVE-24-1234" -1. Format → Invalid (year must be 4 digits) -2. Return error without MCP call; suggest CVE-2024-1234 - -## Example 4: CVE Not Found - -**Request**: "Validate CVE-2024-999999" -1. Format → Valid, get_cve → 404 -2. Return: "Not found. Check NVD, access.redhat.com, or wait 24-48h if recent" - -## Example 5: Batch Validation - -**Request**: "Validate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -1. Validate each sequentially -2. Return summary per CVE (valid/remediable, valid/manual, invalid format) -3. Suggest next steps per CVE diff --git a/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/05-error-handling.md b/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/05-error-handling.md deleted file mode 100644 index 201c193a..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/skills/cve-validation/references/05-error-handling.md +++ /dev/null @@ -1,37 +0,0 @@ -# CVE Validation Error Handling - -Read when errors occur during CVE validation. - -## CVE Format Invalid - -``` -CVE Validation Failed: Invalid Format -Provided: CVE-24-1234 -Expected: CVE-YYYY-NNNNN (YYYY=4-digit year, NNNNN=4-7 digit sequence) -Suggestion: Did you mean CVE-2024-1234? -``` - -## CVE Not Found in Database - -``` -CVE-YYYY-NNNNN was not found in Red Hat CVE database. -Possible reasons: Too recent, doesn't affect RHEL, incorrect ID. -Next steps: Verify at NVD, access.redhat.com/security/cve/CVE-YYYY-NNNNN, wait 24-48h if recent. -``` - -## CVE Exists But No Automated Remediation - -``` -CVE Validation: Valid (No Automated Remediation) -CVE-YYYY-NNNNN is valid but has no automated playbook. -Manual steps: dnf/yum update package-name, restart service if needed, verify fix. -Offer: "Would you like a manual playbook template?" -``` - -## API Access Error - -``` -CVE Validation Failed: API Access Error -Possible causes: Network, auth failure, service unavailable. -Troubleshooting: ping console.redhat.com, verify credentials, status.redhat.com, retry. -``` diff --git a/evaluation/with_skills/rh-sre__remediation/environment/skills/remediation/SKILL.md b/evaluation/with_skills/rh-sre__remediation/environment/skills/remediation/SKILL.md deleted file mode 100644 index 49dcb23f..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/skills/remediation/SKILL.md +++ /dev/null @@ -1,279 +0,0 @@ ---- -name: remediation -description: | - **CRITICAL**: Use this skill for ALL CVE remediation workflows. DO NOT use individual skills piecemeal for end-to-end remediation. - - Use when users request: - - CVE remediation playbooks or security patch deployment - - Multi-step remediation (validation → context → playbook → execution) - - Batch remediation across multiple systems or CVEs - - End-to-end CVE management (analysis + remediation + verification) - - Prioritizing and remediating CVEs (not just listing them) - - Emergency security response with immediate remediation plans - - DO NOT use for simple queries: - - "List critical CVEs" → Use `/cve-impact` skill - - "What's the CVSS score for CVE-X?" → Use `/cve-impact` or `/cve-validation` - - Standalone impact analysis without remediation → Use `/cve-impact` - - This skill orchestrates 6 specialized skills (cve-impact, cve-validation, system-context, playbook-generator, playbook-executor, remediation-verifier) for complete remediation workflows. ---- -model: inherit -color: red -metadata: - author: "Red Hat Ecosystem Engineering" - priority: "high" ---- - -# Remediation Skill - -End-to-end CVE remediation workflow. Orchestrates specialized skills for validation, context gathering, playbook generation, execution, and verification. - -## Prerequisites - -**Required MCP Servers**: `lightspeed-mcp` (CVE data, playbook generation), `aap-mcp-job-management`, `aap-mcp-inventory-management` (execution) - -**Related Skills** (this skill invokes them): -- `/mcp-lightspeed-validator` - Verify Lightspeed MCP before CVE operations -- `/mcp-aap-validator` - Verify AAP MCP before playbook execution -- `/cve-impact` - CVE risk assessment -- `/cve-validation` - CVE validation and remediation availability -- `/system-context` - System inventory and deployment context -- `/playbook-generator` - Ansible playbook generation -- `/playbook-executor` - Playbook execution via AAP -- `/remediation-verifier` - Post-remediation verification - -**Verification**: See Step 0 for MCP validation. Execute `/mcp-aap-validator` before Step 5 (playbook execution) if not already validated. - -## When to Use This Skill - -**Use this skill when**: -- User requests CVE remediation (playbook creation, patching, deployment) -- Full workflow needed: analysis → validation → playbook → execution → verification -- Batch remediation across multiple CVEs or systems - -**Do NOT use when**: -- User only wants CVE listing or impact analysis → Use `/cve-impact` -- User only wants CVE validation → Use `/cve-validation` -- User only wants playbook generation (no execution) → Use `/playbook-generator` directly - -## Workflow - -Execute skills in this order. **MANDATORY**: Use actual Skill tool invocations, NOT text pretending to invoke skills. **Each step must complete before the next begins**—do not start Step N+1 until Step N has returned its result. - -### Upfront: Planned Tasks (Before Step 0) - -**When**: Before executing any step. **Do NOT start Step 0 until the user validates the plan.** - -**Action**: Present the planned task list using **Part A** of [references/01-remediation-plan-template.md](references/01-remediation-plan-template.md). Show the 7 tasks (validate MCP → impact → validate CVE → context → playbook → execute → verify) and ask "Proceed with this plan?" - -**Task list ordering** (CRITICAL): If using TodoWrite or task list UI, create tasks **in workflow order** (Step 0, 1, 2, 3, 4, 5, 6). Do NOT create in completion order or random order—display order must match execution order. - -**Wait for explicit user response** ("yes" or "proceed") before invoking Step 0. If "abort" → stop. - -### Step 0: Validate MCP Prerequisites - -**Action**: Execute `/mcp-lightspeed-validator` (and `/mcp-aap-validator` before Step 5 if executing playbooks) - -**When**: Before any CVE or remediation operations. Can skip if already validated this session. - -**Sequencing (MANDATORY)**: Invoke validators **one at a time**. **Do NOT proceed to Step 1 until Step 0 is complete.** Wait for each validator to return explicit results (PASSED / FAILED / PARTIAL) before moving on. "Successfully loaded skill" alone does NOT mean validation completed—you must see the actual validation outcome. - -**Invocation**: Use the Skill tool for ALL sub-skill invocations (validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier). **Do NOT use "Task Output" with the skill name as task ID**—that causes "No task found" errors (e.g. "No task found with ID: cve-validation"). See [skill-invocation.md](../../docs/references/skill-invocation.md). - -**Handle result**: If validation fails, stop and provide setup instructions. If passed, proceed to Step 1. **If any skill invocation fails** (e.g. "No task found with ID: ..."): Proceed with a warning—do not block. Later steps will surface real errors if MCP is unavailable. - -### Step 1: Impact Analysis (If Requested or Needed) - -**Action**: Execute the `/cve-impact` skill - -**Invoke**: -``` -"Analyze CVE-XXXX-YYYY and assess its impact on affected systems" -``` - -**Expected**: Risk assessment, affected systems list, CVSS interpretation. Integrate into remediation planning. If user only wanted impact analysis, provide assessment and offer remediation options. - -### Step 2: Validate CVE (Remediatable Gate) - -**Action**: Execute the `/cve-validation` skill - -**Invoke**: -``` -"Validate CVE-XXXX-YYYY format, existence, and remediation availability" -``` - -**Expected**: Validation status including `remediation_status.automated_remediation_available` or `validation_status`. - -**Remediatable Gate** (MANDATORY): Trust cve-validation skill output. Do NOT re-interpret raw get_cve response—cve-validation uses advisory_available, remediation, advisories_list (not rules[]). See [cve-validation references/01-remediation-indicators.md](../cve-validation/references/01-remediation-indicators.md). -- **If remediatable** (`remediation_available: true` or `validation_status: "valid"`): Proceed to Step 3. -- **If NOT remediatable** (`remediation_available: false` or `validation_status: "not_remediable"`): - 1. Explain: "CVE-XXXX-YYYY has no automated remediation in Red Hat Lightspeed. Execution may have no effect." - 2. Suggest alternatives: manual patching, check Red Hat errata. - 3. Offer: "Continue anyway? (yes/no)" - 4. **If user says "yes"**: Proceed to Step 3 with warning: "⚠️ Proceeding despite no automated remediation—playbook generation or execution may have no effect." - 5. **If user says "no"**: Stop. Do not proceed to Steps 3–5. - -**Batch**: For multiple CVEs, validate each. Proceed only with remediatable CVEs unless user explicitly confirms to include non-remediatable ones (with same warning). - -### Step 3: Gather Context - -**Action**: Execute the `/system-context` skill - -**Invoke**: -``` -"Gather system context for CVE-XXXX-YYYY: identify affected systems, RHEL versions, and deployment environments" -``` - -**Expected**: Context summary with remediation strategy. Use to inform playbook generation and execution planning. - -### Step 4: Generate Playbook - -**Action**: Execute the `/playbook-generator` skill - -**CRITICAL**: You MUST invoke `/playbook-generator`, NOT generate playbook text yourself. - -**Invoke**: -``` -"Generate an Ansible remediation playbook for CVE-XXXX-YYYY targeting systems [list of system UUIDs]. Apply Red Hat best practices and RHEL-specific patterns from documentation." -``` - -**Expected**: Ansible playbook from Red Hat Lightspeed (returned AS IS by playbook-generator—no modifications). Present to user. **The playbook-generator ONLY GENERATES**—it does NOT execute. After presenting the playbook, present the Remediation Plan for user validation (see below). - -### Remediation Plan (User Validation) — MANDATORY before Step 5 - -**When**: After Step 4 completes. **Do NOT proceed to Step 5 until the user validates the plan.** - -**Action**: Present the plan using the Summary + Table + Checklist format. **Read [references/01-remediation-plan-template.md](references/01-remediation-plan-template.md)** for the exact template. - -**Format**: -1. **Summary** — 1–2 sentences: what will happen and why -2. **Table** — CVE | Target Systems | Key Action -3. **Checklist** — Ordered steps (mark completed as "— done") -4. **Confirm prompt** — "yes"/"proceed", "dry-run only", or "abort" - -**Wait for explicit user response.** If "yes" or "proceed" → invoke playbook-executor. If "abort" → stop. If "dry-run only" → invoke playbook-executor with instruction to run dry-run only and stop. - -### Step 5: Execute Playbook (With User Confirmation) - -**Prerequisite**: Remediation Plan must be presented and user must have responded "yes" or "proceed" (or "dry-run only"). Do NOT invoke playbook-executor until plan validation is complete. - -**CRITICAL**: Before execution, you MUST: -1. Have presented the Remediation Plan (summary + table + checklist) -2. Have received user confirmation ("yes", "proceed", or "dry-run only") -3. Show playbook preview and key tasks when invoking playbook-executor -4. Recommend dry-run first; wait for explicit approval before actual execution - -**Action**: Execute the `/playbook-executor` skill - -**Invoke** (pass playbook metadata from playbook-generator and system-context): -``` -"Execute the generated playbook for CVE-XXXX-YYYY. Playbook file: [filename from playbook-generator]. Content: [in context from playbook-generator output]. Target systems: [list of system UUIDs from system-context]. Start with dry-run (check mode) if user requested it. Monitor job status until completion and report results." -``` - -**Git Flow path**: When playbook-executor performs Git Flow (write playbook to repo), it MUST use the absolute path for the Write tool: `/playbooks/remediation/`. Never use a relative path like `test-aap-project/playbooks/...`—that causes "Error writing file" when the repo is outside the workspace. - -**Expected**: playbook-executor validates AAP, matches templates, offers dry-run, executes on approval, streams progress, generates report. **Validates job log for CVE handling**—confirms from stdout that the playbook addressed the target CVE(s); reports ✓ confirmation or ⚠️ warning if no evidence found. After success, suggest verification with `/remediation-verifier`. - -### Step 6: Verify Deployment (Optional) - -**Action**: Execute the `/remediation-verifier` skill (if user requests verification) - -**Invoke**: -``` -"Verify remediation success for CVE-XXXX-YYYY on systems [list of system UUIDs]. Check CVE status, package versions, and service health." -``` - -**Expected**: Verification report with pass/fail. Present results to user. - -## Dependencies - -### Required MCP Tools -- None (orchestration skill—delegates to other skills that use MCP tools) - -### Required MCP Servers -- `lightspeed-mcp` - CVE data, playbook generation -- `aap-mcp-job-management` - Job launch and monitoring -- `aap-mcp-inventory-management` - Inventory for execution - -### Related Skills -- `cve-impact` - Step 1 -- `cve-validation` - Step 2 -- `system-context` - Step 3 -- `playbook-generator` - Step 4 -- `playbook-executor` - Step 5 -- `remediation-verifier` - Step 6 - -### Reference Documentation -- [references/01-remediation-plan-template.md](references/01-remediation-plan-template.md) - Plan format for user validation -- [lightspeed-mcp-tool-failures.md](../../docs/references/lightspeed-mcp-tool-failures.md) - Backend errors (e.g. explain_cves), user-friendly message, workarounds -- [cve-remediation-templates.md](../../docs/ansible/cve-remediation-templates.md) -- [package-management.md](../../docs/rhel/package-management.md) - -## Critical: Human-in-the-Loop Requirements - -This skill requires explicit user confirmation at: - -1. **Upfront Planned Tasks** (before Step 0) - - Present the 7-task plan. Wait for "yes" or "proceed" before starting any step. - - Do NOT invoke validators or other skills until the user confirms. - -2. **Remediation Plan Validation** (before Step 5) - - Present the plan: Summary + Table + Checklist - - Wait for user response: "yes"/"proceed", "dry-run only", or "abort" - - Do NOT invoke playbook-executor until the user validates the plan - -3. **Before Playbook Execution (Step 5)** - - Display playbook preview and key tasks - - Recommend dry-run first; wait for explicit approval before actual execution - -4. **Before Destructive Actions** - - Offer dry-run (check mode) before actual execution - - If dry-run approved, run first and show results - - Only proceed to actual execution after user confirms - -**Never assume approval**—always wait for explicit user confirmation before execution. - -## MCP Tool Usage - -**vulnerability__explain_cves**: Requires a valid `system_uuid` from inventory. Do NOT call it unless you have the resolved UUID from Step 3 (system-context) or Step 1 (cve-impact). Never pass `system_uuid: "undefined"` or placeholder values—this causes validation errors. For remediation availability at Step 2, use `get_cve` via cve-validation only. - -**Lightspeed tool failures**: If a tool fails with a cryptic backend error (e.g. `'dnf_modules'`), do NOT retry or expose the raw error. Use workarounds from [lightspeed-mcp-tool-failures.md](../../docs/references/lightspeed-mcp-tool-failures.md). - -## Error Handling - -- **Invalid CVE**: "CVE-XXXX-YYYY is not valid or doesn't exist. Please verify the CVE ID." -- **No Remediation Available**: "CVE-XXXX-YYYY doesn't have an automated remediation playbook. Manual patching required." -- **System Not Found**: "System XXXX is not in the Lightspeed inventory. Please ensure it's registered." -- **Batch Partial Failure**: "Successfully processed X of Y CVEs. Failed: [list]. Reason: [explanations]" -- **Lightspeed tool failures** (e.g. explain_cves `'dnf_modules'`): Do NOT show raw error. Use user-friendly message and workaround from [lightspeed-mcp-tool-failures.md](../../docs/references/lightspeed-mcp-tool-failures.md). - -## Output Format - -**Single CVE**: -``` -CVE-XXXX-YYYY Remediation Summary -CVSS Score: X.X (Severity) -Affected Packages: package-name-version -Ansible Playbook Generated: ✓ -Target Systems: N systems -[Playbook YAML or AAP link] -[Execution instructions] -``` - -**Batch**: -``` -Batch Remediation Summary -CVEs: CVE-A, CVE-B, CVE-C -Target Systems: N systems -Total Fixes: X package updates -[Consolidated playbook] -[Execution instructions] -``` - -## Important Reminders - -- **Use actual tool calls**—invoke skills via Skill tool, not text. If tool use count is 0, you are doing it wrong. -- **Orchestrate skills, don't call MCP tools directly**—skills handle docs and tools. -- **Always ask for execution confirmation** before Step 5. -- **Safety**: Test in non-prod first, back up systems, schedule maintenance windows, verify after execution. diff --git a/evaluation/with_skills/rh-sre__remediation/environment/skills/remediation/references/01-remediation-plan-template.md b/evaluation/with_skills/rh-sre__remediation/environment/skills/remediation/references/01-remediation-plan-template.md deleted file mode 100644 index 343d4359..00000000 --- a/evaluation/with_skills/rh-sre__remediation/environment/skills/remediation/references/01-remediation-plan-template.md +++ /dev/null @@ -1,85 +0,0 @@ -# Remediation Plan Template - -Read this reference when presenting plans for user validation. - -## Part A: Upfront Planned Tasks (Before Step 0) - -**When**: Before executing any step. Present immediately after the user requests remediation. - -**Purpose**: Let the user validate the approach before any work begins. - -**Format**: -``` -## Remediation: CVE-XXXX-YYYY - -**Planned tasks** (in order—use this exact order for TodoWrite/task lists; display order must match execution order): -1. Validate MCP (Lightspeed, AAP) -2. Impact analysis (assess CVE risk) -3. CVE validation (remediatable gate) -4. System context (affected systems, RHEL versions) -5. Generate playbook -6. Dry-run → User confirms → Execute -7. Verify (optional) - -❓ Proceed with this plan? -- "yes" or "proceed" — I'll start with Step 0 (validate MCP) -- "abort" — Cancel -``` - -**Wait for user response** before invoking Step 0. Do NOT start any step until the user confirms. - ---- - -## Part B: Execution Plan (After Step 4, Before Step 5) - -**When**: After Step 4 (playbook generated) and before Step 5 (execution). The user must validate before proceeding. - -## Part B Format - -### 1. Summary (1–2 sentences) - -``` -## Remediation Plan: CVE-XXXX-YYYY - -**Summary**: [One sentence describing what will happen and why.] -Example: "Remediate CVE-2026-24882 on ip-172-31-32-201 via Ansible playbook (httpd update to address CVE)." -``` - -### 2. Table (CVE, systems, key actions) - -``` -| CVE | Target Systems | Key Action | -|-----|----------------|------------| -| CVE-XXXX-YYYY | hostname-1, hostname-2 | Update package: httpd 2.4.x → 2.4.y | -``` - -For batch: one row per CVE or combined row if same action. - -### 3. Checklist (ordered steps) - -``` -**Execution steps**: -☐ Step 0: Validate MCP (Lightspeed, AAP) — done -☐ Step 1: Impact analysis — done -☐ Step 2: CVE validation — done -☐ Step 3: System context — done -☐ Step 4: Generate playbook — done -☐ Step 5: Dry-run → Confirm → Execute -☐ Step 6: Verify (optional) -``` - -Mark completed steps as "— done". Show only remaining steps as checkboxes if preferred. - -### 4. Confirm Prompt - -``` -❓ Confirm to proceed? - -- "yes" or "proceed" — Run dry-run first, then execute -- "dry-run only" — Run dry-run only, no execution -- "abort" — Cancel remediation - -Please respond with your choice. -``` - -**Wait for explicit user response** before invoking playbook-executor. diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/semantic-index.json b/evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/INDEX.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/SOURCES.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/README.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/aap-job-execution.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/cve-remediation-templates.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/playbook-integration-aap.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/insights/README.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/insights/vulnerability-logic.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/references/README.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/references/cvss-scoring.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/references/skill-invocation.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/rhel/README.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/rhel/package-management.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/with_skills/rh-sre__system-context/environment/docs/testing/aap-integration-test-guide.md b/evaluation/with_skills/rh-sre__system-context/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 1092dd1d..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- pass rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/with_skills/rh-sre__system-context/environment/skills/system-context/SKILL.md b/evaluation/with_skills/rh-sre__system-context/environment/skills/system-context/SKILL.md deleted file mode 100644 index 704ead67..00000000 --- a/evaluation/with_skills/rh-sre__system-context/environment/skills/system-context/SKILL.md +++ /dev/null @@ -1,488 +0,0 @@ ---- -name: system-context -description: | - **CRITICAL**: This skill must be used for system inventory and context gathering. DO NOT use raw MCP tools like get_cve_systems or get_host_details directly. - - Gather comprehensive system inventory and deployment context for CVE-affected systems, including RHEL version detection, environment classification, and deployment analysis. Use this skill when you need to understand system infrastructure before planning remediation. - - This skill orchestrates MCP tools (get_cve_systems, get_host_details) to provide comprehensive system analysis with RHEL version detection, environment classification, and remediation strategy determination. - - **IMPORTANT**: ALWAYS use this skill instead of calling get_cve_systems or get_host_details directly for system context gathering. ---- - -# System Context Gathering Skill - -This skill gathers comprehensive system inventory and deployment context for CVE-affected systems, enabling informed remediation strategy decisions. - -**Integration with Remediation Skill**: The `/remediation` skill orchestrates this skill as part of its Step 3 (Gather Context) workflow. For standalone system analysis, you can invoke this skill directly. - -## When to Use This Skill - -**Use this skill directly when you need**: -- Understand which systems are affected by a CVE -- Analyze deployment architecture (Kubernetes, bare metal, VMs) -- Detect RHEL versions across infrastructure -- Classify systems by environment (dev/staging/prod) -- Gather context before remediation planning - -**Use the `/remediation` skill when you need**: -- End-to-end CVE remediation workflow -- Integrated analysis → context → playbook → execution -- Automated remediation strategy determination - -**How they work together**: The `/remediation` skill uses this skill's output to determine remediation strategy (batch vs individual, Kubernetes pod eviction requirements, maintenance window needs, etc.). - -## Workflow - -### 1. Identify Affected Systems - -**MCP Tool**: `get_cve_systems` or `vulnerability__get_cve_systems` (from lightspeed-mcp) - -**Parameters**: -- `cve_id`: Exact CVE identifier (format: `"CVE-YYYY-NNNNN"`) - - Example: `"CVE-2024-1234"` -- `limit`: Optional number of systems to return (default: all) - - Example: `100` - - Use for large deployments to paginate results -- `offset`: Optional pagination offset (default: 0) - - Example: `0` - - Use with limit for pagination - -**Expected Output**: List of system UUIDs and basic metadata - -**Example Response**: -```json -{ - "systems": [ - { - "id": "uuid-1", - "hostname": "web-server-01", - "display_name": "web-server-01.prod.example.com" - }, - { - "id": "uuid-2", - "hostname": "web-server-02", - "display_name": "web-server-02.prod.example.com" - } - ], - "total": 2 -} -``` - -### 2. Gather Detailed System Information - -**MCP Tool**: `get_host_details` or `inventory__get_host_details` (from lightspeed-mcp) - -**Parameters**: -- `system_id`: UUID of the system to retrieve (from get_cve_systems result) - - Example: `"uuid-1"` - - Format: UUID string -- `include_system_profile`: `true` (retrieve complete system profile including packages, services) - - Example: `true` - - Recommended: Always true for context gathering -- `include_tags`: Optional boolean to include system tags (default: true) - - Example: `true` - - Tags provide environment, role, criticality classification - -**Expected Output**: Detailed system profile - -**Key Information to Extract**: -- RHEL version (rhel_version, os_release) -- System type (infrastructure_type: bare_metal, virtualized, container) -- IP addresses (network_interfaces) -- Tags (environment, role, criticality) -- System profile (CPU, memory, disk) -- Installed packages (installed_packages) -- Running services (enabled_services, running_processes) -- Last check-in time (updated) - -**System Profile Structure**: -```json -{ - "id": "uuid-1", - "hostname": "web-server-01", - "display_name": "web-server-01.prod.example.com", - "rhel_version": "8.9", - "os_release": "Red Hat Enterprise Linux 8.9 (Ootpa)", - "system_profile": { - "os_release": "8.9", - "arch": "x86_64", - "kernel_version": "4.18.0-513.el8.x86_64", - "number_of_cpus": 4, - "number_of_sockets": 1, - "cores_per_socket": 4, - "system_memory_bytes": 17179869184, - "infrastructure_type": "virtualized", - "infrastructure_vendor": "kvm", - "network_interfaces": [ - { - "name": "eth0", - "ipv4_addresses": ["10.0.1.10"] - } - ], - "installed_packages": [...], - "enabled_services": ["httpd", "sshd", ...], - "running_processes": [...] - }, - "tags": [ - {"namespace": "environment", "key": "env", "value": "production"}, - {"namespace": "role", "key": "role", "value": "web-server"}, - {"namespace": "criticality", "key": "level", "value": "high"} - ] -} -``` - -### 3. Analyze Deployment Context - -Synthesize gathered information to understand deployment architecture: - -**A. RHEL Version Distribution**: -``` -Affected Systems by RHEL Version: -- RHEL 7: 3 systems (15%) -- RHEL 8: 15 systems (75%) -- RHEL 9: 2 systems (10%) - -Remediation Consideration: -→ Playbook must support multiple RHEL versions (use conditional yum/dnf) -``` - -**B. Environment Classification**: -``` -Affected Systems by Environment: -- Production: 12 systems (60%) - HIGH PRIORITY -- Staging: 5 systems (25%) - MEDIUM PRIORITY -- Development: 3 systems (15%) - LOW PRIORITY - -Remediation Strategy: -→ Remediate staging first for validation -→ Schedule maintenance window for production -→ Development can be patched anytime -``` - -**C. System Type Distribution**: -``` -Affected Systems by Type: -- Bare metal: 12 systems (60%) - STANDARD REMEDIATION -- VMs (VMware): 8 systems (40%) - STANDARD REMEDIATION - -Deployment Type: -→ Standard remediation workflow -→ Consider reboot requirements -→ Schedule maintenance windows for critical systems -``` - -**D. System Criticality**: -``` -Affected Systems by Criticality: -- Critical (payment, auth): 5 systems - NEEDS MAINTENANCE WINDOW -- High (web, api): 10 systems - NEEDS TESTING -- Medium (internal tools): 3 systems - STANDARD DEPLOYMENT -- Low (dev, test): 2 systems - IMMEDIATE DEPLOYMENT OK - -Remediation Approach: -→ Test on low-criticality systems first -→ Schedule maintenance for critical systems -→ Use rolling updates for high-availability services -``` - -### 5. Determine Remediation Strategy - -Based on gathered context, recommend remediation strategy: - -**Decision Matrix**: - -| Context | Remediation Strategy | -|---------|---------------------| -| Single system, non-K8s | Standard playbook, immediate execution possible | -| Multiple systems, same RHEL version | Batch playbook, parallel execution | -| Multiple systems, mixed RHEL versions | Batch playbook with version conditionals | -| Kubernetes nodes | Rolling update with pod eviction | -| Critical production systems | Maintenance window required, staged rollout | -| Mixed environments | Remediate staging → validate → production | - -**Strategy Output**: -```yaml -remediation_strategy: - approach: "rolling_update" # or "batch", "individual", "staged" - - requires_maintenance_window: true - suggested_window: "Weekend, off-peak hours" - - requires_pod_eviction: true - pod_eviction_strategy: "one_node_at_a_time" - - batch_size: 5 - parallel_execution: true - - test_first_on: - - "staging-web-01" - - "staging-web-02" - - rollout_order: - - phase: "validation" - systems: ["staging-web-01", "staging-web-02"] - wait_for_verification: true - - - phase: "production_batch_1" - systems: ["prod-web-01", "prod-web-02", "prod-web-03"] - requires_approval: true - - - phase: "production_batch_2" - systems: ["prod-web-04", "prod-web-05"] - requires_approval: false - - estimated_duration_minutes: 60 - estimated_downtime_per_system: 5 -``` - -### 6. Return Context Summary - -Return comprehensive context for remediation planning: - -```json -{ - "cve_id": "CVE-YYYY-NNNNN", - - "affected_systems": { - "total": 20, - "by_rhel_version": { - "rhel7": 3, - "rhel8": 15, - "rhel9": 2 - }, - "by_environment": { - "production": 12, - "staging": 5, - "development": 3 - }, - "by_type": { - "kubernetes": 8, - "bare_metal": 7, - "vm": 5 - }, - "by_criticality": { - "critical": 5, - "high": 10, - "medium": 3, - "low": 2 - } - }, - - "kubernetes_context": { - "has_k8s_nodes": true, - "total_k8s_nodes": 8, - "clusters": ["prod-cluster-01", "staging-cluster-01"], - "total_pods_affected": 150, - "has_pdbs": true, - "daemonsets_present": true - }, - - "remediation_strategy": { - "approach": "rolling_update", - "requires_maintenance_window": true, - "requires_pod_eviction": true, - "batch_size": 5, - "estimated_duration_minutes": 60 - }, - - "recommendations": [ - "Test in staging environment first (5 systems available)", - "Schedule maintenance window for production (12 critical systems)", - "Use rolling updates with pod eviction for Kubernetes nodes", - "Playbook must support RHEL 7, 8, and 9", - "Consider batch size of 5 systems for parallel execution" - ] -} -``` - -## Output Template - -When completing context gathering, provide output in this format: - -```markdown -# System Context Analysis - -## CVE: CVE-YYYY-NNNNN - -## Affected Systems Summary -**Total Systems**: 20 - -### By RHEL Version -- RHEL 7: 3 systems (15%) -- RHEL 8: 15 systems (75%) -- RHEL 9: 2 systems (10%) - -### By Environment -- Production: 12 systems (60%) - HIGH PRIORITY -- Staging: 5 systems (25%) - TEST FIRST -- Development: 3 systems (15%) - -### By System Type -- Kubernetes nodes: 8 systems (40%) - REQUIRES POD EVICTION -- Bare metal: 7 systems (35%) -- VMs: 5 systems (25%) - -### By Criticality -- Critical: 5 systems (payment, auth services) -- High: 10 systems (web, API services) -- Medium: 3 systems (internal tools) -- Low: 2 systems (dev/test) - -## Kubernetes Deployment -**Kubernetes Nodes**: 8 systems -**Clusters**: prod-cluster-01 (5 nodes), staging-cluster-01 (3 nodes) -**Total Pods Affected**: ~150 pods -**PodDisruptionBudgets**: Present (payment service, auth service) -**DaemonSets**: node-exporter, fluent-bit - -## Recommended Remediation Strategy - -**Approach**: Rolling Update with Pod Eviction - -**Execution Plan**: -1. **Phase 1 - Validation** (Staging): - - Systems: staging-web-01, staging-web-02 (5 systems) - - Test playbook execution - - Verify no issues before production - -2. **Phase 2 - Production Batch 1**: - - Systems: prod-web-01 to prod-web-05 (5 systems) - - Requires approval after staging validation - - Rolling update with pod eviction - -3. **Phase 3 - Production Batch 2**: - - Systems: prod-web-06 to prod-web-10 (5 systems) - - Continue if Batch 1 successful - -**Requirements**: -- Maintenance window: Weekend off-peak hours -- Pod eviction strategy: One node at a time -- Batch size: 5 systems (parallel execution) -- Estimated duration: 60 minutes total -- Estimated downtime per system: ~5 minutes - -**Safety Measures**: -- Test in staging first -- Rolling updates maintain service availability -- PodDisruptionBudgets respected -- Rollback capability via snapshots (RHEL 8/9) - -## Next Steps -1. Review remediation strategy -2. Schedule maintenance window for production -3. Generate remediation playbook (use playbook-generator skill) -4. Execute in staging for validation -5. Proceed with production deployment -``` - -## Examples - -### Example 1: Simple Environment - -**User Request**: "Gather context for CVE-2024-1234" - -**Skill Response**: -1. Call `get_cve_systems` → 5 systems affected -2. Call `get_host_details` for each → All RHEL 8, production web servers -3. Analyze context → All same version, same environment, bare metal -4. Return simple remediation strategy: "Batch remediation, 5 systems, standard playbook" - -### Example 2: Complex Multi-Version Deployment - -**User Request**: "Gather context for kernel CVE affecting production environment" - -**Skill Response**: -1. Call `get_cve_systems` → 10 systems affected -2. Call `get_host_details` → Mixed RHEL 8/9, production environment -3. Analyze system types → Mix of bare metal and VMs, high criticality tags -4. Check reboot requirements → Kernel update requires maintenance window -5. Return complex strategy: "Rolling update by RHEL version, separate playbooks for RHEL 8 and 9, coordinate maintenance window for critical systems" - -### Example 3: Multi-Environment Deployment - -**User Request**: "Gather context for CVE affecting dev, staging, and prod" - -**Skill Response**: -1. Call `get_cve_systems` → 20 systems across 3 environments -2. Call `get_host_details` → Extract environment tags -3. Classify by environment → 3 dev, 5 staging, 12 prod -4. Determine criticality → Production has critical services -5. Return staged strategy: "Test on dev (3 systems) → Validate on staging (5 systems) → Deploy to production with approval (12 systems)" - -## Error Handling - -**No systems affected**: -``` -CVE-YYYY-NNNNN Analysis Complete - -Good news! No systems in your infrastructure are currently affected by this CVE. - -Possible reasons: -- Systems are already patched -- Vulnerable packages are not installed -- Systems are running different versions - -No remediation required. -``` - -**Lightspeed inventory access error**: -``` -Unable to retrieve system details from Red Hat Lightspeed inventory. - -Possible causes: -- Systems not registered to Red Hat Lightspeed -- Lightspeed inventory sync pending -- API authentication issue - -Suggestions: -- Verify systems are registered: subscription-manager status -- Check Lightspeed connection: insights-client --status -- Re-run inventory sync: insights-client --register -``` - -**System tagging incomplete**: -``` -Unable to fully classify systems due to incomplete tagging. - -Proceeding with Red Hat Lightspeed data only. -Note: Environment and criticality tags missing from some systems. - -To improve system classification: -1. Add environment tags to systems in Red Hat Lightspeed -2. Add criticality/role tags for better prioritization -3. Ensure all systems are registered and reporting -``` - -## Dependencies - -### Required MCP Servers -- `lightspeed-mcp` - Red Hat Lightspeed platform access - -### Required MCP Tools -- `get_cve_systems` or `vulnerability__get_cve_systems` (from lightspeed-mcp) - List systems affected by CVE - - Parameters: cve_id (string, format CVE-YYYY-NNNNN), limit (number, optional), offset (number, optional) - - Returns: List of system UUIDs and basic metadata (hostname, display_name) -- `get_host_details` or `inventory__get_host_details` (from lightspeed-mcp) - Get detailed system information - - Parameters: system_id (UUID string), include_system_profile (boolean), include_tags (boolean, optional) - - Returns: Complete system profile including RHEL version, infrastructure type, tags, packages, services - -### Related Skills -- `cve-impact` - Provides CVE severity to inform criticality assessment -- `playbook-generator` - Consumes context to generate appropriate remediation playbook -- `remediation-verifier` - Uses system context to verify remediation on correct systems -- `cve-validation` - Validates CVE before gathering affected systems - -### Reference Documentation -- None required (system context is gathered from MCP tool queries) - -## Best Practices - -1. **Always gather full context** - Don't skip system details even if deployment seems simple -2. **Classify by environment** - Always test in staging before production deployment -3. **Check system criticality** - Remediation strategy depends on system importance (critical vs low) -4. **Respect criticality tags** - High-criticality systems need maintenance windows and extra care -5. **Detect RHEL version mix** - Playbooks must handle multiple versions (conditional dnf/yum logic) -6. **Consider batch size** - Balance speed vs risk (5-10 systems per batch recommended) -7. **Plan for rollback** - Always have a backup strategy (snapshots, maintenance windows) -8. **Use pagination for large fleets** - If get_cve_systems returns 100+ systems, use limit/offset parameters diff --git a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/.ai-index/semantic-index.json deleted file mode 100644 index 2512c2a3..00000000 --- a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/.ai-index/semantic-index.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "index_version": "1.0.0", - "last_updated": "2026-02-17", - "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", - - "documents": [ - { - "file": "INDEX.md", - "title": "VM Troubleshooting Guide - Index", - "category": "navigation", - "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], - "use_cases": ["navigation", "tool-reference", "general-guidance"], - "token_estimate": 450, - "skills": ["all"] - }, - { - "file": "scheduling-errors.md", - "title": "VM Scheduling Errors", - "category": "scheduling", - "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], - "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], - "errors_covered": [ - "ErrorUnschedulable - Node Taints", - "ErrorUnschedulable - Insufficient Resources", - "ErrorUnschedulable - Node Selector Mismatch" - ], - "token_estimate": 820, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "storage-errors.md", - "title": "VM Storage Errors", - "category": "storage", - "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], - "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], - "errors_covered": [ - "ErrorDataVolumeNotReady - DataVolume Still Provisioning", - "ErrorDataVolumeNotReady - Storage Class Not Found", - "ErrorDataVolumeNotReady - Insufficient Storage Quota", - "ErrorPvcNotFound", - "Storage Deletion Failures", - "DataVolume Cloning Failures" - ], - "token_estimate": 1350, - "skills": ["vm-creator", "vm-delete", "vm-clone"] - }, - { - "file": "lifecycle-errors.md", - "title": "VM Lifecycle Errors", - "category": "lifecycle", - "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], - "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], - "errors_covered": [ - "VM Stuck in Terminating State", - "VM Won't Start (Non-Scheduling Issues)", - "VM Won't Stop" - ], - "token_estimate": 1150, - "skills": ["vm-delete", "vm-lifecycle-manager"] - }, - { - "file": "runtime-errors.md", - "title": "VM Runtime Errors", - "category": "runtime", - "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], - "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], - "errors_covered": [ - "CrashLoopBackOff" - ], - "token_estimate": 850, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "network-errors.md", - "title": "VM Network Errors", - "category": "networking", - "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], - "use_cases": ["vm-creation", "network-troubleshooting"], - "errors_covered": [ - "Network Attachment Failures" - ], - "token_estimate": 580, - "skills": ["vm-creator"] - } - ], - - "skill_to_docs_mapping": { - "vm-creator": { - "primary_docs": ["scheduling-errors.md", "storage-errors.md"], - "secondary_docs": ["runtime-errors.md", "network-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3650, - "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" - }, - "vm-delete": { - "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 2950, - "description": "VM deletion failures - stuck terminating, storage cleanup" - }, - "vm-clone": { - "primary_docs": ["storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 1800, - "description": "VM cloning failures - DataVolume cloning, storage provisioning" - }, - "vm-lifecycle-manager": { - "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], - "secondary_docs": ["runtime-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3270, - "description": "VM start/stop failures - lifecycle management, scheduling" - }, - "vm-inventory": { - "primary_docs": [], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 450, - "description": "General guidance for status interpretation" - } - }, - - "error_to_docs_mapping": { - "ErrorUnschedulable": "scheduling-errors.md", - "ErrorDataVolumeNotReady": "storage-errors.md", - "ErrorPvcNotFound": "storage-errors.md", - "Terminating": "lifecycle-errors.md", - "CrashLoopBackOff": "runtime-errors.md", - "NetworkAttachmentFailed": "network-errors.md" - }, - - "cross_references": { - "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], - "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], - "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], - "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], - "network-errors.md": ["scheduling-errors.md"] - }, - - "progressive_disclosure_strategy": { - "description": "Load only the minimum documentation needed for current task", - "pattern": "error_type → primary_doc → cross_references (if needed)", - "token_savings": "29% average reduction vs single-file (based on rh-sre data)", - "navigation_overhead_reduction": "85% vs single-file navigation" - } -} diff --git a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/INDEX.md deleted file mode 100644 index a2e76963..00000000 --- a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/INDEX.md +++ /dev/null @@ -1,332 +0,0 @@ ---- -title: VM Troubleshooting Guide - Index -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 - - title: OpenShift Virtualization - Virtual Machine Status - url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html - date_accessed: 2026-02-06 - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] -semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] -use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] -last_updated: 2026-02-17 ---- - -# VM Troubleshooting Guide - Index - -## Overview - -This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: - -- **Creation**: Scheduling failures, storage provisioning issues -- **Lifecycle**: Start/stop failures, stuck states -- **Deletion**: Resources stuck in Terminating, storage cleanup failures -- **Cloning**: DataVolume cloning errors, cross-namespace issues -- **Networking**: Secondary network attachment failures -- **Runtime**: CrashLoopBackOff, guest OS boot failures - -This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. - ---- - -## 🗂️ Troubleshooting Categories - -The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: - -### 1. [Scheduling Errors](scheduling-errors.md) -**When to use**: VM fails to schedule on any node - -**Errors covered**: -- ErrorUnschedulable - Node Taints -- ErrorUnschedulable - Insufficient Resources -- ErrorUnschedulable - Node Selector Mismatch - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 2. [Storage Errors](storage-errors.md) -**When to use**: VM has storage provisioning, deletion, or cloning issues - -**Errors covered**: -- ErrorDataVolumeNotReady (all 3 subsections) -- ErrorPvcNotFound -- Storage Deletion Failures -- DataVolume Cloning Failures - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### 3. [Lifecycle Errors](lifecycle-errors.md) -**When to use**: VM has start/stop/termination issues - -**Errors covered**: -- VM Stuck in Terminating State -- VM Won't Start (Non-Scheduling Issues) -- VM Won't Stop - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### 4. [Runtime Errors](runtime-errors.md) -**When to use**: VM repeatedly crashes or fails at runtime - -**Errors covered**: -- CrashLoopBackOff - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 5. [Network Errors](network-errors.md) -**When to use**: VM has secondary network attachment failures - -**Errors covered**: -- Network Attachment Failures (Multus, SR-IOV) - -**Skills that use this**: vm-create - ---- - -## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last - -### Critical Principle: MCP-First Approach - -**All diagnostic procedures in this guide follow the MCP-first pattern**: - -``` -1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) -2. ⚠️ IF FAILS: Ask user permission to use CLI command -3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval -``` - -### Why MCP Tools First? - -- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources -- **Consistency**: Same interface across all operations -- **Better Error Handling**: MCP tools return structured errors -- **Audit Trail**: MCP tool usage is logged and trackable - -### Command Pattern Examples - -Throughout this guide, you'll see diagnostic steps formatted like this: - -**✅ CORRECT Pattern (MCP First)**: - -```markdown -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Extract**: `.metadata.finalizers` from returned JSON - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` -``` - -### Available MCP Tools - -The `openshift-virtualization` MCP server provides these tools: - -**Resource Operations**: -- `resources_get` - Get specific resource (replaces `oc get `) -- `resources_list` - List resources (replaces `oc get `) -- `resources_delete` - Delete resource (replaces `oc delete`) -- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) - -**Pod Operations**: -- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) -- `pods_get` - Get pod details (replaces `oc get pod`) -- `pods_log` - Get pod logs (replaces `oc logs`) -- `pods_exec` - Execute in pod (replaces `oc exec`) -- `pods_delete` - Delete pod (replaces `oc delete pod`) -- `pods_top` - Pod resource usage (replaces `oc top pods`) - -**Events & Monitoring**: -- `events_list` - List events (replaces `oc get events`) -- `nodes_top` - Node resource usage (replaces `oc top nodes`) -- `nodes_log` - Node logs (replaces `oc adm node-logs`) -- `nodes_stats_summary` - Detailed node stats - -**VM Operations** (KubeVirt toolset): -- `vm_create` - Create VMs -- `vm_lifecycle` - Start/stop/restart VMs - -**Namespaces**: -- `namespaces_list` - List namespaces (replaces `oc get namespaces`) -- `projects_list` - List OpenShift projects (replaces `oc get projects`) - -### When CLI Commands Are Required - -Some operations have **NO MCP equivalent** and require CLI: - -- `virtctl` commands (console, VNC access) -- `oc debug node` (node debugging) -- `oc auth can-i` (permission checks) -- `oc adm taint` (node taint management) -- Complex JSON patch operations - -For these, the guide will note: **"CLI Only - No MCP equivalent"** - -### Quick Reference: CLI → MCP Mapping - -| CLI Command | MCP Tool Equivalent | -|-------------|---------------------| -| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | -| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | -| `oc logs -n ` | `pods_log` with name="", namespace="" | -| `oc get events -n ` | `events_list` with namespace="" | -| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | - -**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. - -### How to Read Diagnostic Sections - -Each error section includes: -1. **Symptom** - What you observe -2. **Description** - What's happening -3. **Possible Causes** - Why it's happening -4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback -5. **Solutions** - **MCP tools first**, then CLI fallback -6. **Verification** - **MCP tools first**, then CLI fallback - -**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. - ---- - -## 🔍 Quick Navigation by Skill - -**vm-create**: -- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics -- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning -- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff -- [Network Errors](network-errors.md) - Network attachment failures - -**vm-delete**: -- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state -- [Storage Errors](storage-errors.md) - Storage deletion failures - -**vm-clone**: -- [Storage Errors](storage-errors.md) - DataVolume cloning failures - -**vm-lifecycle-manager**: -- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop -- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling - -**vm-inventory**: -- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed - ---- - -## 📊 Documentation Coverage & Maintenance - -### Current Coverage - -This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: - -- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) -- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures -- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop -- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) -- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems - -**Total errors documented**: 12 error types across 6 categories - ---- - -### Encountering Undocumented Errors - -#### For AI Agents (Claude Code) - -If you encounter an error **not documented** in the categories above: - -1. **Report to user** with all available details (error message, affected resources, namespace) -2. **Provide best-effort diagnostics** using MCP tools: - - `resources_get` to inspect resource status - - `pods_log` to check virt-launcher or compute container logs - - `events_list` to view Kubernetes events -3. **Suggest documentation update**: - ``` - ⚠️ This error is not yet documented in the troubleshooting guide. - - **Error**: - **Affected resource**: / - - I recommend adding this error to the troubleshooting guide: - - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file - - If it's a new error class → Create new category file in docs/troubleshooting/ - - Would you like me to help document this error for future reference? - ``` - -4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs - ---- - -#### For Human Contributors - -To document a new error: - -1. **Determine the category**: - - Scheduling issue → `scheduling-errors.md` - - Storage problem → `storage-errors.md` - - Lifecycle/start/stop → `lifecycle-errors.md` - - Runtime crash → `runtime-errors.md` - - Network issue → `network-errors.md` - - New error class → Create new category file (e.g., `security-errors.md`) - -2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): - - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback - - **Solutions**: MCP Tool approach → Implementation → CLI Fallback - - **Verification**: MCP Tool checks → Expected results → CLI Fallback - -3. **Update semantic index**: - - Edit `.ai-index/semantic-index.json` - - Add error to `error_to_docs_mapping` - - Update relevant skill's `primary_docs` or `secondary_docs` - - Increment token estimates if needed - -4. **Reference in skill files**: - - Update skill's Reference Documentation section - - Ensure skill's Document Consultation steps point to the right category - -5. **Update this INDEX.md**: - - Add error to the appropriate category section (lines 50-120) - - Update "Current Coverage" list above - ---- - -## 📚 Additional Resources - -- [rh-virt README](../../README.md) - MCP server setup and skill documentation -- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources -- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/lifecycle-errors.md deleted file mode 100644 index 6167fcc8..00000000 --- a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/lifecycle-errors.md +++ /dev/null @@ -1,869 +0,0 @@ ---- -title: VM Lifecycle Errors -category: kubevirt -sources: - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 -tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] -semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] -use_cases: [vm-deletion, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Lifecycle Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM lifecycle issues including start/stop failures and stuck termination states. - -**When to use this document**: -- VM stuck in `Terminating` state -- VM won't start (runStrategy is Always but VM never reaches Running) -- VM won't stop (runStrategy is Halted but VM never reaches Stopped) - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### VM Stuck in Terminating State - -**Symptom**: VM shows status `Terminating` but deletion never completes - -**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). - -**Possible Causes**: -- Finalizers blocking deletion -- PVC/DataVolume still attached and preventing cleanup -- VirtualMachineInstance (VMI) not terminating properly -- Custom controllers or operators blocking deletion -- Stuck virt-launcher pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check finalizers on the VM**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` - -**2. Check if VMI still exists**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name containing "virt-launcher-". - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check events for deletion issues**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results where `involvedObject.name` == "" and sort by timestamp. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' -``` - -**5. Check VM deletion timestamp**: - -Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. - -If present, VM is in deletion process. If null, VM is not being deleted. - -**Common Finalizer Patterns**: -- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer -- `foregroundDeletion` - Waits for dependent objects to be deleted -- Custom finalizers from operators - -**Solutions** (Use MCP Tools First): - -1. **Wait for dependent resources** (safest approach): - - Use `resources_list` to check if PVCs, DataVolumes are still deleting - - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) - -2. **Force delete VMI** (if VMI is stuck): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable - requires explicit user permission): - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -3. **Force delete virt-launcher pod** (if pod is stuck): - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - # Find the virt-launcher pod name first - oc get pods -n | grep virt-launcher- - - # Then delete it - oc delete pod virt-launcher--xxx -n --force --grace-period=0 - ``` - -4. **Remove finalizers** (⚠️ dangerous - use only as last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` - 2. Remove items from `.metadata.finalizers` array - 3. Update VM using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch not easily done via MCP): - ```bash - # This operation is complex for MCP - may need CLI - oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Check for protection labels** (vm-delete skill specific): - - Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. - - If value is "true", the vm-delete skill refuses deletion (this is expected behavior). - -**Verification**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error if deletion successful. - -**CLI Fallback**: -```bash -oc get vm -n -# Should return: Error from server (NotFound) -``` - ---- - - ---- - -### VM Won't Start (Non-Scheduling Issues) - -**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state - -**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. - -**Possible Causes**: -- Guest OS kernel panic or boot failure -- Cloud-init configuration errors -- Missing or corrupted disk image -- Insufficient memory for guest OS to boot -- QEMU/KVM virtualization errors -- VirtualMachineInstance (VMI) creation failures -- virt-launcher pod crashes - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VMI (VirtualMachineInstance) status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and overall VMI state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**2. Check VMI conditions for errors**: - -Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. View virt-launcher pod logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -First, get pod name from step 3, then: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -Look for QEMU errors, memory allocation failures, device errors. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**5. Check virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. - -Alternatively, use `pods_get` to get full pod details: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**6. Access VM console to see guest OS boot messages**: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -Look for kernel panic, initramfs errors, filesystem errors. - -**7. Check VM events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep -A 20 "Events:" -``` - -**Common Error Patterns**: - -1. **Guest OS Boot Failure**: - - Console shows kernel panic - - Guest hangs at GRUB or boot loader - - Cloud-init errors during first boot - -2. **Insufficient Memory**: - - Guest OS kills processes due to OOM - - VMI logs show memory allocation errors - -3. **QEMU Crashes**: - - virt-launcher logs show QEMU segfaults - - VMI repeatedly restarts - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for kernel panic, initramfs errors, filesystem errors. - -2. **Check virt-launcher pod logs for QEMU errors**: - - Use `pods_log` from diagnostic step 4 to view logs. - - Look for: - - "qemu-system-x86_64: ..." errors - - Memory allocation failures - - Device errors - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx - ``` - -3. **Increase memory if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check VMI memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -4. **Verify disk image integrity**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check PVC is bound: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - If using container disk, verify image exists and is accessible (check VMI spec). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get pvc -n - ``` - -5. **Check cloud-init configuration** (if applicable): - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. - - Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq - ``` - -6. **Restart VMI** (soft reset): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete VMI (VM controller will recreate it): - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for new VMI to start (use `resources_get` to check status). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n -w - ``` - -7. **Check virtualization extensions** (KVM): - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - - # In debug shell: - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should eventually return `Running`). - -Check VMI is running: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should show `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should eventually return: Running - -oc get vmi -n -# Should show: Running -``` - ---- - -### VM Won't Stop - -**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state - -**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. - -**Possible Causes**: -- Guest OS not responding to ACPI shutdown signal -- virt-launcher pod stuck and not terminating -- VirtualMachineInstance (VMI) deletion blocked by finalizers -- Guest shutdown scripts hanging -- Filesystem sync issues in guest OS - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (might show `Stopping`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**2. Check VMI status and deletion timestamp**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check VMI events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vmi -n | grep -A 10 "Events:" -``` - -**5. Check VMI finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.metadata.finalizers}' -``` - -**6. Check if guest is responsive** (if VMI still exists): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Patterns**: -- VMI shows `deletionTimestamp` but never actually deletes -- virt-launcher pod in `Terminating` state -- VM runStrategy is `Halted` but printableStatus shows `Stopping` - -**Solutions** (Use MCP Tools First): - -1. **Wait for graceful shutdown** (default: 30 seconds): - - Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. - -2. **Force stop by deleting VMI**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - This is the standard way to force-stop a VM. - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n - # Should return: Error from server (NotFound) - ``` - -3. **Force delete VMI with grace period** (if VMI won't delete): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -4. **Force delete virt-launcher pod**: - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find the pod using `pods_list_in_namespace` from diagnostic step 3. - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - ⚠️ **Note**: For force deletion with grace period, use CLI fallback. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') - oc delete pod $POD_NAME -n --force --grace-period=0 - ``` - -5. **Remove VMI finalizers** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VMI using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update VMI using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Patch VM runStrategy directly** (ensure consistency): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` (diagnostic step 1) - 2. Set `.spec.runStrategy` to `"Halted"` - 3. Update VM using `resources_create_or_update` with modified JSON - - **CLI Fallback** (merge patch easier via CLI): - Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Stopped` or `Halted`). - -Verify VMI is gone: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Verify virt-launcher pod is gone: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for pods matching `virt-launcher-`. Should return no results. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Stopped or Halted - -oc get vmi -n -# Should return: Error from server (NotFound) - -oc get pods -n | grep virt-launcher- -# Should return: No resources found -``` - -**Prevention**: -- Ensure guest OS has ACPI support enabled -- Use proper shutdown commands in guest OS -- Avoid forceful stops unless necessary (can corrupt guest filesystem) - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/network-errors.md deleted file mode 100644 index 82fa313c..00000000 --- a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/network-errors.md +++ /dev/null @@ -1,429 +0,0 @@ ---- -title: VM Network Errors -category: kubevirt -sources: - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] -semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] -use_cases: [vm-creation, network-troubleshooting] -related_docs: [INDEX.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Network Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. - -**When to use this document**: -- VM created successfully but secondary networks not attached -- NetworkAttachmentDefinition not found errors -- Multus CNI failures -- SR-IOV device attachment issues - -**Skills that use this**: vm-create - ---- - -### Network Attachment Failures - -**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working - -**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). - -**Possible Causes**: -- NetworkAttachmentDefinition doesn't exist in the namespace -- Multus CNI not installed or not configured on cluster -- Namespace mismatch (NAD in different namespace than VM) -- Interface name conflicts in VM spec -- Bridge/network configuration errors in NAD -- SR-IOV device not available (if using SR-IOV) - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List NetworkAttachmentDefinitions in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "" -} -``` - -Review `.items[].metadata.name` for available NADs. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -n -``` - -**2. List NetworkAttachmentDefinitions in all namespaces**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition" -} -``` - -Omit `namespace` parameter to list across all namespaces. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -A -``` - -**3. Check specific NetworkAttachmentDefinition**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "", - "name": "" -} -``` - -Review `.spec.config` for CNI configuration. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definition -n -o yaml -``` - -**4. Check VM network configuration**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.networks` to see network references. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq -``` - -**5. Check VM domain interfaces**: - -Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq -``` - -**6. Check virt-launcher pod network annotations** (shows actual attachments): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**7. Check Multus is installed**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "openshift-multus" -} -``` - -Should show Multus CNI pods running. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n openshift-multus -``` - -**8. Check for errors in virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n -``` - -**Common Error Messages**: -- `"network-attachment-definition not found"` - NAD doesn't exist in namespace -- `"multus CNI not configured"` - Multus not installed or misconfigured -- `"interface name conflict"` - Duplicate interface names in VM spec -- `"failed to add network"` - CNI plugin error (check NAD config) -- `"no available devices"` - SR-IOV device not available (if using SR-IOV) - -**Solutions** (Use MCP Tools First): - -1. **Verify NetworkAttachmentDefinition exists in correct namespace**: - - Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. - - If NAD is in different namespace, copy it to VM namespace: - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get NAD from source namespace using `resources_get` - 2. Modify `.metadata.namespace` to target namespace - 3. Create NAD in target namespace using `resources_create_or_update` - - **CLI Fallback** (stream processing easier via CLI): - Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" - ```bash - oc get network-attachment-definition -n -o yaml | \ - sed "s/namespace: /namespace: /" | \ - oc apply -f - - ``` - -2. **Create missing NetworkAttachmentDefinition**: - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - Example: Linux bridge network - - **Parameters**: - ```json - { - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "metadata": { - "name": "vlan100", - "namespace": "" - }, - "spec": { - "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" - ```bash - cat < - spec: - config: '{ - "cniVersion": "0.3.1", - "type": "bridge", - "bridge": "br1", - "vlan": 100, - "ipam": { - "type": "host-local", - "subnet": "192.168.100.0/24" - } - }' - EOF - ``` - -3. **Check Multus CNI installation**: - - Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. - - To check cluster network operator: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "config.openshift.io/v1", - "kind": "ClusterOperator" - } - ``` - - Filter for `network` operator. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n openshift-multus - oc get clusteroperators network - ``` - -4. **Fix interface name conflicts** (if VM has duplicate names): - - Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. - - Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. - - **CLI Fallback** (interactive edit easier via CLI): - Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' - oc edit vm -n - ``` - -5. **Validate NAD configuration syntax**: - - Use `resources_get` from diagnostic step 3, extract `.spec.config`. - - Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get network-attachment-definition -n -o jsonpath='{.spec.config}' - ``` - -6. **Check SR-IOV device availability** (if using SR-IOV networks): - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - List SR-IOV network node policies: - - **Parameters**: - ```json - { - "apiVersion": "sriovnetwork.openshift.io/v1", - "kind": "SriovNetworkNodePolicy", - "namespace": "openshift-sriov-network-operator" - } - ``` - - Check SR-IOV device plugin pods: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "openshift-sriov-network-operator" - } - ``` - - Filter for pods with "device-plugin" in name. - - Check available SR-IOV devices on node: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "Node", - "name": "" - } - ``` - - Review `.status.allocatable` for SR-IOV resources. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get sriovnetworknodepolicy -n openshift-sriov-network-operator - oc get pods -n openshift-sriov-network-operator | grep device-plugin - oc describe node | grep -A 10 "Allocatable:" - ``` - -7. **Recreate VM with corrected network configuration** (if needed): - - If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -After remediation, check virt-launcher pod network status: - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -Should show all attached networks with status. Example output: -```json -[ - { - "name": "openshift-sdn", - "interface": "eth0", - "ips": ["10.128.2.10"], - "default": true - }, - { - "name": "vlan100", - "interface": "net1", - "ips": ["192.168.100.5"] - } -] -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**Check from inside VM** (via console): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# In guest OS: -ip addr show -# Should show all network interfaces (eth0, net1, etc.) -``` - -**Common Network Types**: -- **Linux Bridge**: Layer 2 bridge for VLAN networks -- **SR-IOV**: High-performance direct device assignment -- **macvlan**: MAC-based VLAN for container networks -- **OVN-Kubernetes**: OpenShift native overlay network - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/runtime-errors.md deleted file mode 100644 index 804d3cbc..00000000 --- a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/runtime-errors.md +++ /dev/null @@ -1,616 +0,0 @@ ---- -title: VM Runtime Errors -category: kubevirt -sources: - - title: Kubernetes Pod Lifecycle - url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ - date_accessed: 2026-02-17 -tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] -semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] -use_cases: [vm-creation, vm-lifecycle, diagnostics] -related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Runtime Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. - -**When to use this document**: -- VM or virt-launcher pod shows `CrashLoopBackOff` status -- virt-launcher pod repeatedly restarting -- Guest OS kernel panics on boot - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### CrashLoopBackOff - -**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting - -**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. - -**Possible Causes**: -- Guest OS kernel panic on boot -- Insufficient resources (memory/CPU) for guest OS -- Corrupted disk image or filesystem -- QEMU/libvirt crashes due to configuration errors -- Missing or incompatible device drivers in guest -- Resource limits too low for virt-launcher pod -- Virtualization features (KVM) not available on node - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check virt-launcher pod restart count**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# Look at RESTARTS column -``` - -**2. View recent crash logs** (previous container instance): - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "previous": true, - "tail": 100 -} -``` - -Look for QEMU errors, kernel panics, or segfaults. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n virt-launcher--xxx --previous -``` - -**3. Check current virt-launcher logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. - -**CLI Fallback** (if MCP unavailable or all containers needed): -```bash -oc logs -n virt-launcher--xxx --all-containers -``` - -**4. Check VMI conditions for crash details**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Extract `.status.conditions` for crash details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**5. Check pod events for crash reasons**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" -``` - -**6. Check pod resource limits**: - -**MCP Tool**: `pods_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -Extract `.spec.containers[0].resources` for resource limits. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq -``` - -**7. Check node kubelet logs for OOM kills**: - -⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -oc adm node-logs -u kubelet | grep -i oom -``` - -**8. Access guest console** (if VM briefly starts): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Crash Patterns**: - -1. **Guest Kernel Panic**: - - Console logs show kernel panic messages - - Guest crashes immediately after boot - - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" - -2. **OOM (Out of Memory)**: - - Pod killed with reason: `OOMKilled` - - Guest runs out of memory during boot or operation - - virt-launcher logs show memory allocation failures - -3. **QEMU Crash**: - - virt-launcher logs show QEMU segmentation fault - - Symptoms: "qemu-system-x86_64: terminated by signal" - - Configuration incompatibility or QEMU bug - -4. **Disk Image Corruption**: - - Guest cannot boot from disk - - Filesystem errors in guest console - - DataVolume import failed - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for kernel panic or boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for: - - Kernel panic messages - - Initramfs errors - - Filesystem mounting failures - - Missing device errors - -2. **Review virt-launcher crash logs**: - - Use `pods_log` with `previous: true` from diagnostic step 2. - - Look for: - - QEMU command line errors - - Device initialization failures - - Memory allocation errors - - Signal termination (SIGSEGV, SIGABRT) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx --previous - ``` - -3. **Check for OOM (Out of Memory) kills**: - - **MCP Tool**: `pods_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - Extract `.status.containerStatuses[0].lastState.terminated.reason`. - - If returns `"OOMKilled"`: - - Option 1: Increase virt-launcher memory limits - - Option 2: Decrease guest memory allocation - - Option 3: Use smaller instance type - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' - ``` - -4. **Increase resources if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check current memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -5. **Verify disk image integrity**: - - **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters for list**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" - } - ``` - - **Parameters for specific DV**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check `.status.phase` (should be `Succeeded`). - - If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get datavolume -n -o jsonpath='{.status.phase}' - ``` - -6. **Check virtualization (KVM) availability**: - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -7. **Simplify VM configuration** (eliminate variables): - - Try creating minimal VM using vm-create skill with: - - Small instance type - - No secondary networks - - Simple container disk (e.g., Fedora) - - No cloud-init - - If minimal VM works, add features back one by one. - -8. **Recreate VM with different workload** (test disk image): - - If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) - -After remediation, check pod restart count stops increasing: - -**Parameters for pods**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). - -Check VM reaches Running state: - -**Parameters for VM**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# RESTARTS should stabilize - -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Running -``` - -Verify guest is responsive: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# Should show login prompt or OS console -``` - -**Advanced Debugging**: - -**MCP Tool**: `pods_exec` (from openshift-virtualization) - -Check libvirt domain XML: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["virsh", "dumpxml", "1"] -} -``` - -Check QEMU process: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["ps", "aux"] -} -``` - -Filter output for "qemu" process. - -**CLI Fallback** (if MCP unavailable): -```bash -oc exec -n virt-launcher--xxx -- virsh dumpxml 1 -oc exec -n virt-launcher--xxx -- ps aux | grep qemu -``` - -**Prevention**: -- Start with minimal VM configuration and add complexity gradually -- Use recommended instance types for your workload -- Test disk images before deploying to production -- Ensure nodes have adequate resources and KVM support -- Monitor resource usage and set appropriate limits - ---- - -## Workaround Patterns for MCP Tool Limitations - -### General Pattern: Diagnose → Propose → Confirm → Execute - -When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): - -**1. Diagnose** the root cause using Kubernetes commands -```bash -oc describe vm -n -oc get events -n --field-selector involvedObject.name= -``` - -**2. Propose** a manual workaround to the user with clear explanation -- Explain why the issue occurred -- Show what will be changed -- List alternative options - -**3. Confirm** with user before executing (Human-in-the-Loop) -- Wait for explicit "yes" or "apply workaround" -- Never auto-execute modifications - -**4. Execute** the workaround using oc/kubectl -```bash -oc patch vm -n --type=merge -p '...' -``` - -**5. Verify** the fix was successful -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**6. Document** the limitation and suggest filing enhancement request -- Note this is temporary until MCP tool is enhanced -- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues - -### Example: Adding Tolerations Workaround - -**Diagnostic Output**: -```markdown -## ⚠️ VM Scheduling Issue Detected - -**Root Cause**: Node taints prevent VM scheduling - -**Details**: -- Found 3 nodes with taint: `virtualization=true:NoSchedule` -- VM spec does not include matching tolerations -- This prevents VM scheduling on virtualization-dedicated nodes -``` - -**Proposed Workaround**: -```bash -oc patch vm web-server -n vms --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' -``` - -**User Confirmation Required**: -``` -How would you like to proceed? -- "apply workaround" - I'll patch the VM with tolerations -- "manual" - I'll provide instructions for you to apply manually -- "cancel" - Delete the VM and abort creation -``` - -**After Execution**: -```markdown -## ✓ Workaround Applied Successfully - -**Action**: Added tolerations for taint `virtualization=true:NoSchedule` -**New Status**: Stopped (VM can now be scheduled) - -**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. -Future VMs in this cluster will need the same fix until the tool is enhanced. -``` - ---- - -## VM Status Reference - -### Status Values - -| Status | Meaning | Action Required | -|--------|---------|-----------------| -| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | -| `Running` | VM is running | Normal | -| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | -| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | -| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | -| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | -| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | -| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | -| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | -| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | - -### Checking VM Status - -```bash -# Get printable status -oc get vm -n -o jsonpath='{.status.printableStatus}' - -# Get detailed status and conditions -oc get vm -n -o jsonpath='{.status}' | jq - -# Watch status changes in real-time -oc get vm -n -w -``` - ---- - -## Best Practices for Agents - -When implementing diagnostic workflows: - -1. **Always verify VM status** after creation (wait 5-10 seconds first) -2. **Consult this document** when encountering error status values -3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) -4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) -5. **Respect human-in-the-loop** for all VM modifications -6. **Document temporary workarounds** and their limitations clearly -7. **Suggest filing issues** for missing MCP tool features - -### Document Consultation Pattern - -```markdown -**Document Consultation** (REQUIRED): -1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes -2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." -``` - ---- - -## Known MCP Tool Limitations - -### vm_create tool - -**Currently Supported**: -- ✓ Namespace, name (required) -- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) -- ✓ Size hints (small, medium, large) -- ✓ Storage size -- ✓ Autostart flag -- ✓ Networks (Multus NetworkAttachmentDefinitions) -- ✓ Performance family (u1, o1, c1, m1) -- ✓ Instance type, preference - -**Not Currently Supported** (requires workarounds): -- ✗ Tolerations (for node taints) -- ✗ Node selectors -- ✗ Affinity/anti-affinity rules -- ✗ Resource requests/limits (beyond instance type) -- ✗ Custom labels/annotations -- ✗ SSH keys injection -- ✗ Cloud-init user data - -**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. - -**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues - ---- - -## Additional Resources - -- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) -- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) -- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) -- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) - ---- - -[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/scheduling-errors.md deleted file mode 100644 index e47c8c4f..00000000 --- a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/scheduling-errors.md +++ /dev/null @@ -1,417 +0,0 @@ ---- -title: VM Scheduling Errors -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 -tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] -semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] -use_cases: [vm-creation, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Scheduling Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**When to use this document**: -- VM shows status `ErrorUnschedulable` after creation or start attempt -- VM events mention scheduling failures, taints, resources, or node selectors - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -## ErrorUnschedulable - -**Symptom**: VM shows status `ErrorUnschedulable` after creation - -**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**Possible Causes**: - -### 1. Node Taints (Most Common) - -Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM events for scheduling failures**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `involvedObject.name` == "" and look for messages like: -- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= -``` - -**2. Check node taints in the cluster**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' -``` - -**Common Taint Patterns**: -- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule -- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes -- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads - -**Solution - Add Tolerations to VM**: - -The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. - -**Workaround (post-creation using MCP Tools)**: - -**Step 1**: Get current VM spec - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Step 2**: Modify the returned JSON to add tolerations - -Add to `.spec.template.spec.tolerations`: -```json -{ - "tolerations": [ - { - "key": "virtualization", - "operator": "Equal", - "value": "true", - "effect": "NoSchedule" - } - ] -} -``` - -**Step 3**: Update VM with modified spec - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters**: -```json -{ - "resource": "" -} -``` - -Pass the complete modified VM resource as YAML or JSON string. - -**Step 4**: Verify tolerations were added - -Use `resources_get` again and check `.spec.template.spec.tolerations` in response. - -**Step 5**: Check if VM status improved - -Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. - -**CLI Fallback** (if MCP patch is too complex): -```bash -# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' - -# Verify tolerations -oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq - -# Check status -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**Example - Multiple Tolerations**: -```bash -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" - - key: "dedicated" - operator: "Equal" - value: "virt-workloads" - effect: "NoSchedule" -' -``` - -**Toleration Operators**: -- `Equal` - Key and value must match exactly -- `Exists` - Only key must exist (ignores value) - -**Toleration Effects**: -- `NoSchedule` - Don't schedule new pods (existing pods continue) -- `PreferNoSchedule` - Avoid scheduling if possible -- `NoExecute` - Don't schedule AND evict existing pods - -**Alternative Solutions**: -1. **Remove node taints** (if you have cluster-admin access): - ```bash - oc adm taint nodes virtualization=true:NoSchedule- - ``` - -2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits - -3. **File enhancement request** - Request tolerations support in openshift-mcp-server: - https://github.com/openshift/openshift-mcp-server/issues - ---- - -### 2. Insufficient Resources - -Not enough CPU, memory, or storage available on any node. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM resource requests**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' -``` - -**2. Check node resource availability**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. - -Alternatively, use `nodes_top` MCP tool for current resource usage. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe nodes | grep -A 5 "Allocated resources" -``` - -**3. Look for VM events mentioning "Insufficient"**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep "Insufficient" -``` - -**Example Event**: -``` -0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. -``` - -**Solutions** (Use MCP Tools First): - -1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) -2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill -3. **Delete unused VMs** - Use vm-delete skill to free up resources -4. **Check resource quotas**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters for quota**: - ```json - { - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" - } - ``` - - **Parameters for limit range**: - ```json - { - "apiVersion": "v1", - "kind": "LimitRange", - "namespace": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - oc describe limitrange -n - ``` - ---- - -### 3. Node Selector Mismatch - -VM requires specific node labels that don't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM node selector requirements**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.nodeSelector` to see required node labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' -``` - -**2. List available node labels**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.metadata.labels` for available labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes --show-labels -``` - -**3. Check if any nodes match the selector**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node", - "labelSelector": "=" -} -``` - -Should return at least one node with matching labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -l = -``` - -**Solutions** (Use MCP Tools First): - -**Option 1: Remove node selector from VM** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get current VM using `resources_get` (diagnostic step 1) -2. Remove `.spec.template.spec.nodeSelector` field -3. Update VM using `resources_create_or_update` with modified JSON - -**CLI Fallback** (JSON patch easier via CLI): -Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" -```bash -oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' -``` - -**Option 2: Add label to nodes** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get node using `resources_get` -2. Add label to `.metadata.labels` -3. Update node using `resources_create_or_update` - -⚠️ **Note**: Node labeling typically requires cluster admin privileges. - -**CLI Fallback** (simpler via CLI): -Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" -```bash -oc label node = -``` - ---- - -[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/storage-errors.md deleted file mode 100644 index 16ba584d..00000000 --- a/evaluation/with_skills/rh-virt__vm-clone/environment/docs/troubleshooting/storage-errors.md +++ /dev/null @@ -1,1011 +0,0 @@ ---- -title: VM Storage Errors -category: kubevirt -sources: - - title: KubeVirt CDI - DataVolumes - url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ - date_accessed: 2026-02-17 - - title: Kubernetes Persistent Volumes - url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ - date_accessed: 2026-02-17 -tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] -semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] -use_cases: [vm-creation, vm-deletion, vm-cloning] -related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] -last_updated: 2026-02-17 ---- - -# VM Storage Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. - -**When to use this document**: -- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` -- Storage deletion fails after VM deletion -- DataVolume cloning operations fail -- PVC provisioning issues - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### ErrorDataVolumeNotReady - -**Symptom**: VM shows status `ErrorDataVolumeNotReady` - -**Description**: The DataVolume (persistent storage) backing the VM is not ready. - -**Possible Causes**: - -#### 1. DataVolume Still Provisioning - -Storage provisioning takes time, especially for large disks or when importing images. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Get detailed DataVolume information**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` and `.status.conditions` for provisioning details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o json -``` - -**3. Check PVC (PersistentVolumeClaim) bound status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Check `.status.phase` for each PVC (should be `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. - -#### 2. Storage Class Not Found - -The requested storage class doesn't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List available storage classes**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" -} -``` - -Review the list of available storage classes (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -``` - -**2. Check DataVolume's requested storage class**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.spec.pvc.storageClassName` in the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' -``` - -**Solution**: -1. Use a valid storage class from the cluster -2. Recreate VM with correct storage class parameter - -#### 3. Insufficient Storage Quota - -Namespace has insufficient storage quota to provision the PVC. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check resource quotas**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**2. Check storage usage**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase -``` - -**Solution**: -1. Request quota increase from cluster admin -2. Delete unused PVCs to free quota -3. Reduce VM storage size - ---- - -### ErrorPvcNotFound - -**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List PVCs in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review the list of available PVCs (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check VM's PVC references**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' -``` - -**Solution**: -- Wait for DataVolume to create the PVC -- Manually create missing PVC -- Fix VM spec to reference correct PVC name - ---- - - ---- - -### Storage Deletion Failures - -**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace - -**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. - -**Possible Causes**: -- PVC still bound to active PersistentVolume with `Retain` policy -- DataVolume still being referenced by another resource -- CDI (Containerized Data Importer) controller issues -- Storage class retention policy preventing deletion -- Finalizers on PVC/DataVolume blocking cleanup -- PVC still mounted by a pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check PVC status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check specific PVC phase**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Released` or `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o jsonpath='{.status.phase}' -``` - -**3. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**4. Check what's using the PVC**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' -``` - -**5. Check PVC finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. - -**6. Check DataVolume finalizers**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.metadata.finalizers}' -``` - -**7. Check PV reclaim policy**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume" -} -``` - -Filter results for PV where `.spec.claimRef.name` matches ``. - -To get specific PV policy: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume", - "name": "" -} -``` - -Check `.spec.persistentVolumeReclaimPolicy`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pv | grep -oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' -``` - -**Common Finalizer Patterns**: -- `kubernetes.io/pvc-protection` - Protects PVC while in use -- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer - -**Solutions** (Use MCP Tools First): - -1. **Delete DataVolume first, then PVC**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete DataVolume first (often blocks PVC deletion): - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Wait a few seconds, then delete PVC: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete datavolume -n - oc delete pvc -n - ``` - -2. **Check for pods still using PVC**: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "" - } - ``` - - Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. - - Then delete the pods using `pods_delete`: - - **Parameters**: - ```json - { - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' - oc delete pod -n - ``` - -3. **Force delete PVC** (if safe to do so): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete pvc -n --grace-period=0 --force - ``` - -4. **Remove finalizers from PVC** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PVC using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update PVC using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Remove finalizers from DataVolume** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current DataVolume using `resources_get` (diagnostic step 6) - 2. Remove items from `.metadata.finalizers` array - 3. Update DataVolume using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Change PV reclaim policy** (if PV has Retain policy): - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PV policy using `resources_get` (diagnostic step 7) - 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` - 3. Update PV using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" - ```bash - oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' - oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' - ``` - -**Storage Quota Check** (Use MCP Tools First): - -After deletion, verify storage quota is freed: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters for quota check**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.used` to verify storage quota is freed. - -**Parameters for PVC verification**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -oc get pvc -n -``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Confirm PVC is deleted: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Confirm DataVolume is deleted: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -# Should return: Error from server (NotFound) - -oc get datavolume -n -# Should return: Error from server (NotFound) -``` - ---- - -### DataVolume Cloning Failures - -**Symptom**: VM clone created successfully but DataVolume clone operation fails - -**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. - -**Possible Causes**: -- CSI driver doesn't support volume cloning -- Source PVC storage class incompatible with cloning -- Cross-namespace cloning not permitted by storage backend -- Insufficient storage quota in target namespace -- Source PVC not in `Bound` state -- Storage class doesn't have volume cloning enabled -- CDI (Containerized Data Importer) controller issues - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume clone status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and `.metadata.name`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Check DataVolume events for errors**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe datavolume -n -``` - -**3. Check DataVolume phase**: - -Use `resources_get` from step 1, extract `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -``` - -**4. Check if storage class supports cloning**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass", - "name": "" -} -``` - -Review the full YAML output for cloning-related configurations. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -o yaml | grep -A 5 -i clone -``` - -**5. Check CSI driver capabilities**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "CSIDriver" -} -``` - -Review `.items[].metadata.name` for available CSI drivers. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get csidriver -``` - -**6. Check source PVC status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Bound` for cloning to work). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**7. Check target namespace storage quota**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**8. Check CDI controller logs**: - -**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) - -First, list pods in openshift-cnv namespace: - -**Parameters for pods_list_in_namespace**: -```json -{ - "namespace": "openshift-cnv", - "labelSelector": "app.kubernetes.io/component=cdi-deployment" -} -``` - -Then get logs using `pods_log`: - -**Parameters**: -```json -{ - "namespace": "openshift-cnv", - "name": "", - "tail": 100 -} -``` - -**CLI Fallback** (if MCP unavailable or easier via CLI): -```bash -oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') -``` - -**Common Error Messages**: -- `"volume cloning is not supported"` - CSI driver lacks clone capability -- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage -- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace -- `"insufficient quota"` - Target namespace lacks storage quota -- `"source PVC not bound"` - Source PVC must be in Bound state for cloning -- `"StorageClass does not support cloning"` - Storage class configuration issue - -**Solutions** (Use MCP Tools First): - -1. **Check storage class clone support**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" - } - ``` - - For each storage class in `.items[]`, check: - - `.metadata.name` (storage class name) - - `.provisioner` (CSI driver) - - Storage classes using CSI drivers typically support cloning. Look for provisioners like: - - `csi.ovirt.org` (oVirt CSI) - - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) - - `ebs.csi.aws.com` (AWS EBS CSI) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner - ``` - -2. **Verify source PVC is bound**: - - Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pvc -n -o jsonpath='{.status.phase}' - ``` - -3. **Check target namespace quota**: - - Use `resources_list` from diagnostic step 7 to check quota. - - If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - ``` - -4. **Use snapshot-based cloning** (alternative method): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Step 1**: Create VolumeSnapshot of source PVC - - **Parameters**: - ```json - { - "apiVersion": "snapshot.storage.k8s.io/v1", - "kind": "VolumeSnapshot", - "metadata": { - "name": "-snapshot", - "namespace": "" - }, - "spec": { - "source": { - "persistentVolumeClaimName": "" - } - } - } - ``` - - **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) - - **Step 3**: Create new DataVolume from snapshot - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "metadata": { - "name": "-rootdisk", - "namespace": "" - }, - "spec": { - "source": { - "snapshot": { - "name": "-snapshot", - "namespace": "" - } - }, - "storage": { - "resources": { - "requests": { - "storage": "50Gi" - } - }, - "storageClassName": "" - } - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" - ```bash - cat <-snapshot - namespace: - spec: - source: - persistentVolumeClaimName: - EOF - - oc get volumesnapshot -snapshot -n - - cat <-rootdisk - namespace: - spec: - source: - snapshot: - name: -snapshot - namespace: - storage: - resources: - requests: - storage: 50Gi - storageClassName: - EOF - ``` - -5. **Use "new empty storage" option** (vm-clone skill): - - If cloning isn't supported, create VM with empty storage - - Manually copy data if needed - -6. **Cross-namespace cloning workaround**: - - Some storage backends require snapshot for cross-namespace cloning - - Create snapshot in source namespace, restore in target namespace (see solution 4 above) - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Check DataVolume reached Succeeded phase: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should return `Succeeded`). - -Check PVC was created and bound: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "-rootdisk" -} -``` - -Check `.status.phase` (should return `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -# Should return: Succeeded - -oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' -# Should return: Bound -``` - -**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-clone/environment/skills/vm-clone/SKILL.md b/evaluation/with_skills/rh-virt__vm-clone/environment/skills/vm-clone/SKILL.md deleted file mode 100644 index 09ad791f..00000000 --- a/evaluation/with_skills/rh-virt__vm-clone/environment/skills/vm-clone/SKILL.md +++ /dev/null @@ -1,456 +0,0 @@ ---- -name: vm-clone -description: | - Clone existing virtual machines for testing, scaling, or creating templates. - - Use when: - - "Clone VM [source] to [target]" - - "Create a copy of VM [name]" - - "Duplicate VM [name] for testing" - - "Create 3 copies of template-vm" - - This skill clones VM configuration and optionally creates new storage or references existing storage. - - NOT for snapshots (use vm-snapshot for point-in-time backups). - -model: inherit -color: blue ---- - -# /vm-clone Skill - -Clone existing virtual machines in OpenShift Virtualization, creating new VMs with copied configuration and optional storage cloning. This skill is ideal for creating test environments, scaling workloads, or duplicating VM templates. - -## Prerequisites - -**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools**: -- `resources_get` (from openshift-virtualization) - Get source VM configuration -- `resources_create_or_update` (from openshift-virtualization) - Create cloned VM -- `resources_list` (from openshift-virtualization) - List DataVolumes, PVCs, VMs - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster (>= 4.19) -- OpenShift Virtualization operator installed -- ServiceAccount with RBAC permissions to create VirtualMachine and PVC resources -- Source VM must exist - -### Prerequisite Verification - -**Before executing:** -1. Verify `openshift-virtualization` in `.mcp.json`, `KUBECONFIG` set (never expose value) -2. Optional: Verify RBAC permissions for VirtualMachine, PVC/DataVolume creation - -**When prerequisites fail:** -``` -❌ Cannot execute vm-clone: MCP server 'openshift-virtualization' is not available - -Setup: Add openshift-virtualization to .mcp.json (see https://github.com/openshift/openshift-mcp-server) -Set KUBECONFIG environment variable, restart Claude Code - -Options: "setup" (configure now), "skip" (skip skill), "abort" (stop workflow) -``` - -⚠️ **SECURITY**: Never display KUBECONFIG path or credential values. - -## When to Use This Skill - -**Trigger this skill when:** -- User explicitly invokes `/vm-clone` command -- User wants to duplicate an existing VM -- User needs to create test/dev copies of production VMs -- User wants to scale horizontally by creating VM copies -- User wants to create VMs from a template VM - -**User phrases that trigger this skill:** -- "Clone VM web-server to web-server-test" -- "Create a copy of database-vm" -- "Duplicate production-vm for staging" -- "Make 3 copies of template-vm" -- "/vm-clone" (explicit command) - -**Do NOT use this skill when:** -- User wants to create a new VM from scratch → Use `/vm-create` skill instead -- User wants a point-in-time backup → Use snapshots instead -- User wants to move/migrate a VM → Use migration tools instead -- User wants to resize a VM → Modify existing VM instead - -## Workflow - -### Step 1: Gather Source VM Information - -**Required Information from User:** -1. **Source VM Name** - Name of the VM to clone -2. **Source Namespace** - Namespace where source VM exists -3. **Target VM Name** - Name for the cloned VM -4. **Target Namespace** - Namespace for the cloned VM (can be same or different) - -If user doesn't provide all information, ask for missing details. - -**1.1: Verify Source VM Exists** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Expected Output**: Complete VirtualMachine resource specification - -**Error Handling**: -- If VM not found → Report error, suggest using vm-inventory to find VMs -- If permission denied → Report RBAC error - -**1.2: Check Target VM Name Availability** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**If VM exists**: Offer options (choose different name, delete existing VM, cancel), wait for decision - -**1.3: Discover Source VM Storage** - -Use `resources_list` for DataVolumes (labelSelector: vm.kubevirt.io/name) or PVCs if not found -Parse: Extract storage names, calculate size, determine DataSources vs container disks - -### Step 2: Ask User for Cloning Strategy - -**Present storage cloning options:** - -```markdown -## VM Cloning - Storage Strategy - -**Source VM**: `` (namespace: ``) -**Source Storage**: () - -**Select cloning strategy:** - -1. **Clone Storage** - Full copy, independent storage (~5-10 min, new allocation) -2. **Reference Existing** - ⚠️ Shared disk (dangerous, both VMs access same storage) -3. **New Empty Storage** - Fresh disk, no data copied ( new allocation) -4. **Cancel** - Abort operation - -**Select option (1-4):** -``` - -**Wait for user selection (1-4).** - -**Handle response**: "4"/cancel → stop; "1" → clone_storage=true; "2" → warn + share_storage=true; "3" → new_storage=true - -**Option 2 warning**: `⚠️ Shared Storage Dangerous - Both VMs share disk, data corruption risk. Only safe if source stopped. Use Option 1 instead. Proceed anyway? (yes/cancel)` Wait for explicit "yes". - -### Step 3: Present Clone Configuration for Confirmation - -**Present configuration summary:** - -```markdown -## VM Clone Configuration - Review - -**Source**: `` () - , vCPU, -**Target**: `` () - Same config, starts Stopped -**Storage**: - - -**Resource Impact**: vCPU, RAM, disk - -**What changes**: IP addresses, hostname, MAC addresses, firmware UUID -**What's preserved**: Instance type, vCPU/memory, network config, cloud-init - -**Proceed with VM cloning? (yes/no)** -``` - -**Wait for user confirmation.** - -**Handle response:** -- If "yes" → Proceed to Step 4 (execute cloning) -- If "no", "cancel", "wait", or anything else → Cancel operation - -**On cancellation:** -```markdown -VM cloning cancelled by user. No resources were created. -``` - -**STOP workflow**. - -### Step 4: Execute VM Cloning - -**ONLY PROCEED AFTER**: Source VM validated, target name available, user selected storage strategy, user confirmed configuration - -**4.1: Prepare Cloned VM Specification** - -Modify source VM spec: -1. Change metadata: `name` → target-vm-name, `namespace` → target-namespace; remove `uid`, `resourceVersion`, `creationTimestamp`, `status` -2. Update storage: clone_storage → new DataVolume with source PVC; share_storage → keep PVC refs; new_storage → empty DataVolume -3. Set `runStrategy: Halted` (starts stopped) -4. Generate new firmware UUIDs (`domain.firmware.uuid`, `domain.firmware.serial`) -5. Preserve: instance type, tolerations, network config, cloud-init - -**4.2: Create Storage Resources** - -**Clone storage** - Use `resources_create_or_update` with DataVolume (source.pvc from source, storage from source class/size) -**New empty storage** - Use `resources_create_or_update` with DataVolume (source.blank, storage from source class/size) - -**4.3: Create Cloned VirtualMachine** - -Use `resources_create_or_update` with prepared spec from 4.1 -**Error handling**: Creation fails → report error, rollback storage; permission denied → RBAC error; namespace missing → namespace error - -**4.4: Monitor Storage Cloning Progress** - -Use `resources_get` on DataVolume, check `status.phase` (Pending/Succeeded/Failed), report every 30s, wait up to 15 min - -### Step 5: Report Cloning Results - -**On successful clone:** - -```markdown -## ✓ VM Cloned Successfully - -**Source**: `` () -**Target**: `` () - Status: Stopped (ready to start) -**Config**: , vCPU, , - - -**Storage**: ✓ Cloned in - -**Storage**: ✓ New empty storage created - OS installation may be required - - -**Storage**: ⚠️ Shared PVC `` - Keep source VM stopped to avoid data corruption - - -**Next**: Start with `"Start VM in namespace "` -``` - -**On cloning failure:** - -**Document Consultation** (OPTIONAL - when cloning fails): -- **When to consult**: Storage cloning fails, VM creation fails, PVC clone not supported, storage class issues -- **When NOT to consult**: VM already exists, RBAC errors, namespace not found (clear causes) -- **Action**: Read [storage-errors.md](../../docs/troubleshooting/storage-errors.md) for VM cloning failures, storage provisioning, DataVolume errors -- **Output to user**: "I consulted [storage-errors.md](../../docs/troubleshooting/storage-errors.md) to understand potential causes." - -```markdown -## ❌ VM Cloning Failed - -**Error**: -**Source**: `` () → **Target**: `` () - -**Common Causes**: -- Insufficient storage quota - Namespace lacks storage capacity -- Insufficient RBAC permissions - ServiceAccount lacks create permissions -- Storage class not available - Target namespace cannot access storage class -- PVC clone not supported - Storage class doesn't support cloning -- Source VM still running - Some storage backends require source VM stopped - -**Troubleshooting** (see [storage-errors.md](../../docs/troubleshooting/storage-errors.md)): -1. Check storage quota: `resources_list` for ResourceQuota in target namespace -2. Check permissions: `resources_list` to verify RBAC (note: `oc auth can-i` has no MCP equivalent) -3. Check storage class: `resources_get` for StorageClass config, `resources_list` for available classes -4. Check source VM status: vm-inventory skill `"Show status of VM "` -5. Check DataVolume status: `resources_get` for DataVolume phase and status - -**Partial Resources** (may need cleanup): -- VirtualMachine: `` -- DataVolume: `-rootdisk` - -**Cleanup**: `"Delete VM in namespace "` - -Would you like help troubleshooting this error? -``` - -## Advanced Features - -### Batch Cloning -**User request:** "Create 3 copies of template-vm named web-01, web-02, web-03" -**Workflow**: Validate source once, generate/check target names, present combined scope, ask storage strategy, confirm, execute sequentially -**Batch confirmation**: Show source, targets list, strategy, total impact (VMs, storage, vCPU, memory), estimated time - -### Cross-Namespace Cloning -**User request:** "Clone production-vm from production to staging namespace" -**Note**: Storage cloned across namespaces, network policies/quotas may differ, RBAC required in both namespaces - -### Clone with Modifications (Future) -Allow modifications during clone: instance type/size, storage size, network config, cloud-init customization - -## Common Issues - -**Issue 1: Target VM Name Already Exists** - Choose different name, delete existing VM (if safe), use vm-inventory to check - -**Issue 2: Insufficient Storage Quota** - Check quotas, request increase, use shared storage (if appropriate), delete unused PVCs - -**Issue 3: Storage Class Not Accessible** - Verify storage class exists in target namespace, check cross-namespace cloning support, use different storage class, contact admin - -**Issue 4: PVC Clone Not Supported** - Storage class doesn't support CSI volume cloning; use "new empty storage" option, snapshot and restore, or check storage class capabilities - -**Issue 5: Source VM Running During Clone** - Stop source VM first, use snapshot-based cloning, check storage backend requirements - -## Dependencies - -### Required MCP Servers -- `openshift-virtualization` - OpenShift MCP server with core and kubevirt toolsets - -### Required MCP Tools -- `resources_get` (from openshift-virtualization) - Get source VM and storage details - - Parameters: apiVersion, kind, namespace, name - - Source: https://github.com/openshift/openshift-mcp-server/blob/main/pkg/toolsets/core/resources.go - -- `resources_create_or_update` (from openshift-virtualization) - Create cloned VM and storage - - Parameters: resource (YAML/JSON) - - Source: https://github.com/openshift/openshift-mcp-server/blob/main/pkg/toolsets/core/resources.go - -- `resources_list` (from openshift-virtualization) - List DataVolumes, PVCs, VMs - - Parameters: apiVersion, kind, namespace, labelSelector - - Source: https://github.com/openshift/openshift-mcp-server/blob/main/pkg/toolsets/core/resources.go - -### Related Skills -- `vm-create` - Create new VMs from scratch (alternative to cloning) -- `vm-inventory` - List and verify source/target VMs -- `vm-lifecycle-manager` - Start cloned VMs after creation -- `vm-delete` - Clean up failed clones or unwanted copies - -### Reference Documentation -- [storage-errors.md](../../docs/troubleshooting/storage-errors.md) - VM cloning failure scenarios, storage provisioning issues, and DataVolume cloning errors (optionally consulted when cloning operations fail) -- [Troubleshooting INDEX](../../docs/troubleshooting/INDEX.md) - Navigation hub for discovering additional error categories when encountering unexpected issues outside the categories above -- [OpenShift Virtualization Cloning](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt/virtual_machines/cloning_vms/virt-cloning-vm.html) -- [DataVolume Cloning](https://github.com/kubevirt/containerized-data-importer/blob/main/doc/datavolumes.md#cloning) -- [KubeVirt VirtualMachine API](https://kubevirt.io/api-reference/) -- [CSI Volume Cloning](https://kubernetes.io/docs/concepts/storage/volume-pvc-datasource/) - -## Critical: Human-in-the-Loop Requirements - -**IMPORTANT:** This skill creates new resources that consume cluster capacity. You MUST: - -1. **Before Cloning** - - Verify source VM exists and get full configuration - - Ask user for clone configuration (name, namespace, storage strategy) - - Present clone preview with resource impact - - Wait for explicit user confirmation - -2. **Configuration Confirmation** - - Display source VM details - - Show target VM configuration - - Indicate storage cloning strategy - - Estimate resource consumption (CPU, memory, storage) - - Ask: "Proceed with VM cloning? (yes/no)" - - Wait for explicit "yes" - -3. **Never Auto-Execute** - - **NEVER clone without user confirmation** - - **NEVER assume storage strategy** - always ask user - - **NEVER proceed if user says "no", "wait", "cancel"** - -**Why This Matters:** -- **Resource Consumption**: Clones consume cluster resources (CPU, memory, storage) -- **Storage Costs**: Storage cloning can consume significant disk space -- **Naming Conflicts**: Duplicate names cause errors -- **Network Configuration**: May need adjustment for clones - -## Security Considerations - -- **RBAC Enforcement**: Requires create permissions in target namespace -- **Namespace Isolation**: Cannot clone across namespaces without proper permissions -- **Storage Quotas**: Respects namespace storage quotas -- **MAC Address Generation**: Automatic generation prevents MAC conflicts -- **UUID Generation**: New firmware UUIDs prevent conflicts -- **No Credential Exposure**: KUBECONFIG never exposed in output -- **Audit Trail**: All cloning operations logged in Kubernetes API audit logs - -## Example Usage - -### Example 1: Simple Clone with Storage - -``` -User: "Clone web-server to web-server-test in namespace dev" - -Agent: [Validates source VM web-server (production), checks target name availability] - [Discovers 50Gi storage, presents storage options] - "Select storage strategy (1-4):" - -User: "1" - -Agent: [Shows: Source web-server (u1.medium, 2 vCPU, 4Gi) → Target web-server-test (dev, 50Gi clone)] - "Proceed with VM cloning? (yes/no)" - -User: "yes" - -Agent: "📦 Creating storage... ⏳ Cloning (5-10 min)..." - "✓ VirtualMachine web-server-test created in dev" - "⏳ Storage cloning... Succeeded" - "✓ VM Cloned Successfully - Source: web-server (production), Target: web-server-test (dev)" - "Storage: ✓ Cloned in 7m23s - Independent storage" - "Next: Start VM web-server-test in namespace dev" -``` - -### Example 2: Clone with New Empty Storage - -``` -User: "Clone template-vm to app-01 with empty storage" - -Agent: [Validation complete, presents options] - "Select storage strategy (1-4):" - -User: "3" - -Agent: "Proceed with VM cloning? (yes/no)" - -User: "yes" - -Agent: "✓ VM Cloned Successfully" - "Storage: ✓ New empty storage created - OS installation may be required" -``` - -### Example 3: Batch Cloning - -``` -User: "Create 3 copies of template-vm named web-01, web-02, web-03 in production" - -Agent: [Validates source, checks all names, presents batch review] - "Source: template-vm, Targets: 3 VMs (web-01, web-02, web-03)" - "Total Impact: 90Gi, 6 vCPU, 12Gi, ~20-30 min" - "Proceed? (yes/no)" - -User: "yes" - -Agent: "📦 Cloning VM 1 of 3: web-01... ✓" - "📦 Cloning VM 2 of 3: web-02... ✓" - "📦 Cloning VM 3 of 3: web-03... ✓" - "✓ Batch Cloning Completed - 3 VMs, 90Gi storage, all Stopped" -``` - -### Example 4: Shared Storage Warning - -``` -User: "Clone db-vm to db-test" - -Agent: "Select storage strategy (1-4):" - -User: "2" - -Agent: "⚠️ WARNING: Shared Storage is Dangerous" - "Risks: Data corruption if both VMs run simultaneously" - "Proceed with shared storage anyway? (yes/cancel)" - -User: "cancel" - -Agent: "VM cloning cancelled. No resources created." -``` diff --git a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/.ai-index/semantic-index.json deleted file mode 100644 index 2512c2a3..00000000 --- a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/.ai-index/semantic-index.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "index_version": "1.0.0", - "last_updated": "2026-02-17", - "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", - - "documents": [ - { - "file": "INDEX.md", - "title": "VM Troubleshooting Guide - Index", - "category": "navigation", - "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], - "use_cases": ["navigation", "tool-reference", "general-guidance"], - "token_estimate": 450, - "skills": ["all"] - }, - { - "file": "scheduling-errors.md", - "title": "VM Scheduling Errors", - "category": "scheduling", - "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], - "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], - "errors_covered": [ - "ErrorUnschedulable - Node Taints", - "ErrorUnschedulable - Insufficient Resources", - "ErrorUnschedulable - Node Selector Mismatch" - ], - "token_estimate": 820, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "storage-errors.md", - "title": "VM Storage Errors", - "category": "storage", - "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], - "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], - "errors_covered": [ - "ErrorDataVolumeNotReady - DataVolume Still Provisioning", - "ErrorDataVolumeNotReady - Storage Class Not Found", - "ErrorDataVolumeNotReady - Insufficient Storage Quota", - "ErrorPvcNotFound", - "Storage Deletion Failures", - "DataVolume Cloning Failures" - ], - "token_estimate": 1350, - "skills": ["vm-creator", "vm-delete", "vm-clone"] - }, - { - "file": "lifecycle-errors.md", - "title": "VM Lifecycle Errors", - "category": "lifecycle", - "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], - "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], - "errors_covered": [ - "VM Stuck in Terminating State", - "VM Won't Start (Non-Scheduling Issues)", - "VM Won't Stop" - ], - "token_estimate": 1150, - "skills": ["vm-delete", "vm-lifecycle-manager"] - }, - { - "file": "runtime-errors.md", - "title": "VM Runtime Errors", - "category": "runtime", - "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], - "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], - "errors_covered": [ - "CrashLoopBackOff" - ], - "token_estimate": 850, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "network-errors.md", - "title": "VM Network Errors", - "category": "networking", - "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], - "use_cases": ["vm-creation", "network-troubleshooting"], - "errors_covered": [ - "Network Attachment Failures" - ], - "token_estimate": 580, - "skills": ["vm-creator"] - } - ], - - "skill_to_docs_mapping": { - "vm-creator": { - "primary_docs": ["scheduling-errors.md", "storage-errors.md"], - "secondary_docs": ["runtime-errors.md", "network-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3650, - "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" - }, - "vm-delete": { - "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 2950, - "description": "VM deletion failures - stuck terminating, storage cleanup" - }, - "vm-clone": { - "primary_docs": ["storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 1800, - "description": "VM cloning failures - DataVolume cloning, storage provisioning" - }, - "vm-lifecycle-manager": { - "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], - "secondary_docs": ["runtime-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3270, - "description": "VM start/stop failures - lifecycle management, scheduling" - }, - "vm-inventory": { - "primary_docs": [], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 450, - "description": "General guidance for status interpretation" - } - }, - - "error_to_docs_mapping": { - "ErrorUnschedulable": "scheduling-errors.md", - "ErrorDataVolumeNotReady": "storage-errors.md", - "ErrorPvcNotFound": "storage-errors.md", - "Terminating": "lifecycle-errors.md", - "CrashLoopBackOff": "runtime-errors.md", - "NetworkAttachmentFailed": "network-errors.md" - }, - - "cross_references": { - "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], - "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], - "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], - "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], - "network-errors.md": ["scheduling-errors.md"] - }, - - "progressive_disclosure_strategy": { - "description": "Load only the minimum documentation needed for current task", - "pattern": "error_type → primary_doc → cross_references (if needed)", - "token_savings": "29% average reduction vs single-file (based on rh-sre data)", - "navigation_overhead_reduction": "85% vs single-file navigation" - } -} diff --git a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/INDEX.md deleted file mode 100644 index a2e76963..00000000 --- a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/INDEX.md +++ /dev/null @@ -1,332 +0,0 @@ ---- -title: VM Troubleshooting Guide - Index -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 - - title: OpenShift Virtualization - Virtual Machine Status - url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html - date_accessed: 2026-02-06 - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] -semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] -use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] -last_updated: 2026-02-17 ---- - -# VM Troubleshooting Guide - Index - -## Overview - -This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: - -- **Creation**: Scheduling failures, storage provisioning issues -- **Lifecycle**: Start/stop failures, stuck states -- **Deletion**: Resources stuck in Terminating, storage cleanup failures -- **Cloning**: DataVolume cloning errors, cross-namespace issues -- **Networking**: Secondary network attachment failures -- **Runtime**: CrashLoopBackOff, guest OS boot failures - -This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. - ---- - -## 🗂️ Troubleshooting Categories - -The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: - -### 1. [Scheduling Errors](scheduling-errors.md) -**When to use**: VM fails to schedule on any node - -**Errors covered**: -- ErrorUnschedulable - Node Taints -- ErrorUnschedulable - Insufficient Resources -- ErrorUnschedulable - Node Selector Mismatch - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 2. [Storage Errors](storage-errors.md) -**When to use**: VM has storage provisioning, deletion, or cloning issues - -**Errors covered**: -- ErrorDataVolumeNotReady (all 3 subsections) -- ErrorPvcNotFound -- Storage Deletion Failures -- DataVolume Cloning Failures - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### 3. [Lifecycle Errors](lifecycle-errors.md) -**When to use**: VM has start/stop/termination issues - -**Errors covered**: -- VM Stuck in Terminating State -- VM Won't Start (Non-Scheduling Issues) -- VM Won't Stop - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### 4. [Runtime Errors](runtime-errors.md) -**When to use**: VM repeatedly crashes or fails at runtime - -**Errors covered**: -- CrashLoopBackOff - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 5. [Network Errors](network-errors.md) -**When to use**: VM has secondary network attachment failures - -**Errors covered**: -- Network Attachment Failures (Multus, SR-IOV) - -**Skills that use this**: vm-create - ---- - -## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last - -### Critical Principle: MCP-First Approach - -**All diagnostic procedures in this guide follow the MCP-first pattern**: - -``` -1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) -2. ⚠️ IF FAILS: Ask user permission to use CLI command -3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval -``` - -### Why MCP Tools First? - -- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources -- **Consistency**: Same interface across all operations -- **Better Error Handling**: MCP tools return structured errors -- **Audit Trail**: MCP tool usage is logged and trackable - -### Command Pattern Examples - -Throughout this guide, you'll see diagnostic steps formatted like this: - -**✅ CORRECT Pattern (MCP First)**: - -```markdown -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Extract**: `.metadata.finalizers` from returned JSON - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` -``` - -### Available MCP Tools - -The `openshift-virtualization` MCP server provides these tools: - -**Resource Operations**: -- `resources_get` - Get specific resource (replaces `oc get `) -- `resources_list` - List resources (replaces `oc get `) -- `resources_delete` - Delete resource (replaces `oc delete`) -- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) - -**Pod Operations**: -- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) -- `pods_get` - Get pod details (replaces `oc get pod`) -- `pods_log` - Get pod logs (replaces `oc logs`) -- `pods_exec` - Execute in pod (replaces `oc exec`) -- `pods_delete` - Delete pod (replaces `oc delete pod`) -- `pods_top` - Pod resource usage (replaces `oc top pods`) - -**Events & Monitoring**: -- `events_list` - List events (replaces `oc get events`) -- `nodes_top` - Node resource usage (replaces `oc top nodes`) -- `nodes_log` - Node logs (replaces `oc adm node-logs`) -- `nodes_stats_summary` - Detailed node stats - -**VM Operations** (KubeVirt toolset): -- `vm_create` - Create VMs -- `vm_lifecycle` - Start/stop/restart VMs - -**Namespaces**: -- `namespaces_list` - List namespaces (replaces `oc get namespaces`) -- `projects_list` - List OpenShift projects (replaces `oc get projects`) - -### When CLI Commands Are Required - -Some operations have **NO MCP equivalent** and require CLI: - -- `virtctl` commands (console, VNC access) -- `oc debug node` (node debugging) -- `oc auth can-i` (permission checks) -- `oc adm taint` (node taint management) -- Complex JSON patch operations - -For these, the guide will note: **"CLI Only - No MCP equivalent"** - -### Quick Reference: CLI → MCP Mapping - -| CLI Command | MCP Tool Equivalent | -|-------------|---------------------| -| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | -| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | -| `oc logs -n ` | `pods_log` with name="", namespace="" | -| `oc get events -n ` | `events_list` with namespace="" | -| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | - -**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. - -### How to Read Diagnostic Sections - -Each error section includes: -1. **Symptom** - What you observe -2. **Description** - What's happening -3. **Possible Causes** - Why it's happening -4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback -5. **Solutions** - **MCP tools first**, then CLI fallback -6. **Verification** - **MCP tools first**, then CLI fallback - -**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. - ---- - -## 🔍 Quick Navigation by Skill - -**vm-create**: -- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics -- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning -- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff -- [Network Errors](network-errors.md) - Network attachment failures - -**vm-delete**: -- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state -- [Storage Errors](storage-errors.md) - Storage deletion failures - -**vm-clone**: -- [Storage Errors](storage-errors.md) - DataVolume cloning failures - -**vm-lifecycle-manager**: -- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop -- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling - -**vm-inventory**: -- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed - ---- - -## 📊 Documentation Coverage & Maintenance - -### Current Coverage - -This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: - -- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) -- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures -- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop -- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) -- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems - -**Total errors documented**: 12 error types across 6 categories - ---- - -### Encountering Undocumented Errors - -#### For AI Agents (Claude Code) - -If you encounter an error **not documented** in the categories above: - -1. **Report to user** with all available details (error message, affected resources, namespace) -2. **Provide best-effort diagnostics** using MCP tools: - - `resources_get` to inspect resource status - - `pods_log` to check virt-launcher or compute container logs - - `events_list` to view Kubernetes events -3. **Suggest documentation update**: - ``` - ⚠️ This error is not yet documented in the troubleshooting guide. - - **Error**: - **Affected resource**: / - - I recommend adding this error to the troubleshooting guide: - - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file - - If it's a new error class → Create new category file in docs/troubleshooting/ - - Would you like me to help document this error for future reference? - ``` - -4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs - ---- - -#### For Human Contributors - -To document a new error: - -1. **Determine the category**: - - Scheduling issue → `scheduling-errors.md` - - Storage problem → `storage-errors.md` - - Lifecycle/start/stop → `lifecycle-errors.md` - - Runtime crash → `runtime-errors.md` - - Network issue → `network-errors.md` - - New error class → Create new category file (e.g., `security-errors.md`) - -2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): - - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback - - **Solutions**: MCP Tool approach → Implementation → CLI Fallback - - **Verification**: MCP Tool checks → Expected results → CLI Fallback - -3. **Update semantic index**: - - Edit `.ai-index/semantic-index.json` - - Add error to `error_to_docs_mapping` - - Update relevant skill's `primary_docs` or `secondary_docs` - - Increment token estimates if needed - -4. **Reference in skill files**: - - Update skill's Reference Documentation section - - Ensure skill's Document Consultation steps point to the right category - -5. **Update this INDEX.md**: - - Add error to the appropriate category section (lines 50-120) - - Update "Current Coverage" list above - ---- - -## 📚 Additional Resources - -- [rh-virt README](../../README.md) - MCP server setup and skill documentation -- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources -- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/lifecycle-errors.md deleted file mode 100644 index 6167fcc8..00000000 --- a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/lifecycle-errors.md +++ /dev/null @@ -1,869 +0,0 @@ ---- -title: VM Lifecycle Errors -category: kubevirt -sources: - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 -tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] -semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] -use_cases: [vm-deletion, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Lifecycle Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM lifecycle issues including start/stop failures and stuck termination states. - -**When to use this document**: -- VM stuck in `Terminating` state -- VM won't start (runStrategy is Always but VM never reaches Running) -- VM won't stop (runStrategy is Halted but VM never reaches Stopped) - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### VM Stuck in Terminating State - -**Symptom**: VM shows status `Terminating` but deletion never completes - -**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). - -**Possible Causes**: -- Finalizers blocking deletion -- PVC/DataVolume still attached and preventing cleanup -- VirtualMachineInstance (VMI) not terminating properly -- Custom controllers or operators blocking deletion -- Stuck virt-launcher pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check finalizers on the VM**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` - -**2. Check if VMI still exists**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name containing "virt-launcher-". - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check events for deletion issues**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results where `involvedObject.name` == "" and sort by timestamp. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' -``` - -**5. Check VM deletion timestamp**: - -Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. - -If present, VM is in deletion process. If null, VM is not being deleted. - -**Common Finalizer Patterns**: -- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer -- `foregroundDeletion` - Waits for dependent objects to be deleted -- Custom finalizers from operators - -**Solutions** (Use MCP Tools First): - -1. **Wait for dependent resources** (safest approach): - - Use `resources_list` to check if PVCs, DataVolumes are still deleting - - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) - -2. **Force delete VMI** (if VMI is stuck): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable - requires explicit user permission): - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -3. **Force delete virt-launcher pod** (if pod is stuck): - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - # Find the virt-launcher pod name first - oc get pods -n | grep virt-launcher- - - # Then delete it - oc delete pod virt-launcher--xxx -n --force --grace-period=0 - ``` - -4. **Remove finalizers** (⚠️ dangerous - use only as last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` - 2. Remove items from `.metadata.finalizers` array - 3. Update VM using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch not easily done via MCP): - ```bash - # This operation is complex for MCP - may need CLI - oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Check for protection labels** (vm-delete skill specific): - - Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. - - If value is "true", the vm-delete skill refuses deletion (this is expected behavior). - -**Verification**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error if deletion successful. - -**CLI Fallback**: -```bash -oc get vm -n -# Should return: Error from server (NotFound) -``` - ---- - - ---- - -### VM Won't Start (Non-Scheduling Issues) - -**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state - -**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. - -**Possible Causes**: -- Guest OS kernel panic or boot failure -- Cloud-init configuration errors -- Missing or corrupted disk image -- Insufficient memory for guest OS to boot -- QEMU/KVM virtualization errors -- VirtualMachineInstance (VMI) creation failures -- virt-launcher pod crashes - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VMI (VirtualMachineInstance) status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and overall VMI state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**2. Check VMI conditions for errors**: - -Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. View virt-launcher pod logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -First, get pod name from step 3, then: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -Look for QEMU errors, memory allocation failures, device errors. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**5. Check virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. - -Alternatively, use `pods_get` to get full pod details: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**6. Access VM console to see guest OS boot messages**: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -Look for kernel panic, initramfs errors, filesystem errors. - -**7. Check VM events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep -A 20 "Events:" -``` - -**Common Error Patterns**: - -1. **Guest OS Boot Failure**: - - Console shows kernel panic - - Guest hangs at GRUB or boot loader - - Cloud-init errors during first boot - -2. **Insufficient Memory**: - - Guest OS kills processes due to OOM - - VMI logs show memory allocation errors - -3. **QEMU Crashes**: - - virt-launcher logs show QEMU segfaults - - VMI repeatedly restarts - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for kernel panic, initramfs errors, filesystem errors. - -2. **Check virt-launcher pod logs for QEMU errors**: - - Use `pods_log` from diagnostic step 4 to view logs. - - Look for: - - "qemu-system-x86_64: ..." errors - - Memory allocation failures - - Device errors - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx - ``` - -3. **Increase memory if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check VMI memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -4. **Verify disk image integrity**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check PVC is bound: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - If using container disk, verify image exists and is accessible (check VMI spec). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get pvc -n - ``` - -5. **Check cloud-init configuration** (if applicable): - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. - - Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq - ``` - -6. **Restart VMI** (soft reset): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete VMI (VM controller will recreate it): - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for new VMI to start (use `resources_get` to check status). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n -w - ``` - -7. **Check virtualization extensions** (KVM): - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - - # In debug shell: - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should eventually return `Running`). - -Check VMI is running: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should show `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should eventually return: Running - -oc get vmi -n -# Should show: Running -``` - ---- - -### VM Won't Stop - -**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state - -**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. - -**Possible Causes**: -- Guest OS not responding to ACPI shutdown signal -- virt-launcher pod stuck and not terminating -- VirtualMachineInstance (VMI) deletion blocked by finalizers -- Guest shutdown scripts hanging -- Filesystem sync issues in guest OS - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (might show `Stopping`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**2. Check VMI status and deletion timestamp**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check VMI events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vmi -n | grep -A 10 "Events:" -``` - -**5. Check VMI finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.metadata.finalizers}' -``` - -**6. Check if guest is responsive** (if VMI still exists): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Patterns**: -- VMI shows `deletionTimestamp` but never actually deletes -- virt-launcher pod in `Terminating` state -- VM runStrategy is `Halted` but printableStatus shows `Stopping` - -**Solutions** (Use MCP Tools First): - -1. **Wait for graceful shutdown** (default: 30 seconds): - - Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. - -2. **Force stop by deleting VMI**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - This is the standard way to force-stop a VM. - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n - # Should return: Error from server (NotFound) - ``` - -3. **Force delete VMI with grace period** (if VMI won't delete): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -4. **Force delete virt-launcher pod**: - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find the pod using `pods_list_in_namespace` from diagnostic step 3. - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - ⚠️ **Note**: For force deletion with grace period, use CLI fallback. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') - oc delete pod $POD_NAME -n --force --grace-period=0 - ``` - -5. **Remove VMI finalizers** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VMI using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update VMI using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Patch VM runStrategy directly** (ensure consistency): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` (diagnostic step 1) - 2. Set `.spec.runStrategy` to `"Halted"` - 3. Update VM using `resources_create_or_update` with modified JSON - - **CLI Fallback** (merge patch easier via CLI): - Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Stopped` or `Halted`). - -Verify VMI is gone: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Verify virt-launcher pod is gone: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for pods matching `virt-launcher-`. Should return no results. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Stopped or Halted - -oc get vmi -n -# Should return: Error from server (NotFound) - -oc get pods -n | grep virt-launcher- -# Should return: No resources found -``` - -**Prevention**: -- Ensure guest OS has ACPI support enabled -- Use proper shutdown commands in guest OS -- Avoid forceful stops unless necessary (can corrupt guest filesystem) - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/network-errors.md deleted file mode 100644 index 82fa313c..00000000 --- a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/network-errors.md +++ /dev/null @@ -1,429 +0,0 @@ ---- -title: VM Network Errors -category: kubevirt -sources: - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] -semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] -use_cases: [vm-creation, network-troubleshooting] -related_docs: [INDEX.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Network Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. - -**When to use this document**: -- VM created successfully but secondary networks not attached -- NetworkAttachmentDefinition not found errors -- Multus CNI failures -- SR-IOV device attachment issues - -**Skills that use this**: vm-create - ---- - -### Network Attachment Failures - -**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working - -**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). - -**Possible Causes**: -- NetworkAttachmentDefinition doesn't exist in the namespace -- Multus CNI not installed or not configured on cluster -- Namespace mismatch (NAD in different namespace than VM) -- Interface name conflicts in VM spec -- Bridge/network configuration errors in NAD -- SR-IOV device not available (if using SR-IOV) - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List NetworkAttachmentDefinitions in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "" -} -``` - -Review `.items[].metadata.name` for available NADs. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -n -``` - -**2. List NetworkAttachmentDefinitions in all namespaces**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition" -} -``` - -Omit `namespace` parameter to list across all namespaces. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -A -``` - -**3. Check specific NetworkAttachmentDefinition**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "", - "name": "" -} -``` - -Review `.spec.config` for CNI configuration. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definition -n -o yaml -``` - -**4. Check VM network configuration**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.networks` to see network references. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq -``` - -**5. Check VM domain interfaces**: - -Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq -``` - -**6. Check virt-launcher pod network annotations** (shows actual attachments): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**7. Check Multus is installed**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "openshift-multus" -} -``` - -Should show Multus CNI pods running. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n openshift-multus -``` - -**8. Check for errors in virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n -``` - -**Common Error Messages**: -- `"network-attachment-definition not found"` - NAD doesn't exist in namespace -- `"multus CNI not configured"` - Multus not installed or misconfigured -- `"interface name conflict"` - Duplicate interface names in VM spec -- `"failed to add network"` - CNI plugin error (check NAD config) -- `"no available devices"` - SR-IOV device not available (if using SR-IOV) - -**Solutions** (Use MCP Tools First): - -1. **Verify NetworkAttachmentDefinition exists in correct namespace**: - - Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. - - If NAD is in different namespace, copy it to VM namespace: - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get NAD from source namespace using `resources_get` - 2. Modify `.metadata.namespace` to target namespace - 3. Create NAD in target namespace using `resources_create_or_update` - - **CLI Fallback** (stream processing easier via CLI): - Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" - ```bash - oc get network-attachment-definition -n -o yaml | \ - sed "s/namespace: /namespace: /" | \ - oc apply -f - - ``` - -2. **Create missing NetworkAttachmentDefinition**: - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - Example: Linux bridge network - - **Parameters**: - ```json - { - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "metadata": { - "name": "vlan100", - "namespace": "" - }, - "spec": { - "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" - ```bash - cat < - spec: - config: '{ - "cniVersion": "0.3.1", - "type": "bridge", - "bridge": "br1", - "vlan": 100, - "ipam": { - "type": "host-local", - "subnet": "192.168.100.0/24" - } - }' - EOF - ``` - -3. **Check Multus CNI installation**: - - Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. - - To check cluster network operator: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "config.openshift.io/v1", - "kind": "ClusterOperator" - } - ``` - - Filter for `network` operator. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n openshift-multus - oc get clusteroperators network - ``` - -4. **Fix interface name conflicts** (if VM has duplicate names): - - Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. - - Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. - - **CLI Fallback** (interactive edit easier via CLI): - Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' - oc edit vm -n - ``` - -5. **Validate NAD configuration syntax**: - - Use `resources_get` from diagnostic step 3, extract `.spec.config`. - - Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get network-attachment-definition -n -o jsonpath='{.spec.config}' - ``` - -6. **Check SR-IOV device availability** (if using SR-IOV networks): - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - List SR-IOV network node policies: - - **Parameters**: - ```json - { - "apiVersion": "sriovnetwork.openshift.io/v1", - "kind": "SriovNetworkNodePolicy", - "namespace": "openshift-sriov-network-operator" - } - ``` - - Check SR-IOV device plugin pods: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "openshift-sriov-network-operator" - } - ``` - - Filter for pods with "device-plugin" in name. - - Check available SR-IOV devices on node: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "Node", - "name": "" - } - ``` - - Review `.status.allocatable` for SR-IOV resources. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get sriovnetworknodepolicy -n openshift-sriov-network-operator - oc get pods -n openshift-sriov-network-operator | grep device-plugin - oc describe node | grep -A 10 "Allocatable:" - ``` - -7. **Recreate VM with corrected network configuration** (if needed): - - If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -After remediation, check virt-launcher pod network status: - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -Should show all attached networks with status. Example output: -```json -[ - { - "name": "openshift-sdn", - "interface": "eth0", - "ips": ["10.128.2.10"], - "default": true - }, - { - "name": "vlan100", - "interface": "net1", - "ips": ["192.168.100.5"] - } -] -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**Check from inside VM** (via console): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# In guest OS: -ip addr show -# Should show all network interfaces (eth0, net1, etc.) -``` - -**Common Network Types**: -- **Linux Bridge**: Layer 2 bridge for VLAN networks -- **SR-IOV**: High-performance direct device assignment -- **macvlan**: MAC-based VLAN for container networks -- **OVN-Kubernetes**: OpenShift native overlay network - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/runtime-errors.md deleted file mode 100644 index 804d3cbc..00000000 --- a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/runtime-errors.md +++ /dev/null @@ -1,616 +0,0 @@ ---- -title: VM Runtime Errors -category: kubevirt -sources: - - title: Kubernetes Pod Lifecycle - url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ - date_accessed: 2026-02-17 -tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] -semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] -use_cases: [vm-creation, vm-lifecycle, diagnostics] -related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Runtime Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. - -**When to use this document**: -- VM or virt-launcher pod shows `CrashLoopBackOff` status -- virt-launcher pod repeatedly restarting -- Guest OS kernel panics on boot - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### CrashLoopBackOff - -**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting - -**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. - -**Possible Causes**: -- Guest OS kernel panic on boot -- Insufficient resources (memory/CPU) for guest OS -- Corrupted disk image or filesystem -- QEMU/libvirt crashes due to configuration errors -- Missing or incompatible device drivers in guest -- Resource limits too low for virt-launcher pod -- Virtualization features (KVM) not available on node - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check virt-launcher pod restart count**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# Look at RESTARTS column -``` - -**2. View recent crash logs** (previous container instance): - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "previous": true, - "tail": 100 -} -``` - -Look for QEMU errors, kernel panics, or segfaults. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n virt-launcher--xxx --previous -``` - -**3. Check current virt-launcher logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. - -**CLI Fallback** (if MCP unavailable or all containers needed): -```bash -oc logs -n virt-launcher--xxx --all-containers -``` - -**4. Check VMI conditions for crash details**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Extract `.status.conditions` for crash details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**5. Check pod events for crash reasons**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" -``` - -**6. Check pod resource limits**: - -**MCP Tool**: `pods_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -Extract `.spec.containers[0].resources` for resource limits. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq -``` - -**7. Check node kubelet logs for OOM kills**: - -⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -oc adm node-logs -u kubelet | grep -i oom -``` - -**8. Access guest console** (if VM briefly starts): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Crash Patterns**: - -1. **Guest Kernel Panic**: - - Console logs show kernel panic messages - - Guest crashes immediately after boot - - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" - -2. **OOM (Out of Memory)**: - - Pod killed with reason: `OOMKilled` - - Guest runs out of memory during boot or operation - - virt-launcher logs show memory allocation failures - -3. **QEMU Crash**: - - virt-launcher logs show QEMU segmentation fault - - Symptoms: "qemu-system-x86_64: terminated by signal" - - Configuration incompatibility or QEMU bug - -4. **Disk Image Corruption**: - - Guest cannot boot from disk - - Filesystem errors in guest console - - DataVolume import failed - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for kernel panic or boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for: - - Kernel panic messages - - Initramfs errors - - Filesystem mounting failures - - Missing device errors - -2. **Review virt-launcher crash logs**: - - Use `pods_log` with `previous: true` from diagnostic step 2. - - Look for: - - QEMU command line errors - - Device initialization failures - - Memory allocation errors - - Signal termination (SIGSEGV, SIGABRT) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx --previous - ``` - -3. **Check for OOM (Out of Memory) kills**: - - **MCP Tool**: `pods_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - Extract `.status.containerStatuses[0].lastState.terminated.reason`. - - If returns `"OOMKilled"`: - - Option 1: Increase virt-launcher memory limits - - Option 2: Decrease guest memory allocation - - Option 3: Use smaller instance type - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' - ``` - -4. **Increase resources if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check current memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -5. **Verify disk image integrity**: - - **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters for list**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" - } - ``` - - **Parameters for specific DV**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check `.status.phase` (should be `Succeeded`). - - If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get datavolume -n -o jsonpath='{.status.phase}' - ``` - -6. **Check virtualization (KVM) availability**: - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -7. **Simplify VM configuration** (eliminate variables): - - Try creating minimal VM using vm-create skill with: - - Small instance type - - No secondary networks - - Simple container disk (e.g., Fedora) - - No cloud-init - - If minimal VM works, add features back one by one. - -8. **Recreate VM with different workload** (test disk image): - - If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) - -After remediation, check pod restart count stops increasing: - -**Parameters for pods**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). - -Check VM reaches Running state: - -**Parameters for VM**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# RESTARTS should stabilize - -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Running -``` - -Verify guest is responsive: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# Should show login prompt or OS console -``` - -**Advanced Debugging**: - -**MCP Tool**: `pods_exec` (from openshift-virtualization) - -Check libvirt domain XML: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["virsh", "dumpxml", "1"] -} -``` - -Check QEMU process: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["ps", "aux"] -} -``` - -Filter output for "qemu" process. - -**CLI Fallback** (if MCP unavailable): -```bash -oc exec -n virt-launcher--xxx -- virsh dumpxml 1 -oc exec -n virt-launcher--xxx -- ps aux | grep qemu -``` - -**Prevention**: -- Start with minimal VM configuration and add complexity gradually -- Use recommended instance types for your workload -- Test disk images before deploying to production -- Ensure nodes have adequate resources and KVM support -- Monitor resource usage and set appropriate limits - ---- - -## Workaround Patterns for MCP Tool Limitations - -### General Pattern: Diagnose → Propose → Confirm → Execute - -When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): - -**1. Diagnose** the root cause using Kubernetes commands -```bash -oc describe vm -n -oc get events -n --field-selector involvedObject.name= -``` - -**2. Propose** a manual workaround to the user with clear explanation -- Explain why the issue occurred -- Show what will be changed -- List alternative options - -**3. Confirm** with user before executing (Human-in-the-Loop) -- Wait for explicit "yes" or "apply workaround" -- Never auto-execute modifications - -**4. Execute** the workaround using oc/kubectl -```bash -oc patch vm -n --type=merge -p '...' -``` - -**5. Verify** the fix was successful -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**6. Document** the limitation and suggest filing enhancement request -- Note this is temporary until MCP tool is enhanced -- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues - -### Example: Adding Tolerations Workaround - -**Diagnostic Output**: -```markdown -## ⚠️ VM Scheduling Issue Detected - -**Root Cause**: Node taints prevent VM scheduling - -**Details**: -- Found 3 nodes with taint: `virtualization=true:NoSchedule` -- VM spec does not include matching tolerations -- This prevents VM scheduling on virtualization-dedicated nodes -``` - -**Proposed Workaround**: -```bash -oc patch vm web-server -n vms --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' -``` - -**User Confirmation Required**: -``` -How would you like to proceed? -- "apply workaround" - I'll patch the VM with tolerations -- "manual" - I'll provide instructions for you to apply manually -- "cancel" - Delete the VM and abort creation -``` - -**After Execution**: -```markdown -## ✓ Workaround Applied Successfully - -**Action**: Added tolerations for taint `virtualization=true:NoSchedule` -**New Status**: Stopped (VM can now be scheduled) - -**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. -Future VMs in this cluster will need the same fix until the tool is enhanced. -``` - ---- - -## VM Status Reference - -### Status Values - -| Status | Meaning | Action Required | -|--------|---------|-----------------| -| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | -| `Running` | VM is running | Normal | -| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | -| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | -| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | -| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | -| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | -| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | -| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | -| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | - -### Checking VM Status - -```bash -# Get printable status -oc get vm -n -o jsonpath='{.status.printableStatus}' - -# Get detailed status and conditions -oc get vm -n -o jsonpath='{.status}' | jq - -# Watch status changes in real-time -oc get vm -n -w -``` - ---- - -## Best Practices for Agents - -When implementing diagnostic workflows: - -1. **Always verify VM status** after creation (wait 5-10 seconds first) -2. **Consult this document** when encountering error status values -3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) -4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) -5. **Respect human-in-the-loop** for all VM modifications -6. **Document temporary workarounds** and their limitations clearly -7. **Suggest filing issues** for missing MCP tool features - -### Document Consultation Pattern - -```markdown -**Document Consultation** (REQUIRED): -1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes -2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." -``` - ---- - -## Known MCP Tool Limitations - -### vm_create tool - -**Currently Supported**: -- ✓ Namespace, name (required) -- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) -- ✓ Size hints (small, medium, large) -- ✓ Storage size -- ✓ Autostart flag -- ✓ Networks (Multus NetworkAttachmentDefinitions) -- ✓ Performance family (u1, o1, c1, m1) -- ✓ Instance type, preference - -**Not Currently Supported** (requires workarounds): -- ✗ Tolerations (for node taints) -- ✗ Node selectors -- ✗ Affinity/anti-affinity rules -- ✗ Resource requests/limits (beyond instance type) -- ✗ Custom labels/annotations -- ✗ SSH keys injection -- ✗ Cloud-init user data - -**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. - -**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues - ---- - -## Additional Resources - -- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) -- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) -- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) -- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) - ---- - -[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/scheduling-errors.md deleted file mode 100644 index e47c8c4f..00000000 --- a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/scheduling-errors.md +++ /dev/null @@ -1,417 +0,0 @@ ---- -title: VM Scheduling Errors -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 -tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] -semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] -use_cases: [vm-creation, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Scheduling Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**When to use this document**: -- VM shows status `ErrorUnschedulable` after creation or start attempt -- VM events mention scheduling failures, taints, resources, or node selectors - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -## ErrorUnschedulable - -**Symptom**: VM shows status `ErrorUnschedulable` after creation - -**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**Possible Causes**: - -### 1. Node Taints (Most Common) - -Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM events for scheduling failures**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `involvedObject.name` == "" and look for messages like: -- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= -``` - -**2. Check node taints in the cluster**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' -``` - -**Common Taint Patterns**: -- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule -- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes -- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads - -**Solution - Add Tolerations to VM**: - -The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. - -**Workaround (post-creation using MCP Tools)**: - -**Step 1**: Get current VM spec - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Step 2**: Modify the returned JSON to add tolerations - -Add to `.spec.template.spec.tolerations`: -```json -{ - "tolerations": [ - { - "key": "virtualization", - "operator": "Equal", - "value": "true", - "effect": "NoSchedule" - } - ] -} -``` - -**Step 3**: Update VM with modified spec - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters**: -```json -{ - "resource": "" -} -``` - -Pass the complete modified VM resource as YAML or JSON string. - -**Step 4**: Verify tolerations were added - -Use `resources_get` again and check `.spec.template.spec.tolerations` in response. - -**Step 5**: Check if VM status improved - -Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. - -**CLI Fallback** (if MCP patch is too complex): -```bash -# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' - -# Verify tolerations -oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq - -# Check status -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**Example - Multiple Tolerations**: -```bash -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" - - key: "dedicated" - operator: "Equal" - value: "virt-workloads" - effect: "NoSchedule" -' -``` - -**Toleration Operators**: -- `Equal` - Key and value must match exactly -- `Exists` - Only key must exist (ignores value) - -**Toleration Effects**: -- `NoSchedule` - Don't schedule new pods (existing pods continue) -- `PreferNoSchedule` - Avoid scheduling if possible -- `NoExecute` - Don't schedule AND evict existing pods - -**Alternative Solutions**: -1. **Remove node taints** (if you have cluster-admin access): - ```bash - oc adm taint nodes virtualization=true:NoSchedule- - ``` - -2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits - -3. **File enhancement request** - Request tolerations support in openshift-mcp-server: - https://github.com/openshift/openshift-mcp-server/issues - ---- - -### 2. Insufficient Resources - -Not enough CPU, memory, or storage available on any node. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM resource requests**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' -``` - -**2. Check node resource availability**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. - -Alternatively, use `nodes_top` MCP tool for current resource usage. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe nodes | grep -A 5 "Allocated resources" -``` - -**3. Look for VM events mentioning "Insufficient"**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep "Insufficient" -``` - -**Example Event**: -``` -0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. -``` - -**Solutions** (Use MCP Tools First): - -1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) -2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill -3. **Delete unused VMs** - Use vm-delete skill to free up resources -4. **Check resource quotas**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters for quota**: - ```json - { - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" - } - ``` - - **Parameters for limit range**: - ```json - { - "apiVersion": "v1", - "kind": "LimitRange", - "namespace": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - oc describe limitrange -n - ``` - ---- - -### 3. Node Selector Mismatch - -VM requires specific node labels that don't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM node selector requirements**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.nodeSelector` to see required node labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' -``` - -**2. List available node labels**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.metadata.labels` for available labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes --show-labels -``` - -**3. Check if any nodes match the selector**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node", - "labelSelector": "=" -} -``` - -Should return at least one node with matching labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -l = -``` - -**Solutions** (Use MCP Tools First): - -**Option 1: Remove node selector from VM** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get current VM using `resources_get` (diagnostic step 1) -2. Remove `.spec.template.spec.nodeSelector` field -3. Update VM using `resources_create_or_update` with modified JSON - -**CLI Fallback** (JSON patch easier via CLI): -Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" -```bash -oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' -``` - -**Option 2: Add label to nodes** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get node using `resources_get` -2. Add label to `.metadata.labels` -3. Update node using `resources_create_or_update` - -⚠️ **Note**: Node labeling typically requires cluster admin privileges. - -**CLI Fallback** (simpler via CLI): -Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" -```bash -oc label node = -``` - ---- - -[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/storage-errors.md deleted file mode 100644 index 16ba584d..00000000 --- a/evaluation/with_skills/rh-virt__vm-create/environment/docs/troubleshooting/storage-errors.md +++ /dev/null @@ -1,1011 +0,0 @@ ---- -title: VM Storage Errors -category: kubevirt -sources: - - title: KubeVirt CDI - DataVolumes - url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ - date_accessed: 2026-02-17 - - title: Kubernetes Persistent Volumes - url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ - date_accessed: 2026-02-17 -tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] -semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] -use_cases: [vm-creation, vm-deletion, vm-cloning] -related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] -last_updated: 2026-02-17 ---- - -# VM Storage Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. - -**When to use this document**: -- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` -- Storage deletion fails after VM deletion -- DataVolume cloning operations fail -- PVC provisioning issues - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### ErrorDataVolumeNotReady - -**Symptom**: VM shows status `ErrorDataVolumeNotReady` - -**Description**: The DataVolume (persistent storage) backing the VM is not ready. - -**Possible Causes**: - -#### 1. DataVolume Still Provisioning - -Storage provisioning takes time, especially for large disks or when importing images. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Get detailed DataVolume information**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` and `.status.conditions` for provisioning details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o json -``` - -**3. Check PVC (PersistentVolumeClaim) bound status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Check `.status.phase` for each PVC (should be `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. - -#### 2. Storage Class Not Found - -The requested storage class doesn't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List available storage classes**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" -} -``` - -Review the list of available storage classes (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -``` - -**2. Check DataVolume's requested storage class**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.spec.pvc.storageClassName` in the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' -``` - -**Solution**: -1. Use a valid storage class from the cluster -2. Recreate VM with correct storage class parameter - -#### 3. Insufficient Storage Quota - -Namespace has insufficient storage quota to provision the PVC. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check resource quotas**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**2. Check storage usage**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase -``` - -**Solution**: -1. Request quota increase from cluster admin -2. Delete unused PVCs to free quota -3. Reduce VM storage size - ---- - -### ErrorPvcNotFound - -**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List PVCs in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review the list of available PVCs (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check VM's PVC references**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' -``` - -**Solution**: -- Wait for DataVolume to create the PVC -- Manually create missing PVC -- Fix VM spec to reference correct PVC name - ---- - - ---- - -### Storage Deletion Failures - -**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace - -**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. - -**Possible Causes**: -- PVC still bound to active PersistentVolume with `Retain` policy -- DataVolume still being referenced by another resource -- CDI (Containerized Data Importer) controller issues -- Storage class retention policy preventing deletion -- Finalizers on PVC/DataVolume blocking cleanup -- PVC still mounted by a pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check PVC status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check specific PVC phase**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Released` or `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o jsonpath='{.status.phase}' -``` - -**3. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**4. Check what's using the PVC**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' -``` - -**5. Check PVC finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. - -**6. Check DataVolume finalizers**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.metadata.finalizers}' -``` - -**7. Check PV reclaim policy**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume" -} -``` - -Filter results for PV where `.spec.claimRef.name` matches ``. - -To get specific PV policy: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume", - "name": "" -} -``` - -Check `.spec.persistentVolumeReclaimPolicy`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pv | grep -oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' -``` - -**Common Finalizer Patterns**: -- `kubernetes.io/pvc-protection` - Protects PVC while in use -- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer - -**Solutions** (Use MCP Tools First): - -1. **Delete DataVolume first, then PVC**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete DataVolume first (often blocks PVC deletion): - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Wait a few seconds, then delete PVC: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete datavolume -n - oc delete pvc -n - ``` - -2. **Check for pods still using PVC**: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "" - } - ``` - - Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. - - Then delete the pods using `pods_delete`: - - **Parameters**: - ```json - { - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' - oc delete pod -n - ``` - -3. **Force delete PVC** (if safe to do so): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete pvc -n --grace-period=0 --force - ``` - -4. **Remove finalizers from PVC** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PVC using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update PVC using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Remove finalizers from DataVolume** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current DataVolume using `resources_get` (diagnostic step 6) - 2. Remove items from `.metadata.finalizers` array - 3. Update DataVolume using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Change PV reclaim policy** (if PV has Retain policy): - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PV policy using `resources_get` (diagnostic step 7) - 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` - 3. Update PV using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" - ```bash - oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' - oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' - ``` - -**Storage Quota Check** (Use MCP Tools First): - -After deletion, verify storage quota is freed: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters for quota check**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.used` to verify storage quota is freed. - -**Parameters for PVC verification**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -oc get pvc -n -``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Confirm PVC is deleted: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Confirm DataVolume is deleted: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -# Should return: Error from server (NotFound) - -oc get datavolume -n -# Should return: Error from server (NotFound) -``` - ---- - -### DataVolume Cloning Failures - -**Symptom**: VM clone created successfully but DataVolume clone operation fails - -**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. - -**Possible Causes**: -- CSI driver doesn't support volume cloning -- Source PVC storage class incompatible with cloning -- Cross-namespace cloning not permitted by storage backend -- Insufficient storage quota in target namespace -- Source PVC not in `Bound` state -- Storage class doesn't have volume cloning enabled -- CDI (Containerized Data Importer) controller issues - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume clone status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and `.metadata.name`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Check DataVolume events for errors**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe datavolume -n -``` - -**3. Check DataVolume phase**: - -Use `resources_get` from step 1, extract `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -``` - -**4. Check if storage class supports cloning**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass", - "name": "" -} -``` - -Review the full YAML output for cloning-related configurations. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -o yaml | grep -A 5 -i clone -``` - -**5. Check CSI driver capabilities**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "CSIDriver" -} -``` - -Review `.items[].metadata.name` for available CSI drivers. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get csidriver -``` - -**6. Check source PVC status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Bound` for cloning to work). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**7. Check target namespace storage quota**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**8. Check CDI controller logs**: - -**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) - -First, list pods in openshift-cnv namespace: - -**Parameters for pods_list_in_namespace**: -```json -{ - "namespace": "openshift-cnv", - "labelSelector": "app.kubernetes.io/component=cdi-deployment" -} -``` - -Then get logs using `pods_log`: - -**Parameters**: -```json -{ - "namespace": "openshift-cnv", - "name": "", - "tail": 100 -} -``` - -**CLI Fallback** (if MCP unavailable or easier via CLI): -```bash -oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') -``` - -**Common Error Messages**: -- `"volume cloning is not supported"` - CSI driver lacks clone capability -- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage -- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace -- `"insufficient quota"` - Target namespace lacks storage quota -- `"source PVC not bound"` - Source PVC must be in Bound state for cloning -- `"StorageClass does not support cloning"` - Storage class configuration issue - -**Solutions** (Use MCP Tools First): - -1. **Check storage class clone support**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" - } - ``` - - For each storage class in `.items[]`, check: - - `.metadata.name` (storage class name) - - `.provisioner` (CSI driver) - - Storage classes using CSI drivers typically support cloning. Look for provisioners like: - - `csi.ovirt.org` (oVirt CSI) - - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) - - `ebs.csi.aws.com` (AWS EBS CSI) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner - ``` - -2. **Verify source PVC is bound**: - - Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pvc -n -o jsonpath='{.status.phase}' - ``` - -3. **Check target namespace quota**: - - Use `resources_list` from diagnostic step 7 to check quota. - - If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - ``` - -4. **Use snapshot-based cloning** (alternative method): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Step 1**: Create VolumeSnapshot of source PVC - - **Parameters**: - ```json - { - "apiVersion": "snapshot.storage.k8s.io/v1", - "kind": "VolumeSnapshot", - "metadata": { - "name": "-snapshot", - "namespace": "" - }, - "spec": { - "source": { - "persistentVolumeClaimName": "" - } - } - } - ``` - - **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) - - **Step 3**: Create new DataVolume from snapshot - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "metadata": { - "name": "-rootdisk", - "namespace": "" - }, - "spec": { - "source": { - "snapshot": { - "name": "-snapshot", - "namespace": "" - } - }, - "storage": { - "resources": { - "requests": { - "storage": "50Gi" - } - }, - "storageClassName": "" - } - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" - ```bash - cat <-snapshot - namespace: - spec: - source: - persistentVolumeClaimName: - EOF - - oc get volumesnapshot -snapshot -n - - cat <-rootdisk - namespace: - spec: - source: - snapshot: - name: -snapshot - namespace: - storage: - resources: - requests: - storage: 50Gi - storageClassName: - EOF - ``` - -5. **Use "new empty storage" option** (vm-clone skill): - - If cloning isn't supported, create VM with empty storage - - Manually copy data if needed - -6. **Cross-namespace cloning workaround**: - - Some storage backends require snapshot for cross-namespace cloning - - Create snapshot in source namespace, restore in target namespace (see solution 4 above) - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Check DataVolume reached Succeeded phase: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should return `Succeeded`). - -Check PVC was created and bound: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "-rootdisk" -} -``` - -Check `.status.phase` (should return `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -# Should return: Succeeded - -oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' -# Should return: Bound -``` - -**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-create/environment/skills/vm-create/SKILL.md b/evaluation/with_skills/rh-virt__vm-create/environment/skills/vm-create/SKILL.md deleted file mode 100644 index 9fe35d09..00000000 --- a/evaluation/with_skills/rh-virt__vm-create/environment/skills/vm-create/SKILL.md +++ /dev/null @@ -1,403 +0,0 @@ ---- -name: vm-create -description: | - Create new virtual machines in OpenShift Virtualization with automatic instance type resolution and OS selection. - - Use when: - - "Create a new VM" - - "Deploy a virtual machine with [OS]" - - "Set up a VM in namespace [name]" - - "Provision a [size] VM" - - This skill handles VM creation with intelligent defaults for OpenShift Virtualization. - - NOT for managing existing VMs (use vm-lifecycle-manager or vm-delete instead). - -model: inherit -color: green ---- - -# /vm-create Skill - -Create virtual machines in OpenShift Virtualization using the `vm_create` tool from the openshift-virtualization MCP server. - -## Prerequisites - -**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools**: -- `vm_create` (from openshift-virtualization) - Create VirtualMachine resources -- `resources_get` (from openshift-virtualization) - Check VM existence and status -- `resources_list` (from openshift-virtualization) - List StorageClasses -- `namespaces_list` (from openshift-virtualization) - List available namespaces -- `events_list` (from openshift-virtualization) - Diagnostic event gathering -- `vm_lifecycle` (from openshift-virtualization) - VM restart for workarounds - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster (>= 4.19) -- OpenShift Virtualization operator installed -- ServiceAccount with RBAC permissions to create VirtualMachine resources -- Namespace with appropriate permissions - -### Prerequisite Verification - -**Before executing, verify MCP server availability:** - -1. Check `openshift-virtualization` exists in `.mcp.json` → If missing, report setup instructions -2. Verify `KUBECONFIG` is set (check presence only, never expose value) → If missing, report to user - -**Human Notification Protocol (when prerequisites fail):** - -``` -❌ Cannot execute vm-create: MCP server 'openshift-virtualization' not available -📋 Setup: Add to .mcp.json, set KUBECONFIG env var, restart Claude Code -🔗 Docs: https://github.com/openshift/openshift-mcp-server -``` - -⚠️ **SECURITY**: Never display actual KUBECONFIG path or credential values. - -## When to Use This Skill - -**Trigger this skill when:** -- User explicitly invokes `/vm-create` command -- User requests creating a new virtual machine -- Deploying VMs with specific OS (Fedora, Ubuntu, RHEL, CentOS, Debian) -- Setting up VMs with custom sizing (small, medium, large) -- Provisioning VMs with specific storage requirements - -**User phrases:** -- "Create a Fedora VM in namespace vms" -- "Deploy a medium Ubuntu VM with 100Gi disk" -- "Set up a RHEL VM called database-01" -- "/vm-create" (explicit command) - -**Do NOT use when:** -- Start/stop existing VMs → Use `/vm-lifecycle-manager` -- List VMs → Use `/vm-inventory` -- Delete VMs → Use `/vm-delete` - -## Workflow - -### Step 1: Gather VM Requirements - -**Determine missing parameters:** - -**Required:** VM Name (validate: lowercase, alphanumeric+hyphens, start letter, max 63 chars, unique), Namespace -**Optional (use defaults):** OS (fedora), Size (medium), Storage (30Gi), Performance (u1), Autostart (false) - -**Gather cluster info:** -- Detect current namespace: `kubectl config view --minify -o jsonpath='{..namespace}' || echo "default"` -- List namespaces: `namespaces_list` (from openshift-virtualization) -- List StorageClasses: `resources_list` with apiVersion="storage.k8s.io/v1", kind="StorageClass" -- Identify default SC: annotation `storageclass.kubernetes.io/is-default-class`="true" -- Analyze SC: `.volumeBindingMode` (Immediate/WaitForFirstConsumer), provisioner (rbd/cephfs=RWX hint) - -**If parameters missing, use AskUserQuestion tool with questions for:** VM Name (custom input with validation), Namespace (current + list), OS (fedora/ubuntu/rhel/centos-stream/debian/opensuse), Performance (u1/c1/m1/o1), Size (small/medium/large/xlarge), Storage (30Gi/50Gi/100Gi/custom), StorageClass (dynamic list with hints), Autostart (yes/no). See Example Usage for complete JSON structure. - -**Process responses - map labels to values:** -- OS: "Fedora"→`"fedora"`, "Ubuntu"→`"ubuntu"`, "RHEL"→`"rhel"`, "CentOS Stream"→`"centos-stream"`, "Debian"→`"debian"`, "OpenSUSE"→`"opensuse"` -- Performance: "General Purpose (u1)"→`"u1"`, "Compute (c1)"→`"c1"`, "Memory (m1)"→`"m1"`, "Overcommitted (o1)"→`"o1"` -- Size: "Small"→`"small"`, "Medium"→`"medium"`, "Large"→`"large"`, "XLarge"→`"xlarge"` -- Autostart: "No"→`false`, "Yes"→`true` - -### Step 2: Check VM Existence - -**MCP Tool**: `resources_get` (from openshift-virtualization) -**Parameters**: apiVersion="kubevirt.io/v1", kind="VirtualMachine", namespace=``, name=`` - -**If VM exists:** -``` -⚠️ VM `` already exists in namespace `` -Status: -Options: 1) Different name, 2) Delete existing, 3) Cancel -``` -**STOP** and wait for user decision. - -**If not exists:** Proceed to Step 3. - -### Step 3: Present Configuration for Confirmation - -Display configuration table: -```markdown -## Virtual Machine Configuration - -| Parameter | Value | Notes | -|-----------|-------|-------| -| VM Name | `` | validated | -| Namespace | `` | from user/context | -| OS | `` | from selection | -| Performance | `` | default: u1 | -| Size | `` | default: medium | -| Storage | `` | default: 30Gi | -| StorageClass | `` | default: cluster default | -| Autostart | `` | default: no | - -This will create a VirtualMachine consuming cluster resources. -Confirm: yes/no/modify -``` - -Proceed automatically unless the user explicitly objects. - -### Step 4: Create Virtual Machine - -**MCP Tool**: `vm_create` (from openshift-virtualization) - -**Parameters**: -- `namespace`: `` - REQUIRED -- `name`: `` - REQUIRED -- `workload`: `` - OPTIONAL (default: fedora) -- `size`: `` - OPTIONAL (small/medium/large/xlarge) -- `storage`: `` - OPTIONAL (default: 30Gi) -- `performance`: `` - OPTIONAL (u1/c1/m1/o1) -- `autostart`: `` - OPTIONAL (default: false) - -**Example**: `vm_create({"namespace": "vms", "name": "web-server", "workload": "fedora", "size": "medium", "storage": "50Gi", "autostart": false})` - -**Error Handling:** -- Namespace not found → Report, list available -- RBAC denied → Report permissions error -- Storage fails → Check StorageClass exists -- Scheduling fails → See Step 5 - -### Step 5: Verify Status and Diagnose Issues - -**Wait 5-10 seconds, then check status:** - -**MCP Tool**: `resources_get` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", name=``, namespace=``) -**Extract**: `.status.printableStatus` - -**Status interpretation:** -- Stopped/Halted → Success (VM created, not started) -- Running → Success (if autostart=true) -- Provisioning → Wait 5s, check again -- ErrorUnschedulable → Execute diagnostic workflow (Step 5a) -- ErrorDataVolumeNotReady → Storage issue (see Common Issues) - -#### 5a. Diagnostic Workflow (ErrorUnschedulable) - -**CRITICAL: Document Consultation FIRST:** -1. Read [scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) using Read tool -2. Output: "I detected ErrorUnschedulable. I consulted [scheduling-errors.md] to understand diagnosis strategies." - -**Gather diagnostics:** -- List events: `events_list` (namespace=``) → Filter for VM/VMI -- Get VM: `resources_get` → Check `.status.conditions` -- List nodes: `resources_list` (apiVersion="v1", kind="Node") → Extract `.spec.taints` - -**Parse root cause:** -- "taints" in events → Taints/tolerations issue -- "Insufficient cpu/memory" → Resource constraints -- "no nodes available" → No suitable nodes - -**Present diagnosis:** -```markdown -## ⚠️ VM Scheduling Issue Detected - -**Status**: ErrorUnschedulable | **Root Cause**: | **Details**: - -### Recommended Solution - -**Command**: `oc patch vm -n ...` -**Impact**: -**Options**: 1) Apply workaround, 2) Manual, 3) Cancel, 4) Ignore -⚠️ MCP limitation: vm_create doesn't support tolerations -``` - -**Wait for user decision.** - -**If user confirms:** -1. Apply patch: `resources_create_or_update` (fetch, add tolerations, update) OR `oc patch` -2. Verify: `resources_get` → Check `.spec.template.spec.tolerations` -3. **Restart VM**: `vm_lifecycle` (action="restart") to apply new spec -4. Wait 15-20s, check status → Stopped → Provisioning → Running - -**Report**: `## ✓ Workaround Applied | **Action**: Added tolerations, restarted | **Status**: ` - -### Step 6: Report Creation Status - -**On success:** -```markdown -## ✓ Virtual Machine Created Successfully - -**VM**: `` (namespace: ``) -**OS**: | **Size**: () | **Storage**: | **Status**: -**Provisioning**: ~2-5 min (Provisioning → Stopped) - -### Next Steps -Start: "Start VM " | View: "Show VM " - -### Accessing the VM -1. Serial: `virtctl console -n ` -2. VNC: OpenShift Console → Virtualization → VMs → → Console -3. SSH: Get IP from VMI, `ssh @` -4. Port Forward: `virtctl port-forward vmi/ -n 8080:80` - -### Default Credentials -- Fedora: fedora | Ubuntu: ubuntu | RHEL: cloud-user | CentOS: centos | Debian: debian -- All require SSH key or console password set: `virtctl console `, `sudo passwd ` -``` - -**On failure:** -```markdown -## ❌ Failed to Create Virtual Machine - -**Error**: - -**Common Causes**: -- Namespace not exists → `oc create namespace ` -- RBAC denied → Check ServiceAccount permissions -- Resource constraints → Try smaller size -- Invalid parameters → Verify OS, size, storage format -- Operator not installed → Verify CSVs - -Troubleshooting: See Common Issues -``` - -## Common Issues - -### Issue 1: Namespace Not Found -**Error**: "Namespace 'xyz' not found" -**Solution**: List with `namespaces_list`, create with `resources_create_or_update` or `oc create namespace ` - -### Issue 2: Insufficient Permissions -**Error**: "Forbidden: User cannot create VirtualMachines" -**Solution**: Verify KUBECONFIG RBAC, requires create VirtualMachine permissions, contact admin - -### Issue 3: Resource Constraints (ErrorUnschedulable) -**Error**: "0/N nodes: Insufficient cpu/memory" -**Solution**: Check `nodes_top`, try smaller size (medium→small, o1 overcommitted), scale cluster - -### Issue 4: Node Taints (ErrorUnschedulable) -**Error**: "0/N nodes: taints pod didn't tolerate" -**Solution**: Apply tolerations workaround (Step 5a), restart VM - -### Issue 5: Storage Provisioning (ErrorDataVolumeNotReady) -**Error**: "PVC pending" or "StorageClass not found" -**Solution**: Verify SC (`resources_list`), check default annotation, verify provisioner, check quotas - -### Issue 6: DataVolume Import Failure -**Error**: "DataVolume import failed" or "image pull error" -**Solution**: Verify internet access, check DV status, ensure valid OS, verify registry auth - -### Issue 7: Operator Not Installed -**Error**: "VirtualMachine CRD not found" -**Solution**: Verify operator: `resources_list` (apiVersion="operators.coreos.com/v1alpha1", kind="CSV", namespace="openshift-cnv") - -## Dependencies - -### Required MCP Servers -- `openshift-virtualization` - OpenShift MCP server with KubeVirt toolset (https://github.com/openshift/openshift-mcp-server) - -### Required MCP Tools -- `vm_create` - Create VMs (namespace, name, workload, size, storage, performance, autostart) -- `resources_get` - Get resources (apiVersion, kind, namespace, name) -- `resources_list` - List resources (apiVersion, kind, namespace optional) -- `namespaces_list` - List namespaces -- `events_list` - List events (namespace) -- `vm_lifecycle` - VM lifecycle (namespace, name, action: start/stop/restart) -- `resources_create_or_update` - Update resources (JSON) - -### Related Skills -- `vm-lifecycle-manager` - Start VMs | `vm-inventory` - List VMs | `vm-delete` - Delete VMs | `vm-clone` - Clone VMs | `vm-snapshot-create` - Snapshot VMs - -### Reference Documentation -- [scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) - ErrorUnschedulable (consulted Step 5a) -- [storage-errors.md](../../docs/troubleshooting/storage-errors.md) - ErrorDataVolumeNotReady -- [network-errors.md](../../docs/troubleshooting/network-errors.md) - Network failures -- [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) - CrashLoopBackOff -- [Troubleshooting INDEX](../../docs/troubleshooting/INDEX.md) - Full error index -- [OpenShift Virt Docs](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index) -- [KubeVirt API](https://kubevirt.io/api-reference/) -- [OpenShift MCP](https://github.com/openshift/openshift-mcp-server) - -## Critical: Human-in-the-Loop Requirements - -**IMPORTANT:** This skill creates cluster resources consuming CPU, memory, storage. You MUST: - -1. **Before Creating** - - Display complete configuration in table format - - Show resource impact - - Ask: "Confirm: yes/no/modify" - - Wait for explicit confirmation - -2. **Never Auto-Execute** - - NEVER create VMs without displaying configuration - - NEVER assume approval - - NEVER create multiple VMs without individual confirmations - -**Why**: Resource consumption, cost impact, namespace quotas - -**Rationale**: Prevents unintended resource consumption; maintains user control. - -## Security Considerations - -- **RBAC**: Requires create VirtualMachines (kubevirt.io/v1) in namespace -- **Namespace Isolation**: VMs in specified namespace only -- **Storage Quotas**: Respects ResourceQuotas -- **Image Security**: Uses official images from trusted registries -- **KUBECONFIG**: Never exposed (presence only) -- **Audit**: All ops logged via K8s audit - -## Example Usage - -### Example 1: Complete Interactive Workflow - -``` -User: "Create a VM" -Agent: [No params, detects namespace: production, queries SCs] - [AskUserQuestion with all 8 questions - see JSON below] -[User selects: my-app-server, production, fedora, u1, medium, 30Gi, default SC, no] -Agent: [Validates ✓, checks existence ✓, shows configuration table] -User: "yes" -Agent: [vm_create(...)] -## ✓ Virtual Machine Created Successfully -[Details, next steps, access instructions] -``` - -**AskUserQuestion JSON (8 questions: VM Name, Namespace, OS, Performance, Size, Storage, StorageClass, Autostart):** -```json -{"questions": [ - {"question": "VM name?", "header": "VM Name", "multiSelect": false, "options": [{"label": "Enter custom name", "description": "Lowercase, alphanumeric+hyphens, start letter, max 63"}]}, - {"question": "Namespace?", "header": "Namespace", "multiSelect": false, "options": [{"label": " (Current)", "description": "From kubeconfig"}, {"label": "Other", "description": ""}]}, - {"question": "OS?", "header": "OS", "multiSelect": false, "options": [{"label": "Fedora (Recommended)", "description": "General purpose"}, {"label": "Ubuntu", "description": "Web services"}, {"label": "RHEL", "description": "Enterprise"}, {"label": "CentOS Stream", "description": "Upstream RHEL"}, {"label": "Debian", "description": "Stable minimal"}, {"label": "OpenSUSE", "description": "Community"}]}, - {"question": "Performance?", "header": "Performance", "multiSelect": false, "options": [{"label": "General (u1) (Recommended)", "description": "Balanced - most workloads"}, {"label": "Compute (c1)", "description": "CPU-intensive"}, {"label": "Memory (m1)", "description": "Memory-intensive"}, {"label": "Overcommitted (o1)", "description": "Dev/test"}]}, - {"question": "Size?", "header": "Size", "multiSelect": false, "options": [{"label": "Small", "description": "1 vCPU, 2Gi"}, {"label": "Medium (Recommended)", "description": "2-4 vCPU, 4-8Gi"}, {"label": "Large", "description": "4-8 vCPU, 8-16Gi"}, {"label": "XLarge", "description": "8+ vCPU, 16+ Gi"}]}, - {"question": "Storage?", "header": "Storage", "multiSelect": false, "options": [{"label": "30Gi (Recommended)", "description": "Most workloads"}, {"label": "50Gi", "description": "Moderate needs"}, {"label": "100Gi", "description": "Large requirements"}, {"label": "Custom", "description": "Specify (e.g., 75Gi)"}]}, - {"question": "StorageClass?", "header": "SC", "multiSelect": false, "options": [{"label": " (Default)", "description": ", "}, {"label": "", "description": ""}]}, - {"question": "Autostart?", "header": "Autostart", "multiSelect": false, "options": [{"label": "No (Recommended)", "description": "Start manually"}, {"label": "Yes", "description": "Auto-start"}]} -]} -``` - -### Example 2: Partial Parameters - -``` -User: "Create Ubuntu VM web-01 in apps" -Agent: [Detects: web-01, apps, ubuntu; Asks: Performance, Size, Storage, SC, Autostart] -[User: u1, large, 100Gi, local-storage, yes] -Agent: [Validates, shows config] -User: "yes" -Agent: [Creates with autostart=true] -## ✓ Virtual Machine Created and Starting -``` - -### Example 3: User Modifies Config - -``` -User: "Deploy memory-optimized RHEL VM db-server in production" -Agent: [Detects: db-server, production, rhel, m1; Asks: Size, Storage, SC, Autostart] -[User: large, 100Gi, ceph-rbd, no] -Agent: [Shows config] -User: "Actually, compute-optimized and autostart" -Agent: [Updates: c1, yes; Shows updated] -User: "yes" -Agent: [Creates] -``` - -## Advanced Features - -**Custom Images**: `vm_create({"workload": "quay.io/containerdisks/fedora:latest", ...})` -**Secondary Networks**: `vm_create({"networks": ["vlan-network"], ...})` or `{"networks": [{"name": "eth1", "networkName": "vlan"}], ...}` -**Explicit Instance Type**: `vm_create({"instancetype": "u1.large", ...})` diff --git a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/.ai-index/semantic-index.json deleted file mode 100644 index 2512c2a3..00000000 --- a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/.ai-index/semantic-index.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "index_version": "1.0.0", - "last_updated": "2026-02-17", - "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", - - "documents": [ - { - "file": "INDEX.md", - "title": "VM Troubleshooting Guide - Index", - "category": "navigation", - "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], - "use_cases": ["navigation", "tool-reference", "general-guidance"], - "token_estimate": 450, - "skills": ["all"] - }, - { - "file": "scheduling-errors.md", - "title": "VM Scheduling Errors", - "category": "scheduling", - "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], - "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], - "errors_covered": [ - "ErrorUnschedulable - Node Taints", - "ErrorUnschedulable - Insufficient Resources", - "ErrorUnschedulable - Node Selector Mismatch" - ], - "token_estimate": 820, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "storage-errors.md", - "title": "VM Storage Errors", - "category": "storage", - "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], - "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], - "errors_covered": [ - "ErrorDataVolumeNotReady - DataVolume Still Provisioning", - "ErrorDataVolumeNotReady - Storage Class Not Found", - "ErrorDataVolumeNotReady - Insufficient Storage Quota", - "ErrorPvcNotFound", - "Storage Deletion Failures", - "DataVolume Cloning Failures" - ], - "token_estimate": 1350, - "skills": ["vm-creator", "vm-delete", "vm-clone"] - }, - { - "file": "lifecycle-errors.md", - "title": "VM Lifecycle Errors", - "category": "lifecycle", - "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], - "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], - "errors_covered": [ - "VM Stuck in Terminating State", - "VM Won't Start (Non-Scheduling Issues)", - "VM Won't Stop" - ], - "token_estimate": 1150, - "skills": ["vm-delete", "vm-lifecycle-manager"] - }, - { - "file": "runtime-errors.md", - "title": "VM Runtime Errors", - "category": "runtime", - "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], - "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], - "errors_covered": [ - "CrashLoopBackOff" - ], - "token_estimate": 850, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "network-errors.md", - "title": "VM Network Errors", - "category": "networking", - "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], - "use_cases": ["vm-creation", "network-troubleshooting"], - "errors_covered": [ - "Network Attachment Failures" - ], - "token_estimate": 580, - "skills": ["vm-creator"] - } - ], - - "skill_to_docs_mapping": { - "vm-creator": { - "primary_docs": ["scheduling-errors.md", "storage-errors.md"], - "secondary_docs": ["runtime-errors.md", "network-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3650, - "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" - }, - "vm-delete": { - "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 2950, - "description": "VM deletion failures - stuck terminating, storage cleanup" - }, - "vm-clone": { - "primary_docs": ["storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 1800, - "description": "VM cloning failures - DataVolume cloning, storage provisioning" - }, - "vm-lifecycle-manager": { - "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], - "secondary_docs": ["runtime-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3270, - "description": "VM start/stop failures - lifecycle management, scheduling" - }, - "vm-inventory": { - "primary_docs": [], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 450, - "description": "General guidance for status interpretation" - } - }, - - "error_to_docs_mapping": { - "ErrorUnschedulable": "scheduling-errors.md", - "ErrorDataVolumeNotReady": "storage-errors.md", - "ErrorPvcNotFound": "storage-errors.md", - "Terminating": "lifecycle-errors.md", - "CrashLoopBackOff": "runtime-errors.md", - "NetworkAttachmentFailed": "network-errors.md" - }, - - "cross_references": { - "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], - "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], - "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], - "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], - "network-errors.md": ["scheduling-errors.md"] - }, - - "progressive_disclosure_strategy": { - "description": "Load only the minimum documentation needed for current task", - "pattern": "error_type → primary_doc → cross_references (if needed)", - "token_savings": "29% average reduction vs single-file (based on rh-sre data)", - "navigation_overhead_reduction": "85% vs single-file navigation" - } -} diff --git a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/INDEX.md deleted file mode 100644 index a2e76963..00000000 --- a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/INDEX.md +++ /dev/null @@ -1,332 +0,0 @@ ---- -title: VM Troubleshooting Guide - Index -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 - - title: OpenShift Virtualization - Virtual Machine Status - url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html - date_accessed: 2026-02-06 - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] -semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] -use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] -last_updated: 2026-02-17 ---- - -# VM Troubleshooting Guide - Index - -## Overview - -This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: - -- **Creation**: Scheduling failures, storage provisioning issues -- **Lifecycle**: Start/stop failures, stuck states -- **Deletion**: Resources stuck in Terminating, storage cleanup failures -- **Cloning**: DataVolume cloning errors, cross-namespace issues -- **Networking**: Secondary network attachment failures -- **Runtime**: CrashLoopBackOff, guest OS boot failures - -This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. - ---- - -## 🗂️ Troubleshooting Categories - -The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: - -### 1. [Scheduling Errors](scheduling-errors.md) -**When to use**: VM fails to schedule on any node - -**Errors covered**: -- ErrorUnschedulable - Node Taints -- ErrorUnschedulable - Insufficient Resources -- ErrorUnschedulable - Node Selector Mismatch - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 2. [Storage Errors](storage-errors.md) -**When to use**: VM has storage provisioning, deletion, or cloning issues - -**Errors covered**: -- ErrorDataVolumeNotReady (all 3 subsections) -- ErrorPvcNotFound -- Storage Deletion Failures -- DataVolume Cloning Failures - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### 3. [Lifecycle Errors](lifecycle-errors.md) -**When to use**: VM has start/stop/termination issues - -**Errors covered**: -- VM Stuck in Terminating State -- VM Won't Start (Non-Scheduling Issues) -- VM Won't Stop - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### 4. [Runtime Errors](runtime-errors.md) -**When to use**: VM repeatedly crashes or fails at runtime - -**Errors covered**: -- CrashLoopBackOff - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 5. [Network Errors](network-errors.md) -**When to use**: VM has secondary network attachment failures - -**Errors covered**: -- Network Attachment Failures (Multus, SR-IOV) - -**Skills that use this**: vm-create - ---- - -## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last - -### Critical Principle: MCP-First Approach - -**All diagnostic procedures in this guide follow the MCP-first pattern**: - -``` -1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) -2. ⚠️ IF FAILS: Ask user permission to use CLI command -3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval -``` - -### Why MCP Tools First? - -- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources -- **Consistency**: Same interface across all operations -- **Better Error Handling**: MCP tools return structured errors -- **Audit Trail**: MCP tool usage is logged and trackable - -### Command Pattern Examples - -Throughout this guide, you'll see diagnostic steps formatted like this: - -**✅ CORRECT Pattern (MCP First)**: - -```markdown -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Extract**: `.metadata.finalizers` from returned JSON - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` -``` - -### Available MCP Tools - -The `openshift-virtualization` MCP server provides these tools: - -**Resource Operations**: -- `resources_get` - Get specific resource (replaces `oc get `) -- `resources_list` - List resources (replaces `oc get `) -- `resources_delete` - Delete resource (replaces `oc delete`) -- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) - -**Pod Operations**: -- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) -- `pods_get` - Get pod details (replaces `oc get pod`) -- `pods_log` - Get pod logs (replaces `oc logs`) -- `pods_exec` - Execute in pod (replaces `oc exec`) -- `pods_delete` - Delete pod (replaces `oc delete pod`) -- `pods_top` - Pod resource usage (replaces `oc top pods`) - -**Events & Monitoring**: -- `events_list` - List events (replaces `oc get events`) -- `nodes_top` - Node resource usage (replaces `oc top nodes`) -- `nodes_log` - Node logs (replaces `oc adm node-logs`) -- `nodes_stats_summary` - Detailed node stats - -**VM Operations** (KubeVirt toolset): -- `vm_create` - Create VMs -- `vm_lifecycle` - Start/stop/restart VMs - -**Namespaces**: -- `namespaces_list` - List namespaces (replaces `oc get namespaces`) -- `projects_list` - List OpenShift projects (replaces `oc get projects`) - -### When CLI Commands Are Required - -Some operations have **NO MCP equivalent** and require CLI: - -- `virtctl` commands (console, VNC access) -- `oc debug node` (node debugging) -- `oc auth can-i` (permission checks) -- `oc adm taint` (node taint management) -- Complex JSON patch operations - -For these, the guide will note: **"CLI Only - No MCP equivalent"** - -### Quick Reference: CLI → MCP Mapping - -| CLI Command | MCP Tool Equivalent | -|-------------|---------------------| -| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | -| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | -| `oc logs -n ` | `pods_log` with name="", namespace="" | -| `oc get events -n ` | `events_list` with namespace="" | -| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | - -**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. - -### How to Read Diagnostic Sections - -Each error section includes: -1. **Symptom** - What you observe -2. **Description** - What's happening -3. **Possible Causes** - Why it's happening -4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback -5. **Solutions** - **MCP tools first**, then CLI fallback -6. **Verification** - **MCP tools first**, then CLI fallback - -**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. - ---- - -## 🔍 Quick Navigation by Skill - -**vm-create**: -- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics -- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning -- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff -- [Network Errors](network-errors.md) - Network attachment failures - -**vm-delete**: -- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state -- [Storage Errors](storage-errors.md) - Storage deletion failures - -**vm-clone**: -- [Storage Errors](storage-errors.md) - DataVolume cloning failures - -**vm-lifecycle-manager**: -- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop -- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling - -**vm-inventory**: -- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed - ---- - -## 📊 Documentation Coverage & Maintenance - -### Current Coverage - -This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: - -- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) -- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures -- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop -- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) -- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems - -**Total errors documented**: 12 error types across 6 categories - ---- - -### Encountering Undocumented Errors - -#### For AI Agents (Claude Code) - -If you encounter an error **not documented** in the categories above: - -1. **Report to user** with all available details (error message, affected resources, namespace) -2. **Provide best-effort diagnostics** using MCP tools: - - `resources_get` to inspect resource status - - `pods_log` to check virt-launcher or compute container logs - - `events_list` to view Kubernetes events -3. **Suggest documentation update**: - ``` - ⚠️ This error is not yet documented in the troubleshooting guide. - - **Error**: - **Affected resource**: / - - I recommend adding this error to the troubleshooting guide: - - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file - - If it's a new error class → Create new category file in docs/troubleshooting/ - - Would you like me to help document this error for future reference? - ``` - -4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs - ---- - -#### For Human Contributors - -To document a new error: - -1. **Determine the category**: - - Scheduling issue → `scheduling-errors.md` - - Storage problem → `storage-errors.md` - - Lifecycle/start/stop → `lifecycle-errors.md` - - Runtime crash → `runtime-errors.md` - - Network issue → `network-errors.md` - - New error class → Create new category file (e.g., `security-errors.md`) - -2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): - - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback - - **Solutions**: MCP Tool approach → Implementation → CLI Fallback - - **Verification**: MCP Tool checks → Expected results → CLI Fallback - -3. **Update semantic index**: - - Edit `.ai-index/semantic-index.json` - - Add error to `error_to_docs_mapping` - - Update relevant skill's `primary_docs` or `secondary_docs` - - Increment token estimates if needed - -4. **Reference in skill files**: - - Update skill's Reference Documentation section - - Ensure skill's Document Consultation steps point to the right category - -5. **Update this INDEX.md**: - - Add error to the appropriate category section (lines 50-120) - - Update "Current Coverage" list above - ---- - -## 📚 Additional Resources - -- [rh-virt README](../../README.md) - MCP server setup and skill documentation -- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources -- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/lifecycle-errors.md deleted file mode 100644 index 6167fcc8..00000000 --- a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/lifecycle-errors.md +++ /dev/null @@ -1,869 +0,0 @@ ---- -title: VM Lifecycle Errors -category: kubevirt -sources: - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 -tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] -semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] -use_cases: [vm-deletion, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Lifecycle Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM lifecycle issues including start/stop failures and stuck termination states. - -**When to use this document**: -- VM stuck in `Terminating` state -- VM won't start (runStrategy is Always but VM never reaches Running) -- VM won't stop (runStrategy is Halted but VM never reaches Stopped) - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### VM Stuck in Terminating State - -**Symptom**: VM shows status `Terminating` but deletion never completes - -**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). - -**Possible Causes**: -- Finalizers blocking deletion -- PVC/DataVolume still attached and preventing cleanup -- VirtualMachineInstance (VMI) not terminating properly -- Custom controllers or operators blocking deletion -- Stuck virt-launcher pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check finalizers on the VM**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` - -**2. Check if VMI still exists**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name containing "virt-launcher-". - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check events for deletion issues**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results where `involvedObject.name` == "" and sort by timestamp. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' -``` - -**5. Check VM deletion timestamp**: - -Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. - -If present, VM is in deletion process. If null, VM is not being deleted. - -**Common Finalizer Patterns**: -- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer -- `foregroundDeletion` - Waits for dependent objects to be deleted -- Custom finalizers from operators - -**Solutions** (Use MCP Tools First): - -1. **Wait for dependent resources** (safest approach): - - Use `resources_list` to check if PVCs, DataVolumes are still deleting - - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) - -2. **Force delete VMI** (if VMI is stuck): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable - requires explicit user permission): - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -3. **Force delete virt-launcher pod** (if pod is stuck): - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - # Find the virt-launcher pod name first - oc get pods -n | grep virt-launcher- - - # Then delete it - oc delete pod virt-launcher--xxx -n --force --grace-period=0 - ``` - -4. **Remove finalizers** (⚠️ dangerous - use only as last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` - 2. Remove items from `.metadata.finalizers` array - 3. Update VM using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch not easily done via MCP): - ```bash - # This operation is complex for MCP - may need CLI - oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Check for protection labels** (vm-delete skill specific): - - Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. - - If value is "true", the vm-delete skill refuses deletion (this is expected behavior). - -**Verification**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error if deletion successful. - -**CLI Fallback**: -```bash -oc get vm -n -# Should return: Error from server (NotFound) -``` - ---- - - ---- - -### VM Won't Start (Non-Scheduling Issues) - -**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state - -**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. - -**Possible Causes**: -- Guest OS kernel panic or boot failure -- Cloud-init configuration errors -- Missing or corrupted disk image -- Insufficient memory for guest OS to boot -- QEMU/KVM virtualization errors -- VirtualMachineInstance (VMI) creation failures -- virt-launcher pod crashes - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VMI (VirtualMachineInstance) status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and overall VMI state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**2. Check VMI conditions for errors**: - -Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. View virt-launcher pod logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -First, get pod name from step 3, then: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -Look for QEMU errors, memory allocation failures, device errors. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**5. Check virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. - -Alternatively, use `pods_get` to get full pod details: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**6. Access VM console to see guest OS boot messages**: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -Look for kernel panic, initramfs errors, filesystem errors. - -**7. Check VM events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep -A 20 "Events:" -``` - -**Common Error Patterns**: - -1. **Guest OS Boot Failure**: - - Console shows kernel panic - - Guest hangs at GRUB or boot loader - - Cloud-init errors during first boot - -2. **Insufficient Memory**: - - Guest OS kills processes due to OOM - - VMI logs show memory allocation errors - -3. **QEMU Crashes**: - - virt-launcher logs show QEMU segfaults - - VMI repeatedly restarts - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for kernel panic, initramfs errors, filesystem errors. - -2. **Check virt-launcher pod logs for QEMU errors**: - - Use `pods_log` from diagnostic step 4 to view logs. - - Look for: - - "qemu-system-x86_64: ..." errors - - Memory allocation failures - - Device errors - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx - ``` - -3. **Increase memory if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check VMI memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -4. **Verify disk image integrity**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check PVC is bound: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - If using container disk, verify image exists and is accessible (check VMI spec). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get pvc -n - ``` - -5. **Check cloud-init configuration** (if applicable): - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. - - Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq - ``` - -6. **Restart VMI** (soft reset): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete VMI (VM controller will recreate it): - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for new VMI to start (use `resources_get` to check status). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n -w - ``` - -7. **Check virtualization extensions** (KVM): - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - - # In debug shell: - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should eventually return `Running`). - -Check VMI is running: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should show `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should eventually return: Running - -oc get vmi -n -# Should show: Running -``` - ---- - -### VM Won't Stop - -**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state - -**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. - -**Possible Causes**: -- Guest OS not responding to ACPI shutdown signal -- virt-launcher pod stuck and not terminating -- VirtualMachineInstance (VMI) deletion blocked by finalizers -- Guest shutdown scripts hanging -- Filesystem sync issues in guest OS - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (might show `Stopping`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**2. Check VMI status and deletion timestamp**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check VMI events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vmi -n | grep -A 10 "Events:" -``` - -**5. Check VMI finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.metadata.finalizers}' -``` - -**6. Check if guest is responsive** (if VMI still exists): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Patterns**: -- VMI shows `deletionTimestamp` but never actually deletes -- virt-launcher pod in `Terminating` state -- VM runStrategy is `Halted` but printableStatus shows `Stopping` - -**Solutions** (Use MCP Tools First): - -1. **Wait for graceful shutdown** (default: 30 seconds): - - Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. - -2. **Force stop by deleting VMI**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - This is the standard way to force-stop a VM. - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n - # Should return: Error from server (NotFound) - ``` - -3. **Force delete VMI with grace period** (if VMI won't delete): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -4. **Force delete virt-launcher pod**: - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find the pod using `pods_list_in_namespace` from diagnostic step 3. - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - ⚠️ **Note**: For force deletion with grace period, use CLI fallback. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') - oc delete pod $POD_NAME -n --force --grace-period=0 - ``` - -5. **Remove VMI finalizers** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VMI using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update VMI using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Patch VM runStrategy directly** (ensure consistency): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` (diagnostic step 1) - 2. Set `.spec.runStrategy` to `"Halted"` - 3. Update VM using `resources_create_or_update` with modified JSON - - **CLI Fallback** (merge patch easier via CLI): - Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Stopped` or `Halted`). - -Verify VMI is gone: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Verify virt-launcher pod is gone: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for pods matching `virt-launcher-`. Should return no results. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Stopped or Halted - -oc get vmi -n -# Should return: Error from server (NotFound) - -oc get pods -n | grep virt-launcher- -# Should return: No resources found -``` - -**Prevention**: -- Ensure guest OS has ACPI support enabled -- Use proper shutdown commands in guest OS -- Avoid forceful stops unless necessary (can corrupt guest filesystem) - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/network-errors.md deleted file mode 100644 index 82fa313c..00000000 --- a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/network-errors.md +++ /dev/null @@ -1,429 +0,0 @@ ---- -title: VM Network Errors -category: kubevirt -sources: - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] -semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] -use_cases: [vm-creation, network-troubleshooting] -related_docs: [INDEX.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Network Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. - -**When to use this document**: -- VM created successfully but secondary networks not attached -- NetworkAttachmentDefinition not found errors -- Multus CNI failures -- SR-IOV device attachment issues - -**Skills that use this**: vm-create - ---- - -### Network Attachment Failures - -**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working - -**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). - -**Possible Causes**: -- NetworkAttachmentDefinition doesn't exist in the namespace -- Multus CNI not installed or not configured on cluster -- Namespace mismatch (NAD in different namespace than VM) -- Interface name conflicts in VM spec -- Bridge/network configuration errors in NAD -- SR-IOV device not available (if using SR-IOV) - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List NetworkAttachmentDefinitions in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "" -} -``` - -Review `.items[].metadata.name` for available NADs. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -n -``` - -**2. List NetworkAttachmentDefinitions in all namespaces**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition" -} -``` - -Omit `namespace` parameter to list across all namespaces. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -A -``` - -**3. Check specific NetworkAttachmentDefinition**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "", - "name": "" -} -``` - -Review `.spec.config` for CNI configuration. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definition -n -o yaml -``` - -**4. Check VM network configuration**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.networks` to see network references. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq -``` - -**5. Check VM domain interfaces**: - -Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq -``` - -**6. Check virt-launcher pod network annotations** (shows actual attachments): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**7. Check Multus is installed**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "openshift-multus" -} -``` - -Should show Multus CNI pods running. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n openshift-multus -``` - -**8. Check for errors in virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n -``` - -**Common Error Messages**: -- `"network-attachment-definition not found"` - NAD doesn't exist in namespace -- `"multus CNI not configured"` - Multus not installed or misconfigured -- `"interface name conflict"` - Duplicate interface names in VM spec -- `"failed to add network"` - CNI plugin error (check NAD config) -- `"no available devices"` - SR-IOV device not available (if using SR-IOV) - -**Solutions** (Use MCP Tools First): - -1. **Verify NetworkAttachmentDefinition exists in correct namespace**: - - Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. - - If NAD is in different namespace, copy it to VM namespace: - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get NAD from source namespace using `resources_get` - 2. Modify `.metadata.namespace` to target namespace - 3. Create NAD in target namespace using `resources_create_or_update` - - **CLI Fallback** (stream processing easier via CLI): - Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" - ```bash - oc get network-attachment-definition -n -o yaml | \ - sed "s/namespace: /namespace: /" | \ - oc apply -f - - ``` - -2. **Create missing NetworkAttachmentDefinition**: - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - Example: Linux bridge network - - **Parameters**: - ```json - { - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "metadata": { - "name": "vlan100", - "namespace": "" - }, - "spec": { - "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" - ```bash - cat < - spec: - config: '{ - "cniVersion": "0.3.1", - "type": "bridge", - "bridge": "br1", - "vlan": 100, - "ipam": { - "type": "host-local", - "subnet": "192.168.100.0/24" - } - }' - EOF - ``` - -3. **Check Multus CNI installation**: - - Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. - - To check cluster network operator: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "config.openshift.io/v1", - "kind": "ClusterOperator" - } - ``` - - Filter for `network` operator. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n openshift-multus - oc get clusteroperators network - ``` - -4. **Fix interface name conflicts** (if VM has duplicate names): - - Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. - - Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. - - **CLI Fallback** (interactive edit easier via CLI): - Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' - oc edit vm -n - ``` - -5. **Validate NAD configuration syntax**: - - Use `resources_get` from diagnostic step 3, extract `.spec.config`. - - Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get network-attachment-definition -n -o jsonpath='{.spec.config}' - ``` - -6. **Check SR-IOV device availability** (if using SR-IOV networks): - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - List SR-IOV network node policies: - - **Parameters**: - ```json - { - "apiVersion": "sriovnetwork.openshift.io/v1", - "kind": "SriovNetworkNodePolicy", - "namespace": "openshift-sriov-network-operator" - } - ``` - - Check SR-IOV device plugin pods: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "openshift-sriov-network-operator" - } - ``` - - Filter for pods with "device-plugin" in name. - - Check available SR-IOV devices on node: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "Node", - "name": "" - } - ``` - - Review `.status.allocatable` for SR-IOV resources. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get sriovnetworknodepolicy -n openshift-sriov-network-operator - oc get pods -n openshift-sriov-network-operator | grep device-plugin - oc describe node | grep -A 10 "Allocatable:" - ``` - -7. **Recreate VM with corrected network configuration** (if needed): - - If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -After remediation, check virt-launcher pod network status: - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -Should show all attached networks with status. Example output: -```json -[ - { - "name": "openshift-sdn", - "interface": "eth0", - "ips": ["10.128.2.10"], - "default": true - }, - { - "name": "vlan100", - "interface": "net1", - "ips": ["192.168.100.5"] - } -] -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**Check from inside VM** (via console): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# In guest OS: -ip addr show -# Should show all network interfaces (eth0, net1, etc.) -``` - -**Common Network Types**: -- **Linux Bridge**: Layer 2 bridge for VLAN networks -- **SR-IOV**: High-performance direct device assignment -- **macvlan**: MAC-based VLAN for container networks -- **OVN-Kubernetes**: OpenShift native overlay network - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/runtime-errors.md deleted file mode 100644 index 804d3cbc..00000000 --- a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/runtime-errors.md +++ /dev/null @@ -1,616 +0,0 @@ ---- -title: VM Runtime Errors -category: kubevirt -sources: - - title: Kubernetes Pod Lifecycle - url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ - date_accessed: 2026-02-17 -tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] -semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] -use_cases: [vm-creation, vm-lifecycle, diagnostics] -related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Runtime Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. - -**When to use this document**: -- VM or virt-launcher pod shows `CrashLoopBackOff` status -- virt-launcher pod repeatedly restarting -- Guest OS kernel panics on boot - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### CrashLoopBackOff - -**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting - -**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. - -**Possible Causes**: -- Guest OS kernel panic on boot -- Insufficient resources (memory/CPU) for guest OS -- Corrupted disk image or filesystem -- QEMU/libvirt crashes due to configuration errors -- Missing or incompatible device drivers in guest -- Resource limits too low for virt-launcher pod -- Virtualization features (KVM) not available on node - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check virt-launcher pod restart count**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# Look at RESTARTS column -``` - -**2. View recent crash logs** (previous container instance): - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "previous": true, - "tail": 100 -} -``` - -Look for QEMU errors, kernel panics, or segfaults. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n virt-launcher--xxx --previous -``` - -**3. Check current virt-launcher logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. - -**CLI Fallback** (if MCP unavailable or all containers needed): -```bash -oc logs -n virt-launcher--xxx --all-containers -``` - -**4. Check VMI conditions for crash details**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Extract `.status.conditions` for crash details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**5. Check pod events for crash reasons**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" -``` - -**6. Check pod resource limits**: - -**MCP Tool**: `pods_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -Extract `.spec.containers[0].resources` for resource limits. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq -``` - -**7. Check node kubelet logs for OOM kills**: - -⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -oc adm node-logs -u kubelet | grep -i oom -``` - -**8. Access guest console** (if VM briefly starts): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Crash Patterns**: - -1. **Guest Kernel Panic**: - - Console logs show kernel panic messages - - Guest crashes immediately after boot - - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" - -2. **OOM (Out of Memory)**: - - Pod killed with reason: `OOMKilled` - - Guest runs out of memory during boot or operation - - virt-launcher logs show memory allocation failures - -3. **QEMU Crash**: - - virt-launcher logs show QEMU segmentation fault - - Symptoms: "qemu-system-x86_64: terminated by signal" - - Configuration incompatibility or QEMU bug - -4. **Disk Image Corruption**: - - Guest cannot boot from disk - - Filesystem errors in guest console - - DataVolume import failed - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for kernel panic or boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for: - - Kernel panic messages - - Initramfs errors - - Filesystem mounting failures - - Missing device errors - -2. **Review virt-launcher crash logs**: - - Use `pods_log` with `previous: true` from diagnostic step 2. - - Look for: - - QEMU command line errors - - Device initialization failures - - Memory allocation errors - - Signal termination (SIGSEGV, SIGABRT) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx --previous - ``` - -3. **Check for OOM (Out of Memory) kills**: - - **MCP Tool**: `pods_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - Extract `.status.containerStatuses[0].lastState.terminated.reason`. - - If returns `"OOMKilled"`: - - Option 1: Increase virt-launcher memory limits - - Option 2: Decrease guest memory allocation - - Option 3: Use smaller instance type - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' - ``` - -4. **Increase resources if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check current memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -5. **Verify disk image integrity**: - - **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters for list**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" - } - ``` - - **Parameters for specific DV**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check `.status.phase` (should be `Succeeded`). - - If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get datavolume -n -o jsonpath='{.status.phase}' - ``` - -6. **Check virtualization (KVM) availability**: - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -7. **Simplify VM configuration** (eliminate variables): - - Try creating minimal VM using vm-create skill with: - - Small instance type - - No secondary networks - - Simple container disk (e.g., Fedora) - - No cloud-init - - If minimal VM works, add features back one by one. - -8. **Recreate VM with different workload** (test disk image): - - If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) - -After remediation, check pod restart count stops increasing: - -**Parameters for pods**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). - -Check VM reaches Running state: - -**Parameters for VM**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# RESTARTS should stabilize - -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Running -``` - -Verify guest is responsive: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# Should show login prompt or OS console -``` - -**Advanced Debugging**: - -**MCP Tool**: `pods_exec` (from openshift-virtualization) - -Check libvirt domain XML: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["virsh", "dumpxml", "1"] -} -``` - -Check QEMU process: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["ps", "aux"] -} -``` - -Filter output for "qemu" process. - -**CLI Fallback** (if MCP unavailable): -```bash -oc exec -n virt-launcher--xxx -- virsh dumpxml 1 -oc exec -n virt-launcher--xxx -- ps aux | grep qemu -``` - -**Prevention**: -- Start with minimal VM configuration and add complexity gradually -- Use recommended instance types for your workload -- Test disk images before deploying to production -- Ensure nodes have adequate resources and KVM support -- Monitor resource usage and set appropriate limits - ---- - -## Workaround Patterns for MCP Tool Limitations - -### General Pattern: Diagnose → Propose → Confirm → Execute - -When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): - -**1. Diagnose** the root cause using Kubernetes commands -```bash -oc describe vm -n -oc get events -n --field-selector involvedObject.name= -``` - -**2. Propose** a manual workaround to the user with clear explanation -- Explain why the issue occurred -- Show what will be changed -- List alternative options - -**3. Confirm** with user before executing (Human-in-the-Loop) -- Wait for explicit "yes" or "apply workaround" -- Never auto-execute modifications - -**4. Execute** the workaround using oc/kubectl -```bash -oc patch vm -n --type=merge -p '...' -``` - -**5. Verify** the fix was successful -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**6. Document** the limitation and suggest filing enhancement request -- Note this is temporary until MCP tool is enhanced -- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues - -### Example: Adding Tolerations Workaround - -**Diagnostic Output**: -```markdown -## ⚠️ VM Scheduling Issue Detected - -**Root Cause**: Node taints prevent VM scheduling - -**Details**: -- Found 3 nodes with taint: `virtualization=true:NoSchedule` -- VM spec does not include matching tolerations -- This prevents VM scheduling on virtualization-dedicated nodes -``` - -**Proposed Workaround**: -```bash -oc patch vm web-server -n vms --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' -``` - -**User Confirmation Required**: -``` -How would you like to proceed? -- "apply workaround" - I'll patch the VM with tolerations -- "manual" - I'll provide instructions for you to apply manually -- "cancel" - Delete the VM and abort creation -``` - -**After Execution**: -```markdown -## ✓ Workaround Applied Successfully - -**Action**: Added tolerations for taint `virtualization=true:NoSchedule` -**New Status**: Stopped (VM can now be scheduled) - -**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. -Future VMs in this cluster will need the same fix until the tool is enhanced. -``` - ---- - -## VM Status Reference - -### Status Values - -| Status | Meaning | Action Required | -|--------|---------|-----------------| -| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | -| `Running` | VM is running | Normal | -| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | -| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | -| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | -| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | -| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | -| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | -| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | -| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | - -### Checking VM Status - -```bash -# Get printable status -oc get vm -n -o jsonpath='{.status.printableStatus}' - -# Get detailed status and conditions -oc get vm -n -o jsonpath='{.status}' | jq - -# Watch status changes in real-time -oc get vm -n -w -``` - ---- - -## Best Practices for Agents - -When implementing diagnostic workflows: - -1. **Always verify VM status** after creation (wait 5-10 seconds first) -2. **Consult this document** when encountering error status values -3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) -4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) -5. **Respect human-in-the-loop** for all VM modifications -6. **Document temporary workarounds** and their limitations clearly -7. **Suggest filing issues** for missing MCP tool features - -### Document Consultation Pattern - -```markdown -**Document Consultation** (REQUIRED): -1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes -2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." -``` - ---- - -## Known MCP Tool Limitations - -### vm_create tool - -**Currently Supported**: -- ✓ Namespace, name (required) -- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) -- ✓ Size hints (small, medium, large) -- ✓ Storage size -- ✓ Autostart flag -- ✓ Networks (Multus NetworkAttachmentDefinitions) -- ✓ Performance family (u1, o1, c1, m1) -- ✓ Instance type, preference - -**Not Currently Supported** (requires workarounds): -- ✗ Tolerations (for node taints) -- ✗ Node selectors -- ✗ Affinity/anti-affinity rules -- ✗ Resource requests/limits (beyond instance type) -- ✗ Custom labels/annotations -- ✗ SSH keys injection -- ✗ Cloud-init user data - -**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. - -**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues - ---- - -## Additional Resources - -- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) -- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) -- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) -- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) - ---- - -[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/scheduling-errors.md deleted file mode 100644 index e47c8c4f..00000000 --- a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/scheduling-errors.md +++ /dev/null @@ -1,417 +0,0 @@ ---- -title: VM Scheduling Errors -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 -tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] -semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] -use_cases: [vm-creation, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Scheduling Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**When to use this document**: -- VM shows status `ErrorUnschedulable` after creation or start attempt -- VM events mention scheduling failures, taints, resources, or node selectors - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -## ErrorUnschedulable - -**Symptom**: VM shows status `ErrorUnschedulable` after creation - -**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**Possible Causes**: - -### 1. Node Taints (Most Common) - -Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM events for scheduling failures**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `involvedObject.name` == "" and look for messages like: -- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= -``` - -**2. Check node taints in the cluster**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' -``` - -**Common Taint Patterns**: -- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule -- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes -- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads - -**Solution - Add Tolerations to VM**: - -The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. - -**Workaround (post-creation using MCP Tools)**: - -**Step 1**: Get current VM spec - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Step 2**: Modify the returned JSON to add tolerations - -Add to `.spec.template.spec.tolerations`: -```json -{ - "tolerations": [ - { - "key": "virtualization", - "operator": "Equal", - "value": "true", - "effect": "NoSchedule" - } - ] -} -``` - -**Step 3**: Update VM with modified spec - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters**: -```json -{ - "resource": "" -} -``` - -Pass the complete modified VM resource as YAML or JSON string. - -**Step 4**: Verify tolerations were added - -Use `resources_get` again and check `.spec.template.spec.tolerations` in response. - -**Step 5**: Check if VM status improved - -Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. - -**CLI Fallback** (if MCP patch is too complex): -```bash -# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' - -# Verify tolerations -oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq - -# Check status -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**Example - Multiple Tolerations**: -```bash -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" - - key: "dedicated" - operator: "Equal" - value: "virt-workloads" - effect: "NoSchedule" -' -``` - -**Toleration Operators**: -- `Equal` - Key and value must match exactly -- `Exists` - Only key must exist (ignores value) - -**Toleration Effects**: -- `NoSchedule` - Don't schedule new pods (existing pods continue) -- `PreferNoSchedule` - Avoid scheduling if possible -- `NoExecute` - Don't schedule AND evict existing pods - -**Alternative Solutions**: -1. **Remove node taints** (if you have cluster-admin access): - ```bash - oc adm taint nodes virtualization=true:NoSchedule- - ``` - -2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits - -3. **File enhancement request** - Request tolerations support in openshift-mcp-server: - https://github.com/openshift/openshift-mcp-server/issues - ---- - -### 2. Insufficient Resources - -Not enough CPU, memory, or storage available on any node. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM resource requests**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' -``` - -**2. Check node resource availability**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. - -Alternatively, use `nodes_top` MCP tool for current resource usage. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe nodes | grep -A 5 "Allocated resources" -``` - -**3. Look for VM events mentioning "Insufficient"**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep "Insufficient" -``` - -**Example Event**: -``` -0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. -``` - -**Solutions** (Use MCP Tools First): - -1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) -2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill -3. **Delete unused VMs** - Use vm-delete skill to free up resources -4. **Check resource quotas**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters for quota**: - ```json - { - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" - } - ``` - - **Parameters for limit range**: - ```json - { - "apiVersion": "v1", - "kind": "LimitRange", - "namespace": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - oc describe limitrange -n - ``` - ---- - -### 3. Node Selector Mismatch - -VM requires specific node labels that don't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM node selector requirements**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.nodeSelector` to see required node labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' -``` - -**2. List available node labels**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.metadata.labels` for available labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes --show-labels -``` - -**3. Check if any nodes match the selector**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node", - "labelSelector": "=" -} -``` - -Should return at least one node with matching labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -l = -``` - -**Solutions** (Use MCP Tools First): - -**Option 1: Remove node selector from VM** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get current VM using `resources_get` (diagnostic step 1) -2. Remove `.spec.template.spec.nodeSelector` field -3. Update VM using `resources_create_or_update` with modified JSON - -**CLI Fallback** (JSON patch easier via CLI): -Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" -```bash -oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' -``` - -**Option 2: Add label to nodes** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get node using `resources_get` -2. Add label to `.metadata.labels` -3. Update node using `resources_create_or_update` - -⚠️ **Note**: Node labeling typically requires cluster admin privileges. - -**CLI Fallback** (simpler via CLI): -Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" -```bash -oc label node = -``` - ---- - -[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/storage-errors.md deleted file mode 100644 index 16ba584d..00000000 --- a/evaluation/with_skills/rh-virt__vm-delete/environment/docs/troubleshooting/storage-errors.md +++ /dev/null @@ -1,1011 +0,0 @@ ---- -title: VM Storage Errors -category: kubevirt -sources: - - title: KubeVirt CDI - DataVolumes - url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ - date_accessed: 2026-02-17 - - title: Kubernetes Persistent Volumes - url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ - date_accessed: 2026-02-17 -tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] -semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] -use_cases: [vm-creation, vm-deletion, vm-cloning] -related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] -last_updated: 2026-02-17 ---- - -# VM Storage Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. - -**When to use this document**: -- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` -- Storage deletion fails after VM deletion -- DataVolume cloning operations fail -- PVC provisioning issues - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### ErrorDataVolumeNotReady - -**Symptom**: VM shows status `ErrorDataVolumeNotReady` - -**Description**: The DataVolume (persistent storage) backing the VM is not ready. - -**Possible Causes**: - -#### 1. DataVolume Still Provisioning - -Storage provisioning takes time, especially for large disks or when importing images. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Get detailed DataVolume information**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` and `.status.conditions` for provisioning details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o json -``` - -**3. Check PVC (PersistentVolumeClaim) bound status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Check `.status.phase` for each PVC (should be `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. - -#### 2. Storage Class Not Found - -The requested storage class doesn't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List available storage classes**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" -} -``` - -Review the list of available storage classes (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -``` - -**2. Check DataVolume's requested storage class**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.spec.pvc.storageClassName` in the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' -``` - -**Solution**: -1. Use a valid storage class from the cluster -2. Recreate VM with correct storage class parameter - -#### 3. Insufficient Storage Quota - -Namespace has insufficient storage quota to provision the PVC. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check resource quotas**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**2. Check storage usage**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase -``` - -**Solution**: -1. Request quota increase from cluster admin -2. Delete unused PVCs to free quota -3. Reduce VM storage size - ---- - -### ErrorPvcNotFound - -**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List PVCs in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review the list of available PVCs (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check VM's PVC references**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' -``` - -**Solution**: -- Wait for DataVolume to create the PVC -- Manually create missing PVC -- Fix VM spec to reference correct PVC name - ---- - - ---- - -### Storage Deletion Failures - -**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace - -**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. - -**Possible Causes**: -- PVC still bound to active PersistentVolume with `Retain` policy -- DataVolume still being referenced by another resource -- CDI (Containerized Data Importer) controller issues -- Storage class retention policy preventing deletion -- Finalizers on PVC/DataVolume blocking cleanup -- PVC still mounted by a pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check PVC status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check specific PVC phase**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Released` or `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o jsonpath='{.status.phase}' -``` - -**3. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**4. Check what's using the PVC**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' -``` - -**5. Check PVC finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. - -**6. Check DataVolume finalizers**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.metadata.finalizers}' -``` - -**7. Check PV reclaim policy**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume" -} -``` - -Filter results for PV where `.spec.claimRef.name` matches ``. - -To get specific PV policy: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume", - "name": "" -} -``` - -Check `.spec.persistentVolumeReclaimPolicy`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pv | grep -oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' -``` - -**Common Finalizer Patterns**: -- `kubernetes.io/pvc-protection` - Protects PVC while in use -- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer - -**Solutions** (Use MCP Tools First): - -1. **Delete DataVolume first, then PVC**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete DataVolume first (often blocks PVC deletion): - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Wait a few seconds, then delete PVC: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete datavolume -n - oc delete pvc -n - ``` - -2. **Check for pods still using PVC**: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "" - } - ``` - - Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. - - Then delete the pods using `pods_delete`: - - **Parameters**: - ```json - { - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' - oc delete pod -n - ``` - -3. **Force delete PVC** (if safe to do so): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete pvc -n --grace-period=0 --force - ``` - -4. **Remove finalizers from PVC** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PVC using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update PVC using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Remove finalizers from DataVolume** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current DataVolume using `resources_get` (diagnostic step 6) - 2. Remove items from `.metadata.finalizers` array - 3. Update DataVolume using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Change PV reclaim policy** (if PV has Retain policy): - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PV policy using `resources_get` (diagnostic step 7) - 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` - 3. Update PV using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" - ```bash - oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' - oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' - ``` - -**Storage Quota Check** (Use MCP Tools First): - -After deletion, verify storage quota is freed: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters for quota check**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.used` to verify storage quota is freed. - -**Parameters for PVC verification**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -oc get pvc -n -``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Confirm PVC is deleted: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Confirm DataVolume is deleted: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -# Should return: Error from server (NotFound) - -oc get datavolume -n -# Should return: Error from server (NotFound) -``` - ---- - -### DataVolume Cloning Failures - -**Symptom**: VM clone created successfully but DataVolume clone operation fails - -**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. - -**Possible Causes**: -- CSI driver doesn't support volume cloning -- Source PVC storage class incompatible with cloning -- Cross-namespace cloning not permitted by storage backend -- Insufficient storage quota in target namespace -- Source PVC not in `Bound` state -- Storage class doesn't have volume cloning enabled -- CDI (Containerized Data Importer) controller issues - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume clone status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and `.metadata.name`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Check DataVolume events for errors**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe datavolume -n -``` - -**3. Check DataVolume phase**: - -Use `resources_get` from step 1, extract `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -``` - -**4. Check if storage class supports cloning**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass", - "name": "" -} -``` - -Review the full YAML output for cloning-related configurations. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -o yaml | grep -A 5 -i clone -``` - -**5. Check CSI driver capabilities**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "CSIDriver" -} -``` - -Review `.items[].metadata.name` for available CSI drivers. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get csidriver -``` - -**6. Check source PVC status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Bound` for cloning to work). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**7. Check target namespace storage quota**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**8. Check CDI controller logs**: - -**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) - -First, list pods in openshift-cnv namespace: - -**Parameters for pods_list_in_namespace**: -```json -{ - "namespace": "openshift-cnv", - "labelSelector": "app.kubernetes.io/component=cdi-deployment" -} -``` - -Then get logs using `pods_log`: - -**Parameters**: -```json -{ - "namespace": "openshift-cnv", - "name": "", - "tail": 100 -} -``` - -**CLI Fallback** (if MCP unavailable or easier via CLI): -```bash -oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') -``` - -**Common Error Messages**: -- `"volume cloning is not supported"` - CSI driver lacks clone capability -- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage -- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace -- `"insufficient quota"` - Target namespace lacks storage quota -- `"source PVC not bound"` - Source PVC must be in Bound state for cloning -- `"StorageClass does not support cloning"` - Storage class configuration issue - -**Solutions** (Use MCP Tools First): - -1. **Check storage class clone support**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" - } - ``` - - For each storage class in `.items[]`, check: - - `.metadata.name` (storage class name) - - `.provisioner` (CSI driver) - - Storage classes using CSI drivers typically support cloning. Look for provisioners like: - - `csi.ovirt.org` (oVirt CSI) - - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) - - `ebs.csi.aws.com` (AWS EBS CSI) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner - ``` - -2. **Verify source PVC is bound**: - - Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pvc -n -o jsonpath='{.status.phase}' - ``` - -3. **Check target namespace quota**: - - Use `resources_list` from diagnostic step 7 to check quota. - - If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - ``` - -4. **Use snapshot-based cloning** (alternative method): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Step 1**: Create VolumeSnapshot of source PVC - - **Parameters**: - ```json - { - "apiVersion": "snapshot.storage.k8s.io/v1", - "kind": "VolumeSnapshot", - "metadata": { - "name": "-snapshot", - "namespace": "" - }, - "spec": { - "source": { - "persistentVolumeClaimName": "" - } - } - } - ``` - - **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) - - **Step 3**: Create new DataVolume from snapshot - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "metadata": { - "name": "-rootdisk", - "namespace": "" - }, - "spec": { - "source": { - "snapshot": { - "name": "-snapshot", - "namespace": "" - } - }, - "storage": { - "resources": { - "requests": { - "storage": "50Gi" - } - }, - "storageClassName": "" - } - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" - ```bash - cat <-snapshot - namespace: - spec: - source: - persistentVolumeClaimName: - EOF - - oc get volumesnapshot -snapshot -n - - cat <-rootdisk - namespace: - spec: - source: - snapshot: - name: -snapshot - namespace: - storage: - resources: - requests: - storage: 50Gi - storageClassName: - EOF - ``` - -5. **Use "new empty storage" option** (vm-clone skill): - - If cloning isn't supported, create VM with empty storage - - Manually copy data if needed - -6. **Cross-namespace cloning workaround**: - - Some storage backends require snapshot for cross-namespace cloning - - Create snapshot in source namespace, restore in target namespace (see solution 4 above) - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Check DataVolume reached Succeeded phase: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should return `Succeeded`). - -Check PVC was created and bound: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "-rootdisk" -} -``` - -Check `.status.phase` (should return `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -# Should return: Succeeded - -oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' -# Should return: Bound -``` - -**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-delete/environment/skills/vm-delete/SKILL.md b/evaluation/with_skills/rh-virt__vm-delete/environment/skills/vm-delete/SKILL.md deleted file mode 100644 index ee231a9a..00000000 --- a/evaluation/with_skills/rh-virt__vm-delete/environment/skills/vm-delete/SKILL.md +++ /dev/null @@ -1,403 +0,0 @@ ---- -name: vm-delete -description: | - Permanently delete virtual machines and their associated resources from OpenShift Virtualization. - - Use when: - - "Delete VM [name]" - - "Remove virtual machine [name]" - - "Destroy VM [name]" - - "Clean up VM [name]" - - This skill handles permanent VM deletion with strict safety confirmations and typed verification. - - NOT for power management (use vm-lifecycle-manager to stop VMs). - -model: inherit -color: red ---- - -# /vm-delete Skill - -Permanently delete virtual machines and their associated resources (storage, DataVolumes) from OpenShift Virtualization clusters. This skill enforces strict safety protocols including typed confirmation and pre-deletion validation. - -## Prerequisites - -**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools**: -- `resources_get` (from openshift-virtualization) - Verify VM exists and get details -- `resources_delete` (from openshift-virtualization) - Delete Kubernetes resources -- `resources_list` (from openshift-virtualization) - List dependent resources (PVCs, DataVolumes) -- `resources_create_or_update` (from openshift-virtualization) - Update resources (e.g., remove finalizers) -- `vm_lifecycle` (from openshift-virtualization) - Stop running VMs before deletion -- `pods_list_in_namespace` (from openshift-virtualization) - List pods for diagnostics - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster (>= 4.19) -- OpenShift Virtualization operator installed -- ServiceAccount with RBAC permissions to delete VirtualMachine and PVC resources - -### Prerequisite Verification - -**Before executing:** - -1. Check `openshift-virtualization` exists in `.mcp.json` → If missing, report setup -2. Verify `KUBECONFIG` is set (presence only, never expose value) → If missing, report -3. Check RBAC permissions (optional) → Verify delete permissions for VirtualMachine and PVC - -**Human Notification Protocol:** `❌ Cannot execute vm-delete: MCP server not available. Setup: Add to .mcp.json, set KUBECONFIG, restart Claude Code. Docs: https://github.com/openshift/openshift-mcp-server` - -⚠️ **SECURITY**: Never display KUBECONFIG path or credential values. - -## When to Use This Skill - -**Trigger when:** -- User explicitly invokes `/vm-delete` command -- User requests permanent VM deletion -- User wants to clean up test/development VMs -- User needs to free cluster resources -- User wants to decommission VMs - -**User phrases:** -- "Delete VM test-vm in namespace dev" -- "Remove virtual machine web-server" -- "Destroy VM old-database" -- "/vm-delete" - -**Do NOT use when:** -- Stop VM temporarily → `/vm-lifecycle-manager` -- Create VM → `/vm-create` -- View VMs → `/vm-inventory` - -## Workflow - -### Step 1: Gather and Validate - -**CRITICAL**: Complete ALL validation BEFORE user confirmation. - -**Required from user:** VM Name, Namespace - -**1.1: Verify VM Exists** - -**MCP Tool**: `resources_get` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", namespace=``, name=``) - -**Errors:** -- Not found → Report error, suggest vm-inventory -- Permission denied → Report RBAC error - -**1.2: Check Protection Label** - -Check `metadata.labels` for `protected: "true"`. - -**If protected:** Report: `❌ Cannot Delete Protected VM. VM has protected label. Remove: oc label vm -n protected-. Operation cancelled.` **STOP workflow.** - -**1.3: Check Running State** - -Check `status.printableStatus` (Running/Starting/Migrating = running, Stopped/Halted = stopped). - -**If running:** Report: `⚠️ VM Running. Must stop before deletion. Options: stop-and-delete / cancel` **Wait for response.** - -**1.4: Stop VM (if applicable)** - -**ONLY if user chose "stop-and-delete".** - -**MCP Tool**: `vm_lifecycle` (namespace=``, name=``, action="stop") - -Report: `⏸️ Stopping VM... Wait 10-30s.` **Wait 10s**, verify stopped. - -**1.5: Discover Storage** - -**MCP Tool**: `resources_list` - -**DataVolumes**: apiVersion="cdi.kubevirt.io/v1beta1", kind="DataVolume", namespace=``, labelSelector="vm.kubevirt.io/name=``" - -**PVCs** (if no DVs): apiVersion="v1", kind="PersistentVolumeClaim", namespace=``, labelSelector="vm.kubevirt.io/name=``" - -Parse: Extract names, calculate total storage size. - -### Step 2: Present Scope and Get Options - -Display deletion scope in this format: - -```markdown -## ⚠️ VM Deletion - Review Scope - -**VM**: `` | **Namespace**: `` | **Status**: - -**Resources**: VM `` (Age: , vCPU: , Memory: ) -**Storage**: DataVolume `` (30Gi), PVC `` (30Gi) - Total: 30Gi -OR **Storage**: None (ephemeral) - -### Deletion Options -**1: VM Only** - Preserves storage for reuse -**2: VM + Storage** ← Recommended (test/dev) - Frees storage -**3: Cancel** - -Select (1, 2, or 3): -``` - -**Wait for selection.** Handle: 3→Cancel, 1→delete_storage=false, 2→delete_storage=true - -### Step 3: Typed Confirmation (MANDATORY) - -**CRITICAL**: User MUST type exact VM name. - -Display typed confirmation prompt (adjust based on delete_storage flag): - -```markdown -## 🔴 PERMANENT DELETION - Typed Confirmation Required - -**CANNOT BE UNDONE** - -**Will delete**: -✗ VirtualMachine: `` (namespace: ``) -[If delete_storage=true, show:] -✗ DataVolume: `` | ✗ PVC: `` | ✗ All data lost -[If delete_storage=false, show:] -✓ Storage PRESERVED - -Type `` to confirm: _____ -``` - -**Validation:** -- Match → Continue to Step 4 (Execute Deletion) -- Mismatch → Report: `❌ Confirmation Failed. You typed: . Expected: . Cancelled.` **STOP.** - -### Step 4: Execute Deletion - -**ONLY AFTER**: ✓ Validation ✓ Option selected ✓ Typed name confirmed - -**4.1: Delete VM** - -**MCP Tool**: `resources_delete` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", namespace=``, name=``) - -**Errors:** Fails → Report, don't delete storage; Not found → Continue - -Report: `🗑️ Deleting VM... ✓ Deleted` - -**4.2: Delete Storage (if delete_storage=true)** - -**For each DataVolume:** -**MCP Tool**: `resources_delete` (apiVersion="cdi.kubevirt.io/v1beta1", kind="DataVolume", namespace=``, name=``) - -**For each PVC:** -**MCP Tool**: `resources_delete` (apiVersion="v1", kind="PersistentVolumeClaim", namespace=``, name=``) - -**Errors:** Report which failed, continue with others - -Report: `🗑️ Deleting storage... ✓ DV deleted (storage freed) ✓ PVC deleted` - -### Step 5: Report Results - -**Success (with storage):** - -```markdown -## ✓ VM Deleted (Complete Cleanup) -**Deleted**: VM + DataVolume + PVC | **Freed**: -**Impact**: Permanent removal. Cannot recover. -**Verify**: "List VMs in namespace " - VM should not appear -``` - -**Success (storage preserved):** - -```markdown -## ✓ VM Deleted (Storage Preserved) -**Deleted**: VM | **Preserved**: DataVolume + PVC () -**Reuse**: Create new VM with existing DV/PVC -**Delete later**: `oc delete datavolume -n ` -``` - -**Partial failure (storage failed):** - -**OPTIONAL**: Read [storage-errors.md](../../docs/troubleshooting/storage-errors.md) for PVC cleanup. Output: "Consulted storage-errors.md for failure." - -```markdown -## ⚠️ Partial Deletion -**Deleted**: VM | **Failed**: DV/PVC (error: ) -**Action**: Manual cleanup: `oc delete datavolume -n ` -``` - -**Complete failure:** - -**OPTIONAL**: Read [lifecycle-errors.md](../../docs/troubleshooting/lifecycle-errors.md) for deletion failures. Output: "Consulted lifecycle-errors.md for failure causes." - -```markdown -## ❌ VM Deletion Failed -**Error**: -**Troubleshooting**: Check permissions, verify VM exists, check finalizers (see lifecycle-errors.md) -``` - -## Common Issues - -### Issue 1: VM Not Found -**Error**: "VirtualMachine not found" -**Solution**: Verify name/namespace with vm-inventory. Check spelling. - -### Issue 2: RBAC Permissions -**Error**: "Forbidden: Cannot delete VirtualMachines" -**Solution**: Verify delete permissions for VirtualMachine and PVC. Contact admin. Check: `oc auth can-i delete virtualmachines -n ` - -### Issue 3: VM Has Finalizers -**Error**: "VM deletion blocked by finalizers" -**Solution**: Consult [lifecycle-errors.md](../../docs/troubleshooting/lifecycle-errors.md) "VM Stuck in Terminating" for MCP-first approach using `resources_get` to check finalizers, `resources_create_or_update` to remove if needed. - -### Issue 4: Storage Deletion Failure -**Error**: "PVC deletion failed: resource in use" -**Solution**: Verify VM deleted first. Consult [storage-errors.md](../../docs/troubleshooting/storage-errors.md) for MCP-first diagnostics using `pods_list_in_namespace` to check mounts, `resources_get` for PVC status. - -### Issue 5: Confirmation Mismatch -**Error**: "Names do not match" -**Solution**: Type exact VM name (case-sensitive). Copy-paste from deletion scope. Retry. - -### Issue 6: Protected VM -**Error**: "VM has protected label" -**Solution**: Remove: `oc label vm -n protected-`. Retry deletion. - -## Dependencies - -### Required MCP Servers -- `openshift-virtualization` - OpenShift MCP with KubeVirt toolset (https://github.com/openshift/openshift-mcp-server) - -### Required MCP Tools -- `resources_get` - Get VM (apiVersion, kind, namespace, name) -- `resources_delete` - Delete resources (apiVersion, kind, namespace, name) -- `resources_list` - List resources (apiVersion, kind, namespace, labelSelector) -- `resources_create_or_update` - Update resources (resource JSON) - for finalizer removal -- `vm_lifecycle` - VM lifecycle (namespace, name, action: stop) -- `pods_list_in_namespace` - List pods (namespace) - for PVC mount diagnostics - -### Related Skills -- `vm-lifecycle-manager` - Stop VMs | `vm-inventory` - List VMs | `vm-create` - Create VMs | `vm-clone` - Clone VMs - -### Reference Documentation -- [lifecycle-errors.md](../../docs/troubleshooting/lifecycle-errors.md) - Deletion failures, finalizers, stuck Terminating (consulted on deletion failure) -- [storage-errors.md](../../docs/troubleshooting/storage-errors.md) - Storage deletion, PVC cleanup (consulted on storage failure) -- [Troubleshooting INDEX](../../docs/troubleshooting/INDEX.md) - Full error index -- [OpenShift Virt Docs](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index) -- [KubeVirt API](https://kubevirt.io/api-reference/) -- [K8s Finalizers](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/) - -## Critical: Human-in-the-Loop Requirements - -**CRITICAL: IRREVERSIBLE operations.** You MUST: - -1. **Pre-Deletion Validation** (Execute FIRST) - - Verify VM exists (`resources_get`) - - Check VM running state - - Discover dependent resources - - Check protection labels (`protected: "true"`) - -2. **Safety Checks** - - **REFUSE deletion** if protected label exists - - **REQUIRE VM stopped** if running - - **List all resources** to be deleted - -3. **Typed Confirmation (MANDATORY)** - - Display complete deletion scope - - **Require exact VM name** to confirm - - Accept only exact match (case-sensitive) - - Mismatch → Cancel, don't proceed - - Match → Proceed directly to deletion - -4. **Deletion Options** - - Ask: VM only / VM+Storage / Cancel - - **NEVER assume** which option - -5. **Never Auto-Execute** - - NEVER delete without typed confirmation - - NEVER proceed if typed name mismatches - - NEVER skip typed verification - -**Why**: Permanent data loss, service impact, accidental deletion prevention - -## Security Considerations - -- **RBAC**: Requires delete VirtualMachine, PVC, DataVolume -- **Typed Verification**: Prevents accidental deletion -- **Protection Labels**: `protected: "true"` blocks deletion -- **Audit**: Kubernetes audit logs -- **Graceful Shutdown**: Stops VMs before deletion -- **KUBECONFIG**: Never exposed -- **Namespace Isolation**: RBAC-enforced -- **No Force Delete**: Never uses `--force` or `--grace-period=0` - -## Example Usage - -### Example 1: Complete Deletion (VM + Storage) - -``` -User: "Delete test-vm in dev" -Agent: [Validates ✓, discovers 30Gi storage] - [Shows deletion scope - see Step 2 format] -User: "2" [VM+Storage] -Agent: [Shows typed confirmation - see Step 3 format] -User: "test-vm" -Agent: [Executes: VM ✓, DV ✓, PVC ✓] - [Shows success report - see Step 5 format] -## ✓ VM Deleted (Complete Cleanup) | Freed: 30Gi -``` - -### Example 2: VM Only (Preserve Storage) - -``` -User: "Remove web-server but keep storage" -Agent: [Validates, shows scope] -User: "1" [VM only] -Agent: [Typed confirmation] -User: "web-server" -Agent: [Executes deletion] -## ✓ VM Deleted (Storage Preserved) | Preserved: 50Gi -``` - -### Example 3: Protected VM (Blocked) - -``` -User: "Delete production-db" -Agent: [Validates: protected label found] -❌ Cannot Delete Protected VM. Remove label: oc label vm production-db protected- -``` - -### Example 4: Running VM (Stop First) - -``` -User: "Delete api-server" -Agent: [Validates: VM running] -⚠️ VM Running. Options: stop-and-delete / cancel -User: "stop-and-delete" -Agent: [Stops VM, waits, continues with deletion] -``` - -### Example 5: User Cancels - -``` -User: "Delete test-vm" -Agent: [Shows typed confirmation] -Type `test-vm` to confirm: -User: "wait, cancel" -❌ Confirmation Failed. You typed: wait, cancel. Expected: test-vm. Cancelled. -``` - -### Example 6: Dry-Run Preview - -``` -User: "What would be deleted if I delete test-vm?" -Agent: [Execute Step 1-2 only, stop before confirmation] -## 🔍 Deletion Preview -**Would delete (Option 2)**: VM + DV + PVC (30Gi freed) -This is preview only. No resources deleted. -``` - -## Advanced Features - -### Batch Deletion -Delete multiple VMs with confirmation for each: `"Delete VMs test-01, test-02, test-03 in dev"` → Process each individually with full workflow. Use typed confirmation: `DELETE-3-VMS` for batch. - -### Dry-Run Mode -Show deletion scope without executing: Execute Step 1-2, skip Steps 3-4. User request: "Show what would be deleted if I delete VM xyz" - -### Protected VM Label -Automatic enforcement: If VM has `protected: "true"` label, refuse deletion in Step 1.2. Example YAML: `metadata.labels.protected: "true"` diff --git a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/.ai-index/semantic-index.json deleted file mode 100644 index 2512c2a3..00000000 --- a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/.ai-index/semantic-index.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "index_version": "1.0.0", - "last_updated": "2026-02-17", - "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", - - "documents": [ - { - "file": "INDEX.md", - "title": "VM Troubleshooting Guide - Index", - "category": "navigation", - "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], - "use_cases": ["navigation", "tool-reference", "general-guidance"], - "token_estimate": 450, - "skills": ["all"] - }, - { - "file": "scheduling-errors.md", - "title": "VM Scheduling Errors", - "category": "scheduling", - "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], - "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], - "errors_covered": [ - "ErrorUnschedulable - Node Taints", - "ErrorUnschedulable - Insufficient Resources", - "ErrorUnschedulable - Node Selector Mismatch" - ], - "token_estimate": 820, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "storage-errors.md", - "title": "VM Storage Errors", - "category": "storage", - "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], - "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], - "errors_covered": [ - "ErrorDataVolumeNotReady - DataVolume Still Provisioning", - "ErrorDataVolumeNotReady - Storage Class Not Found", - "ErrorDataVolumeNotReady - Insufficient Storage Quota", - "ErrorPvcNotFound", - "Storage Deletion Failures", - "DataVolume Cloning Failures" - ], - "token_estimate": 1350, - "skills": ["vm-creator", "vm-delete", "vm-clone"] - }, - { - "file": "lifecycle-errors.md", - "title": "VM Lifecycle Errors", - "category": "lifecycle", - "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], - "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], - "errors_covered": [ - "VM Stuck in Terminating State", - "VM Won't Start (Non-Scheduling Issues)", - "VM Won't Stop" - ], - "token_estimate": 1150, - "skills": ["vm-delete", "vm-lifecycle-manager"] - }, - { - "file": "runtime-errors.md", - "title": "VM Runtime Errors", - "category": "runtime", - "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], - "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], - "errors_covered": [ - "CrashLoopBackOff" - ], - "token_estimate": 850, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "network-errors.md", - "title": "VM Network Errors", - "category": "networking", - "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], - "use_cases": ["vm-creation", "network-troubleshooting"], - "errors_covered": [ - "Network Attachment Failures" - ], - "token_estimate": 580, - "skills": ["vm-creator"] - } - ], - - "skill_to_docs_mapping": { - "vm-creator": { - "primary_docs": ["scheduling-errors.md", "storage-errors.md"], - "secondary_docs": ["runtime-errors.md", "network-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3650, - "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" - }, - "vm-delete": { - "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 2950, - "description": "VM deletion failures - stuck terminating, storage cleanup" - }, - "vm-clone": { - "primary_docs": ["storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 1800, - "description": "VM cloning failures - DataVolume cloning, storage provisioning" - }, - "vm-lifecycle-manager": { - "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], - "secondary_docs": ["runtime-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3270, - "description": "VM start/stop failures - lifecycle management, scheduling" - }, - "vm-inventory": { - "primary_docs": [], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 450, - "description": "General guidance for status interpretation" - } - }, - - "error_to_docs_mapping": { - "ErrorUnschedulable": "scheduling-errors.md", - "ErrorDataVolumeNotReady": "storage-errors.md", - "ErrorPvcNotFound": "storage-errors.md", - "Terminating": "lifecycle-errors.md", - "CrashLoopBackOff": "runtime-errors.md", - "NetworkAttachmentFailed": "network-errors.md" - }, - - "cross_references": { - "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], - "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], - "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], - "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], - "network-errors.md": ["scheduling-errors.md"] - }, - - "progressive_disclosure_strategy": { - "description": "Load only the minimum documentation needed for current task", - "pattern": "error_type → primary_doc → cross_references (if needed)", - "token_savings": "29% average reduction vs single-file (based on rh-sre data)", - "navigation_overhead_reduction": "85% vs single-file navigation" - } -} diff --git a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/INDEX.md deleted file mode 100644 index a2e76963..00000000 --- a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/INDEX.md +++ /dev/null @@ -1,332 +0,0 @@ ---- -title: VM Troubleshooting Guide - Index -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 - - title: OpenShift Virtualization - Virtual Machine Status - url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html - date_accessed: 2026-02-06 - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] -semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] -use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] -last_updated: 2026-02-17 ---- - -# VM Troubleshooting Guide - Index - -## Overview - -This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: - -- **Creation**: Scheduling failures, storage provisioning issues -- **Lifecycle**: Start/stop failures, stuck states -- **Deletion**: Resources stuck in Terminating, storage cleanup failures -- **Cloning**: DataVolume cloning errors, cross-namespace issues -- **Networking**: Secondary network attachment failures -- **Runtime**: CrashLoopBackOff, guest OS boot failures - -This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. - ---- - -## 🗂️ Troubleshooting Categories - -The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: - -### 1. [Scheduling Errors](scheduling-errors.md) -**When to use**: VM fails to schedule on any node - -**Errors covered**: -- ErrorUnschedulable - Node Taints -- ErrorUnschedulable - Insufficient Resources -- ErrorUnschedulable - Node Selector Mismatch - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 2. [Storage Errors](storage-errors.md) -**When to use**: VM has storage provisioning, deletion, or cloning issues - -**Errors covered**: -- ErrorDataVolumeNotReady (all 3 subsections) -- ErrorPvcNotFound -- Storage Deletion Failures -- DataVolume Cloning Failures - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### 3. [Lifecycle Errors](lifecycle-errors.md) -**When to use**: VM has start/stop/termination issues - -**Errors covered**: -- VM Stuck in Terminating State -- VM Won't Start (Non-Scheduling Issues) -- VM Won't Stop - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### 4. [Runtime Errors](runtime-errors.md) -**When to use**: VM repeatedly crashes or fails at runtime - -**Errors covered**: -- CrashLoopBackOff - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 5. [Network Errors](network-errors.md) -**When to use**: VM has secondary network attachment failures - -**Errors covered**: -- Network Attachment Failures (Multus, SR-IOV) - -**Skills that use this**: vm-create - ---- - -## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last - -### Critical Principle: MCP-First Approach - -**All diagnostic procedures in this guide follow the MCP-first pattern**: - -``` -1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) -2. ⚠️ IF FAILS: Ask user permission to use CLI command -3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval -``` - -### Why MCP Tools First? - -- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources -- **Consistency**: Same interface across all operations -- **Better Error Handling**: MCP tools return structured errors -- **Audit Trail**: MCP tool usage is logged and trackable - -### Command Pattern Examples - -Throughout this guide, you'll see diagnostic steps formatted like this: - -**✅ CORRECT Pattern (MCP First)**: - -```markdown -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Extract**: `.metadata.finalizers` from returned JSON - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` -``` - -### Available MCP Tools - -The `openshift-virtualization` MCP server provides these tools: - -**Resource Operations**: -- `resources_get` - Get specific resource (replaces `oc get `) -- `resources_list` - List resources (replaces `oc get `) -- `resources_delete` - Delete resource (replaces `oc delete`) -- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) - -**Pod Operations**: -- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) -- `pods_get` - Get pod details (replaces `oc get pod`) -- `pods_log` - Get pod logs (replaces `oc logs`) -- `pods_exec` - Execute in pod (replaces `oc exec`) -- `pods_delete` - Delete pod (replaces `oc delete pod`) -- `pods_top` - Pod resource usage (replaces `oc top pods`) - -**Events & Monitoring**: -- `events_list` - List events (replaces `oc get events`) -- `nodes_top` - Node resource usage (replaces `oc top nodes`) -- `nodes_log` - Node logs (replaces `oc adm node-logs`) -- `nodes_stats_summary` - Detailed node stats - -**VM Operations** (KubeVirt toolset): -- `vm_create` - Create VMs -- `vm_lifecycle` - Start/stop/restart VMs - -**Namespaces**: -- `namespaces_list` - List namespaces (replaces `oc get namespaces`) -- `projects_list` - List OpenShift projects (replaces `oc get projects`) - -### When CLI Commands Are Required - -Some operations have **NO MCP equivalent** and require CLI: - -- `virtctl` commands (console, VNC access) -- `oc debug node` (node debugging) -- `oc auth can-i` (permission checks) -- `oc adm taint` (node taint management) -- Complex JSON patch operations - -For these, the guide will note: **"CLI Only - No MCP equivalent"** - -### Quick Reference: CLI → MCP Mapping - -| CLI Command | MCP Tool Equivalent | -|-------------|---------------------| -| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | -| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | -| `oc logs -n ` | `pods_log` with name="", namespace="" | -| `oc get events -n ` | `events_list` with namespace="" | -| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | - -**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. - -### How to Read Diagnostic Sections - -Each error section includes: -1. **Symptom** - What you observe -2. **Description** - What's happening -3. **Possible Causes** - Why it's happening -4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback -5. **Solutions** - **MCP tools first**, then CLI fallback -6. **Verification** - **MCP tools first**, then CLI fallback - -**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. - ---- - -## 🔍 Quick Navigation by Skill - -**vm-create**: -- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics -- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning -- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff -- [Network Errors](network-errors.md) - Network attachment failures - -**vm-delete**: -- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state -- [Storage Errors](storage-errors.md) - Storage deletion failures - -**vm-clone**: -- [Storage Errors](storage-errors.md) - DataVolume cloning failures - -**vm-lifecycle-manager**: -- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop -- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling - -**vm-inventory**: -- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed - ---- - -## 📊 Documentation Coverage & Maintenance - -### Current Coverage - -This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: - -- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) -- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures -- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop -- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) -- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems - -**Total errors documented**: 12 error types across 6 categories - ---- - -### Encountering Undocumented Errors - -#### For AI Agents (Claude Code) - -If you encounter an error **not documented** in the categories above: - -1. **Report to user** with all available details (error message, affected resources, namespace) -2. **Provide best-effort diagnostics** using MCP tools: - - `resources_get` to inspect resource status - - `pods_log` to check virt-launcher or compute container logs - - `events_list` to view Kubernetes events -3. **Suggest documentation update**: - ``` - ⚠️ This error is not yet documented in the troubleshooting guide. - - **Error**: - **Affected resource**: / - - I recommend adding this error to the troubleshooting guide: - - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file - - If it's a new error class → Create new category file in docs/troubleshooting/ - - Would you like me to help document this error for future reference? - ``` - -4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs - ---- - -#### For Human Contributors - -To document a new error: - -1. **Determine the category**: - - Scheduling issue → `scheduling-errors.md` - - Storage problem → `storage-errors.md` - - Lifecycle/start/stop → `lifecycle-errors.md` - - Runtime crash → `runtime-errors.md` - - Network issue → `network-errors.md` - - New error class → Create new category file (e.g., `security-errors.md`) - -2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): - - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback - - **Solutions**: MCP Tool approach → Implementation → CLI Fallback - - **Verification**: MCP Tool checks → Expected results → CLI Fallback - -3. **Update semantic index**: - - Edit `.ai-index/semantic-index.json` - - Add error to `error_to_docs_mapping` - - Update relevant skill's `primary_docs` or `secondary_docs` - - Increment token estimates if needed - -4. **Reference in skill files**: - - Update skill's Reference Documentation section - - Ensure skill's Document Consultation steps point to the right category - -5. **Update this INDEX.md**: - - Add error to the appropriate category section (lines 50-120) - - Update "Current Coverage" list above - ---- - -## 📚 Additional Resources - -- [rh-virt README](../../README.md) - MCP server setup and skill documentation -- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources -- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/lifecycle-errors.md deleted file mode 100644 index 6167fcc8..00000000 --- a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/lifecycle-errors.md +++ /dev/null @@ -1,869 +0,0 @@ ---- -title: VM Lifecycle Errors -category: kubevirt -sources: - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 -tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] -semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] -use_cases: [vm-deletion, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Lifecycle Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM lifecycle issues including start/stop failures and stuck termination states. - -**When to use this document**: -- VM stuck in `Terminating` state -- VM won't start (runStrategy is Always but VM never reaches Running) -- VM won't stop (runStrategy is Halted but VM never reaches Stopped) - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### VM Stuck in Terminating State - -**Symptom**: VM shows status `Terminating` but deletion never completes - -**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). - -**Possible Causes**: -- Finalizers blocking deletion -- PVC/DataVolume still attached and preventing cleanup -- VirtualMachineInstance (VMI) not terminating properly -- Custom controllers or operators blocking deletion -- Stuck virt-launcher pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check finalizers on the VM**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` - -**2. Check if VMI still exists**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name containing "virt-launcher-". - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check events for deletion issues**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results where `involvedObject.name` == "" and sort by timestamp. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' -``` - -**5. Check VM deletion timestamp**: - -Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. - -If present, VM is in deletion process. If null, VM is not being deleted. - -**Common Finalizer Patterns**: -- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer -- `foregroundDeletion` - Waits for dependent objects to be deleted -- Custom finalizers from operators - -**Solutions** (Use MCP Tools First): - -1. **Wait for dependent resources** (safest approach): - - Use `resources_list` to check if PVCs, DataVolumes are still deleting - - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) - -2. **Force delete VMI** (if VMI is stuck): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable - requires explicit user permission): - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -3. **Force delete virt-launcher pod** (if pod is stuck): - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - # Find the virt-launcher pod name first - oc get pods -n | grep virt-launcher- - - # Then delete it - oc delete pod virt-launcher--xxx -n --force --grace-period=0 - ``` - -4. **Remove finalizers** (⚠️ dangerous - use only as last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` - 2. Remove items from `.metadata.finalizers` array - 3. Update VM using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch not easily done via MCP): - ```bash - # This operation is complex for MCP - may need CLI - oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Check for protection labels** (vm-delete skill specific): - - Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. - - If value is "true", the vm-delete skill refuses deletion (this is expected behavior). - -**Verification**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error if deletion successful. - -**CLI Fallback**: -```bash -oc get vm -n -# Should return: Error from server (NotFound) -``` - ---- - - ---- - -### VM Won't Start (Non-Scheduling Issues) - -**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state - -**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. - -**Possible Causes**: -- Guest OS kernel panic or boot failure -- Cloud-init configuration errors -- Missing or corrupted disk image -- Insufficient memory for guest OS to boot -- QEMU/KVM virtualization errors -- VirtualMachineInstance (VMI) creation failures -- virt-launcher pod crashes - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VMI (VirtualMachineInstance) status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and overall VMI state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**2. Check VMI conditions for errors**: - -Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. View virt-launcher pod logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -First, get pod name from step 3, then: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -Look for QEMU errors, memory allocation failures, device errors. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**5. Check virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. - -Alternatively, use `pods_get` to get full pod details: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**6. Access VM console to see guest OS boot messages**: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -Look for kernel panic, initramfs errors, filesystem errors. - -**7. Check VM events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep -A 20 "Events:" -``` - -**Common Error Patterns**: - -1. **Guest OS Boot Failure**: - - Console shows kernel panic - - Guest hangs at GRUB or boot loader - - Cloud-init errors during first boot - -2. **Insufficient Memory**: - - Guest OS kills processes due to OOM - - VMI logs show memory allocation errors - -3. **QEMU Crashes**: - - virt-launcher logs show QEMU segfaults - - VMI repeatedly restarts - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for kernel panic, initramfs errors, filesystem errors. - -2. **Check virt-launcher pod logs for QEMU errors**: - - Use `pods_log` from diagnostic step 4 to view logs. - - Look for: - - "qemu-system-x86_64: ..." errors - - Memory allocation failures - - Device errors - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx - ``` - -3. **Increase memory if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check VMI memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -4. **Verify disk image integrity**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check PVC is bound: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - If using container disk, verify image exists and is accessible (check VMI spec). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get pvc -n - ``` - -5. **Check cloud-init configuration** (if applicable): - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. - - Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq - ``` - -6. **Restart VMI** (soft reset): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete VMI (VM controller will recreate it): - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for new VMI to start (use `resources_get` to check status). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n -w - ``` - -7. **Check virtualization extensions** (KVM): - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - - # In debug shell: - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should eventually return `Running`). - -Check VMI is running: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should show `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should eventually return: Running - -oc get vmi -n -# Should show: Running -``` - ---- - -### VM Won't Stop - -**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state - -**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. - -**Possible Causes**: -- Guest OS not responding to ACPI shutdown signal -- virt-launcher pod stuck and not terminating -- VirtualMachineInstance (VMI) deletion blocked by finalizers -- Guest shutdown scripts hanging -- Filesystem sync issues in guest OS - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (might show `Stopping`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**2. Check VMI status and deletion timestamp**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check VMI events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vmi -n | grep -A 10 "Events:" -``` - -**5. Check VMI finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.metadata.finalizers}' -``` - -**6. Check if guest is responsive** (if VMI still exists): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Patterns**: -- VMI shows `deletionTimestamp` but never actually deletes -- virt-launcher pod in `Terminating` state -- VM runStrategy is `Halted` but printableStatus shows `Stopping` - -**Solutions** (Use MCP Tools First): - -1. **Wait for graceful shutdown** (default: 30 seconds): - - Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. - -2. **Force stop by deleting VMI**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - This is the standard way to force-stop a VM. - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n - # Should return: Error from server (NotFound) - ``` - -3. **Force delete VMI with grace period** (if VMI won't delete): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -4. **Force delete virt-launcher pod**: - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find the pod using `pods_list_in_namespace` from diagnostic step 3. - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - ⚠️ **Note**: For force deletion with grace period, use CLI fallback. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') - oc delete pod $POD_NAME -n --force --grace-period=0 - ``` - -5. **Remove VMI finalizers** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VMI using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update VMI using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Patch VM runStrategy directly** (ensure consistency): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` (diagnostic step 1) - 2. Set `.spec.runStrategy` to `"Halted"` - 3. Update VM using `resources_create_or_update` with modified JSON - - **CLI Fallback** (merge patch easier via CLI): - Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Stopped` or `Halted`). - -Verify VMI is gone: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Verify virt-launcher pod is gone: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for pods matching `virt-launcher-`. Should return no results. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Stopped or Halted - -oc get vmi -n -# Should return: Error from server (NotFound) - -oc get pods -n | grep virt-launcher- -# Should return: No resources found -``` - -**Prevention**: -- Ensure guest OS has ACPI support enabled -- Use proper shutdown commands in guest OS -- Avoid forceful stops unless necessary (can corrupt guest filesystem) - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/network-errors.md deleted file mode 100644 index 82fa313c..00000000 --- a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/network-errors.md +++ /dev/null @@ -1,429 +0,0 @@ ---- -title: VM Network Errors -category: kubevirt -sources: - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] -semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] -use_cases: [vm-creation, network-troubleshooting] -related_docs: [INDEX.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Network Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. - -**When to use this document**: -- VM created successfully but secondary networks not attached -- NetworkAttachmentDefinition not found errors -- Multus CNI failures -- SR-IOV device attachment issues - -**Skills that use this**: vm-create - ---- - -### Network Attachment Failures - -**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working - -**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). - -**Possible Causes**: -- NetworkAttachmentDefinition doesn't exist in the namespace -- Multus CNI not installed or not configured on cluster -- Namespace mismatch (NAD in different namespace than VM) -- Interface name conflicts in VM spec -- Bridge/network configuration errors in NAD -- SR-IOV device not available (if using SR-IOV) - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List NetworkAttachmentDefinitions in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "" -} -``` - -Review `.items[].metadata.name` for available NADs. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -n -``` - -**2. List NetworkAttachmentDefinitions in all namespaces**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition" -} -``` - -Omit `namespace` parameter to list across all namespaces. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -A -``` - -**3. Check specific NetworkAttachmentDefinition**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "", - "name": "" -} -``` - -Review `.spec.config` for CNI configuration. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definition -n -o yaml -``` - -**4. Check VM network configuration**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.networks` to see network references. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq -``` - -**5. Check VM domain interfaces**: - -Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq -``` - -**6. Check virt-launcher pod network annotations** (shows actual attachments): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**7. Check Multus is installed**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "openshift-multus" -} -``` - -Should show Multus CNI pods running. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n openshift-multus -``` - -**8. Check for errors in virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n -``` - -**Common Error Messages**: -- `"network-attachment-definition not found"` - NAD doesn't exist in namespace -- `"multus CNI not configured"` - Multus not installed or misconfigured -- `"interface name conflict"` - Duplicate interface names in VM spec -- `"failed to add network"` - CNI plugin error (check NAD config) -- `"no available devices"` - SR-IOV device not available (if using SR-IOV) - -**Solutions** (Use MCP Tools First): - -1. **Verify NetworkAttachmentDefinition exists in correct namespace**: - - Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. - - If NAD is in different namespace, copy it to VM namespace: - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get NAD from source namespace using `resources_get` - 2. Modify `.metadata.namespace` to target namespace - 3. Create NAD in target namespace using `resources_create_or_update` - - **CLI Fallback** (stream processing easier via CLI): - Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" - ```bash - oc get network-attachment-definition -n -o yaml | \ - sed "s/namespace: /namespace: /" | \ - oc apply -f - - ``` - -2. **Create missing NetworkAttachmentDefinition**: - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - Example: Linux bridge network - - **Parameters**: - ```json - { - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "metadata": { - "name": "vlan100", - "namespace": "" - }, - "spec": { - "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" - ```bash - cat < - spec: - config: '{ - "cniVersion": "0.3.1", - "type": "bridge", - "bridge": "br1", - "vlan": 100, - "ipam": { - "type": "host-local", - "subnet": "192.168.100.0/24" - } - }' - EOF - ``` - -3. **Check Multus CNI installation**: - - Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. - - To check cluster network operator: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "config.openshift.io/v1", - "kind": "ClusterOperator" - } - ``` - - Filter for `network` operator. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n openshift-multus - oc get clusteroperators network - ``` - -4. **Fix interface name conflicts** (if VM has duplicate names): - - Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. - - Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. - - **CLI Fallback** (interactive edit easier via CLI): - Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' - oc edit vm -n - ``` - -5. **Validate NAD configuration syntax**: - - Use `resources_get` from diagnostic step 3, extract `.spec.config`. - - Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get network-attachment-definition -n -o jsonpath='{.spec.config}' - ``` - -6. **Check SR-IOV device availability** (if using SR-IOV networks): - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - List SR-IOV network node policies: - - **Parameters**: - ```json - { - "apiVersion": "sriovnetwork.openshift.io/v1", - "kind": "SriovNetworkNodePolicy", - "namespace": "openshift-sriov-network-operator" - } - ``` - - Check SR-IOV device plugin pods: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "openshift-sriov-network-operator" - } - ``` - - Filter for pods with "device-plugin" in name. - - Check available SR-IOV devices on node: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "Node", - "name": "" - } - ``` - - Review `.status.allocatable` for SR-IOV resources. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get sriovnetworknodepolicy -n openshift-sriov-network-operator - oc get pods -n openshift-sriov-network-operator | grep device-plugin - oc describe node | grep -A 10 "Allocatable:" - ``` - -7. **Recreate VM with corrected network configuration** (if needed): - - If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -After remediation, check virt-launcher pod network status: - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -Should show all attached networks with status. Example output: -```json -[ - { - "name": "openshift-sdn", - "interface": "eth0", - "ips": ["10.128.2.10"], - "default": true - }, - { - "name": "vlan100", - "interface": "net1", - "ips": ["192.168.100.5"] - } -] -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**Check from inside VM** (via console): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# In guest OS: -ip addr show -# Should show all network interfaces (eth0, net1, etc.) -``` - -**Common Network Types**: -- **Linux Bridge**: Layer 2 bridge for VLAN networks -- **SR-IOV**: High-performance direct device assignment -- **macvlan**: MAC-based VLAN for container networks -- **OVN-Kubernetes**: OpenShift native overlay network - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/runtime-errors.md deleted file mode 100644 index 804d3cbc..00000000 --- a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/runtime-errors.md +++ /dev/null @@ -1,616 +0,0 @@ ---- -title: VM Runtime Errors -category: kubevirt -sources: - - title: Kubernetes Pod Lifecycle - url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ - date_accessed: 2026-02-17 -tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] -semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] -use_cases: [vm-creation, vm-lifecycle, diagnostics] -related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Runtime Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. - -**When to use this document**: -- VM or virt-launcher pod shows `CrashLoopBackOff` status -- virt-launcher pod repeatedly restarting -- Guest OS kernel panics on boot - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### CrashLoopBackOff - -**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting - -**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. - -**Possible Causes**: -- Guest OS kernel panic on boot -- Insufficient resources (memory/CPU) for guest OS -- Corrupted disk image or filesystem -- QEMU/libvirt crashes due to configuration errors -- Missing or incompatible device drivers in guest -- Resource limits too low for virt-launcher pod -- Virtualization features (KVM) not available on node - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check virt-launcher pod restart count**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# Look at RESTARTS column -``` - -**2. View recent crash logs** (previous container instance): - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "previous": true, - "tail": 100 -} -``` - -Look for QEMU errors, kernel panics, or segfaults. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n virt-launcher--xxx --previous -``` - -**3. Check current virt-launcher logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. - -**CLI Fallback** (if MCP unavailable or all containers needed): -```bash -oc logs -n virt-launcher--xxx --all-containers -``` - -**4. Check VMI conditions for crash details**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Extract `.status.conditions` for crash details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**5. Check pod events for crash reasons**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" -``` - -**6. Check pod resource limits**: - -**MCP Tool**: `pods_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -Extract `.spec.containers[0].resources` for resource limits. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq -``` - -**7. Check node kubelet logs for OOM kills**: - -⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -oc adm node-logs -u kubelet | grep -i oom -``` - -**8. Access guest console** (if VM briefly starts): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Crash Patterns**: - -1. **Guest Kernel Panic**: - - Console logs show kernel panic messages - - Guest crashes immediately after boot - - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" - -2. **OOM (Out of Memory)**: - - Pod killed with reason: `OOMKilled` - - Guest runs out of memory during boot or operation - - virt-launcher logs show memory allocation failures - -3. **QEMU Crash**: - - virt-launcher logs show QEMU segmentation fault - - Symptoms: "qemu-system-x86_64: terminated by signal" - - Configuration incompatibility or QEMU bug - -4. **Disk Image Corruption**: - - Guest cannot boot from disk - - Filesystem errors in guest console - - DataVolume import failed - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for kernel panic or boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for: - - Kernel panic messages - - Initramfs errors - - Filesystem mounting failures - - Missing device errors - -2. **Review virt-launcher crash logs**: - - Use `pods_log` with `previous: true` from diagnostic step 2. - - Look for: - - QEMU command line errors - - Device initialization failures - - Memory allocation errors - - Signal termination (SIGSEGV, SIGABRT) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx --previous - ``` - -3. **Check for OOM (Out of Memory) kills**: - - **MCP Tool**: `pods_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - Extract `.status.containerStatuses[0].lastState.terminated.reason`. - - If returns `"OOMKilled"`: - - Option 1: Increase virt-launcher memory limits - - Option 2: Decrease guest memory allocation - - Option 3: Use smaller instance type - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' - ``` - -4. **Increase resources if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check current memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -5. **Verify disk image integrity**: - - **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters for list**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" - } - ``` - - **Parameters for specific DV**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check `.status.phase` (should be `Succeeded`). - - If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get datavolume -n -o jsonpath='{.status.phase}' - ``` - -6. **Check virtualization (KVM) availability**: - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -7. **Simplify VM configuration** (eliminate variables): - - Try creating minimal VM using vm-create skill with: - - Small instance type - - No secondary networks - - Simple container disk (e.g., Fedora) - - No cloud-init - - If minimal VM works, add features back one by one. - -8. **Recreate VM with different workload** (test disk image): - - If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) - -After remediation, check pod restart count stops increasing: - -**Parameters for pods**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). - -Check VM reaches Running state: - -**Parameters for VM**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# RESTARTS should stabilize - -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Running -``` - -Verify guest is responsive: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# Should show login prompt or OS console -``` - -**Advanced Debugging**: - -**MCP Tool**: `pods_exec` (from openshift-virtualization) - -Check libvirt domain XML: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["virsh", "dumpxml", "1"] -} -``` - -Check QEMU process: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["ps", "aux"] -} -``` - -Filter output for "qemu" process. - -**CLI Fallback** (if MCP unavailable): -```bash -oc exec -n virt-launcher--xxx -- virsh dumpxml 1 -oc exec -n virt-launcher--xxx -- ps aux | grep qemu -``` - -**Prevention**: -- Start with minimal VM configuration and add complexity gradually -- Use recommended instance types for your workload -- Test disk images before deploying to production -- Ensure nodes have adequate resources and KVM support -- Monitor resource usage and set appropriate limits - ---- - -## Workaround Patterns for MCP Tool Limitations - -### General Pattern: Diagnose → Propose → Confirm → Execute - -When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): - -**1. Diagnose** the root cause using Kubernetes commands -```bash -oc describe vm -n -oc get events -n --field-selector involvedObject.name= -``` - -**2. Propose** a manual workaround to the user with clear explanation -- Explain why the issue occurred -- Show what will be changed -- List alternative options - -**3. Confirm** with user before executing (Human-in-the-Loop) -- Wait for explicit "yes" or "apply workaround" -- Never auto-execute modifications - -**4. Execute** the workaround using oc/kubectl -```bash -oc patch vm -n --type=merge -p '...' -``` - -**5. Verify** the fix was successful -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**6. Document** the limitation and suggest filing enhancement request -- Note this is temporary until MCP tool is enhanced -- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues - -### Example: Adding Tolerations Workaround - -**Diagnostic Output**: -```markdown -## ⚠️ VM Scheduling Issue Detected - -**Root Cause**: Node taints prevent VM scheduling - -**Details**: -- Found 3 nodes with taint: `virtualization=true:NoSchedule` -- VM spec does not include matching tolerations -- This prevents VM scheduling on virtualization-dedicated nodes -``` - -**Proposed Workaround**: -```bash -oc patch vm web-server -n vms --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' -``` - -**User Confirmation Required**: -``` -How would you like to proceed? -- "apply workaround" - I'll patch the VM with tolerations -- "manual" - I'll provide instructions for you to apply manually -- "cancel" - Delete the VM and abort creation -``` - -**After Execution**: -```markdown -## ✓ Workaround Applied Successfully - -**Action**: Added tolerations for taint `virtualization=true:NoSchedule` -**New Status**: Stopped (VM can now be scheduled) - -**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. -Future VMs in this cluster will need the same fix until the tool is enhanced. -``` - ---- - -## VM Status Reference - -### Status Values - -| Status | Meaning | Action Required | -|--------|---------|-----------------| -| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | -| `Running` | VM is running | Normal | -| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | -| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | -| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | -| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | -| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | -| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | -| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | -| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | - -### Checking VM Status - -```bash -# Get printable status -oc get vm -n -o jsonpath='{.status.printableStatus}' - -# Get detailed status and conditions -oc get vm -n -o jsonpath='{.status}' | jq - -# Watch status changes in real-time -oc get vm -n -w -``` - ---- - -## Best Practices for Agents - -When implementing diagnostic workflows: - -1. **Always verify VM status** after creation (wait 5-10 seconds first) -2. **Consult this document** when encountering error status values -3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) -4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) -5. **Respect human-in-the-loop** for all VM modifications -6. **Document temporary workarounds** and their limitations clearly -7. **Suggest filing issues** for missing MCP tool features - -### Document Consultation Pattern - -```markdown -**Document Consultation** (REQUIRED): -1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes -2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." -``` - ---- - -## Known MCP Tool Limitations - -### vm_create tool - -**Currently Supported**: -- ✓ Namespace, name (required) -- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) -- ✓ Size hints (small, medium, large) -- ✓ Storage size -- ✓ Autostart flag -- ✓ Networks (Multus NetworkAttachmentDefinitions) -- ✓ Performance family (u1, o1, c1, m1) -- ✓ Instance type, preference - -**Not Currently Supported** (requires workarounds): -- ✗ Tolerations (for node taints) -- ✗ Node selectors -- ✗ Affinity/anti-affinity rules -- ✗ Resource requests/limits (beyond instance type) -- ✗ Custom labels/annotations -- ✗ SSH keys injection -- ✗ Cloud-init user data - -**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. - -**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues - ---- - -## Additional Resources - -- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) -- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) -- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) -- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) - ---- - -[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/scheduling-errors.md deleted file mode 100644 index e47c8c4f..00000000 --- a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/scheduling-errors.md +++ /dev/null @@ -1,417 +0,0 @@ ---- -title: VM Scheduling Errors -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 -tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] -semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] -use_cases: [vm-creation, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Scheduling Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**When to use this document**: -- VM shows status `ErrorUnschedulable` after creation or start attempt -- VM events mention scheduling failures, taints, resources, or node selectors - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -## ErrorUnschedulable - -**Symptom**: VM shows status `ErrorUnschedulable` after creation - -**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**Possible Causes**: - -### 1. Node Taints (Most Common) - -Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM events for scheduling failures**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `involvedObject.name` == "" and look for messages like: -- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= -``` - -**2. Check node taints in the cluster**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' -``` - -**Common Taint Patterns**: -- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule -- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes -- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads - -**Solution - Add Tolerations to VM**: - -The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. - -**Workaround (post-creation using MCP Tools)**: - -**Step 1**: Get current VM spec - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Step 2**: Modify the returned JSON to add tolerations - -Add to `.spec.template.spec.tolerations`: -```json -{ - "tolerations": [ - { - "key": "virtualization", - "operator": "Equal", - "value": "true", - "effect": "NoSchedule" - } - ] -} -``` - -**Step 3**: Update VM with modified spec - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters**: -```json -{ - "resource": "" -} -``` - -Pass the complete modified VM resource as YAML or JSON string. - -**Step 4**: Verify tolerations were added - -Use `resources_get` again and check `.spec.template.spec.tolerations` in response. - -**Step 5**: Check if VM status improved - -Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. - -**CLI Fallback** (if MCP patch is too complex): -```bash -# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' - -# Verify tolerations -oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq - -# Check status -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**Example - Multiple Tolerations**: -```bash -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" - - key: "dedicated" - operator: "Equal" - value: "virt-workloads" - effect: "NoSchedule" -' -``` - -**Toleration Operators**: -- `Equal` - Key and value must match exactly -- `Exists` - Only key must exist (ignores value) - -**Toleration Effects**: -- `NoSchedule` - Don't schedule new pods (existing pods continue) -- `PreferNoSchedule` - Avoid scheduling if possible -- `NoExecute` - Don't schedule AND evict existing pods - -**Alternative Solutions**: -1. **Remove node taints** (if you have cluster-admin access): - ```bash - oc adm taint nodes virtualization=true:NoSchedule- - ``` - -2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits - -3. **File enhancement request** - Request tolerations support in openshift-mcp-server: - https://github.com/openshift/openshift-mcp-server/issues - ---- - -### 2. Insufficient Resources - -Not enough CPU, memory, or storage available on any node. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM resource requests**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' -``` - -**2. Check node resource availability**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. - -Alternatively, use `nodes_top` MCP tool for current resource usage. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe nodes | grep -A 5 "Allocated resources" -``` - -**3. Look for VM events mentioning "Insufficient"**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep "Insufficient" -``` - -**Example Event**: -``` -0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. -``` - -**Solutions** (Use MCP Tools First): - -1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) -2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill -3. **Delete unused VMs** - Use vm-delete skill to free up resources -4. **Check resource quotas**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters for quota**: - ```json - { - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" - } - ``` - - **Parameters for limit range**: - ```json - { - "apiVersion": "v1", - "kind": "LimitRange", - "namespace": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - oc describe limitrange -n - ``` - ---- - -### 3. Node Selector Mismatch - -VM requires specific node labels that don't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM node selector requirements**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.nodeSelector` to see required node labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' -``` - -**2. List available node labels**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.metadata.labels` for available labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes --show-labels -``` - -**3. Check if any nodes match the selector**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node", - "labelSelector": "=" -} -``` - -Should return at least one node with matching labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -l = -``` - -**Solutions** (Use MCP Tools First): - -**Option 1: Remove node selector from VM** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get current VM using `resources_get` (diagnostic step 1) -2. Remove `.spec.template.spec.nodeSelector` field -3. Update VM using `resources_create_or_update` with modified JSON - -**CLI Fallback** (JSON patch easier via CLI): -Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" -```bash -oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' -``` - -**Option 2: Add label to nodes** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get node using `resources_get` -2. Add label to `.metadata.labels` -3. Update node using `resources_create_or_update` - -⚠️ **Note**: Node labeling typically requires cluster admin privileges. - -**CLI Fallback** (simpler via CLI): -Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" -```bash -oc label node = -``` - ---- - -[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/storage-errors.md deleted file mode 100644 index 16ba584d..00000000 --- a/evaluation/with_skills/rh-virt__vm-inventory/environment/docs/troubleshooting/storage-errors.md +++ /dev/null @@ -1,1011 +0,0 @@ ---- -title: VM Storage Errors -category: kubevirt -sources: - - title: KubeVirt CDI - DataVolumes - url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ - date_accessed: 2026-02-17 - - title: Kubernetes Persistent Volumes - url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ - date_accessed: 2026-02-17 -tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] -semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] -use_cases: [vm-creation, vm-deletion, vm-cloning] -related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] -last_updated: 2026-02-17 ---- - -# VM Storage Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. - -**When to use this document**: -- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` -- Storage deletion fails after VM deletion -- DataVolume cloning operations fail -- PVC provisioning issues - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### ErrorDataVolumeNotReady - -**Symptom**: VM shows status `ErrorDataVolumeNotReady` - -**Description**: The DataVolume (persistent storage) backing the VM is not ready. - -**Possible Causes**: - -#### 1. DataVolume Still Provisioning - -Storage provisioning takes time, especially for large disks or when importing images. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Get detailed DataVolume information**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` and `.status.conditions` for provisioning details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o json -``` - -**3. Check PVC (PersistentVolumeClaim) bound status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Check `.status.phase` for each PVC (should be `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. - -#### 2. Storage Class Not Found - -The requested storage class doesn't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List available storage classes**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" -} -``` - -Review the list of available storage classes (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -``` - -**2. Check DataVolume's requested storage class**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.spec.pvc.storageClassName` in the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' -``` - -**Solution**: -1. Use a valid storage class from the cluster -2. Recreate VM with correct storage class parameter - -#### 3. Insufficient Storage Quota - -Namespace has insufficient storage quota to provision the PVC. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check resource quotas**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**2. Check storage usage**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase -``` - -**Solution**: -1. Request quota increase from cluster admin -2. Delete unused PVCs to free quota -3. Reduce VM storage size - ---- - -### ErrorPvcNotFound - -**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List PVCs in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review the list of available PVCs (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check VM's PVC references**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' -``` - -**Solution**: -- Wait for DataVolume to create the PVC -- Manually create missing PVC -- Fix VM spec to reference correct PVC name - ---- - - ---- - -### Storage Deletion Failures - -**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace - -**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. - -**Possible Causes**: -- PVC still bound to active PersistentVolume with `Retain` policy -- DataVolume still being referenced by another resource -- CDI (Containerized Data Importer) controller issues -- Storage class retention policy preventing deletion -- Finalizers on PVC/DataVolume blocking cleanup -- PVC still mounted by a pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check PVC status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check specific PVC phase**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Released` or `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o jsonpath='{.status.phase}' -``` - -**3. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**4. Check what's using the PVC**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' -``` - -**5. Check PVC finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. - -**6. Check DataVolume finalizers**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.metadata.finalizers}' -``` - -**7. Check PV reclaim policy**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume" -} -``` - -Filter results for PV where `.spec.claimRef.name` matches ``. - -To get specific PV policy: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume", - "name": "" -} -``` - -Check `.spec.persistentVolumeReclaimPolicy`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pv | grep -oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' -``` - -**Common Finalizer Patterns**: -- `kubernetes.io/pvc-protection` - Protects PVC while in use -- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer - -**Solutions** (Use MCP Tools First): - -1. **Delete DataVolume first, then PVC**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete DataVolume first (often blocks PVC deletion): - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Wait a few seconds, then delete PVC: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete datavolume -n - oc delete pvc -n - ``` - -2. **Check for pods still using PVC**: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "" - } - ``` - - Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. - - Then delete the pods using `pods_delete`: - - **Parameters**: - ```json - { - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' - oc delete pod -n - ``` - -3. **Force delete PVC** (if safe to do so): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete pvc -n --grace-period=0 --force - ``` - -4. **Remove finalizers from PVC** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PVC using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update PVC using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Remove finalizers from DataVolume** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current DataVolume using `resources_get` (diagnostic step 6) - 2. Remove items from `.metadata.finalizers` array - 3. Update DataVolume using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Change PV reclaim policy** (if PV has Retain policy): - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PV policy using `resources_get` (diagnostic step 7) - 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` - 3. Update PV using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" - ```bash - oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' - oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' - ``` - -**Storage Quota Check** (Use MCP Tools First): - -After deletion, verify storage quota is freed: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters for quota check**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.used` to verify storage quota is freed. - -**Parameters for PVC verification**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -oc get pvc -n -``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Confirm PVC is deleted: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Confirm DataVolume is deleted: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -# Should return: Error from server (NotFound) - -oc get datavolume -n -# Should return: Error from server (NotFound) -``` - ---- - -### DataVolume Cloning Failures - -**Symptom**: VM clone created successfully but DataVolume clone operation fails - -**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. - -**Possible Causes**: -- CSI driver doesn't support volume cloning -- Source PVC storage class incompatible with cloning -- Cross-namespace cloning not permitted by storage backend -- Insufficient storage quota in target namespace -- Source PVC not in `Bound` state -- Storage class doesn't have volume cloning enabled -- CDI (Containerized Data Importer) controller issues - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume clone status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and `.metadata.name`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Check DataVolume events for errors**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe datavolume -n -``` - -**3. Check DataVolume phase**: - -Use `resources_get` from step 1, extract `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -``` - -**4. Check if storage class supports cloning**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass", - "name": "" -} -``` - -Review the full YAML output for cloning-related configurations. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -o yaml | grep -A 5 -i clone -``` - -**5. Check CSI driver capabilities**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "CSIDriver" -} -``` - -Review `.items[].metadata.name` for available CSI drivers. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get csidriver -``` - -**6. Check source PVC status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Bound` for cloning to work). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**7. Check target namespace storage quota**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**8. Check CDI controller logs**: - -**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) - -First, list pods in openshift-cnv namespace: - -**Parameters for pods_list_in_namespace**: -```json -{ - "namespace": "openshift-cnv", - "labelSelector": "app.kubernetes.io/component=cdi-deployment" -} -``` - -Then get logs using `pods_log`: - -**Parameters**: -```json -{ - "namespace": "openshift-cnv", - "name": "", - "tail": 100 -} -``` - -**CLI Fallback** (if MCP unavailable or easier via CLI): -```bash -oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') -``` - -**Common Error Messages**: -- `"volume cloning is not supported"` - CSI driver lacks clone capability -- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage -- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace -- `"insufficient quota"` - Target namespace lacks storage quota -- `"source PVC not bound"` - Source PVC must be in Bound state for cloning -- `"StorageClass does not support cloning"` - Storage class configuration issue - -**Solutions** (Use MCP Tools First): - -1. **Check storage class clone support**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" - } - ``` - - For each storage class in `.items[]`, check: - - `.metadata.name` (storage class name) - - `.provisioner` (CSI driver) - - Storage classes using CSI drivers typically support cloning. Look for provisioners like: - - `csi.ovirt.org` (oVirt CSI) - - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) - - `ebs.csi.aws.com` (AWS EBS CSI) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner - ``` - -2. **Verify source PVC is bound**: - - Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pvc -n -o jsonpath='{.status.phase}' - ``` - -3. **Check target namespace quota**: - - Use `resources_list` from diagnostic step 7 to check quota. - - If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - ``` - -4. **Use snapshot-based cloning** (alternative method): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Step 1**: Create VolumeSnapshot of source PVC - - **Parameters**: - ```json - { - "apiVersion": "snapshot.storage.k8s.io/v1", - "kind": "VolumeSnapshot", - "metadata": { - "name": "-snapshot", - "namespace": "" - }, - "spec": { - "source": { - "persistentVolumeClaimName": "" - } - } - } - ``` - - **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) - - **Step 3**: Create new DataVolume from snapshot - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "metadata": { - "name": "-rootdisk", - "namespace": "" - }, - "spec": { - "source": { - "snapshot": { - "name": "-snapshot", - "namespace": "" - } - }, - "storage": { - "resources": { - "requests": { - "storage": "50Gi" - } - }, - "storageClassName": "" - } - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" - ```bash - cat <-snapshot - namespace: - spec: - source: - persistentVolumeClaimName: - EOF - - oc get volumesnapshot -snapshot -n - - cat <-rootdisk - namespace: - spec: - source: - snapshot: - name: -snapshot - namespace: - storage: - resources: - requests: - storage: 50Gi - storageClassName: - EOF - ``` - -5. **Use "new empty storage" option** (vm-clone skill): - - If cloning isn't supported, create VM with empty storage - - Manually copy data if needed - -6. **Cross-namespace cloning workaround**: - - Some storage backends require snapshot for cross-namespace cloning - - Create snapshot in source namespace, restore in target namespace (see solution 4 above) - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Check DataVolume reached Succeeded phase: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should return `Succeeded`). - -Check PVC was created and bound: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "-rootdisk" -} -``` - -Check `.status.phase` (should return `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -# Should return: Succeeded - -oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' -# Should return: Bound -``` - -**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-inventory/environment/skills/vm-inventory/SKILL.md b/evaluation/with_skills/rh-virt__vm-inventory/environment/skills/vm-inventory/SKILL.md deleted file mode 100644 index 32873b81..00000000 --- a/evaluation/with_skills/rh-virt__vm-inventory/environment/skills/vm-inventory/SKILL.md +++ /dev/null @@ -1,390 +0,0 @@ ---- -name: vm-inventory -description: | - List and view virtual machines across namespaces with status, resource usage, and health information. - - Use when: - - "List all VMs" - - "Show VMs in namespace [name]" - - "What VMs are running?" - - "Get details of VM [name]" - - This skill provides comprehensive VM inventory and status reporting. - - NOT for creating or modifying VMs (use vm-create or vm-lifecycle-manager instead). - -model: inherit -color: cyan ---- - -# /vm-inventory Skill - -List and inspect virtual machines in OpenShift Virtualization clusters. This skill provides read-only access to VM information without making any modifications. - -## Prerequisites - -**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools**: -- `resources_list` (from openshift-virtualization) - List Kubernetes resources including VirtualMachines -- `resources_get` (from openshift-virtualization) - Get specific Kubernetes resource details - -**Fallback CLI Commands** (if MCP tools unavailable): -- `oc get virtualmachines` / `oc get vm` - List VirtualMachines -- `oc get vm -n -o yaml` - Get VM details - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster (>= 4.19) -- OpenShift Virtualization operator installed -- ServiceAccount with RBAC permissions to list and get VirtualMachine resources - -### Prerequisite Verification - -**Before executing:** - -1. Check `openshift-virtualization` exists in `.mcp.json` → If missing, report setup -2. Verify `KUBECONFIG` is set (presence only, never expose value) → If missing, report -3. (Optional) Test basic connectivity to cluster → If fails, report connection error - -**Human Notification Protocol:** `❌ Cannot execute vm-inventory: MCP server not available. Setup: Add to .mcp.json, set KUBECONFIG, restart Claude Code. Docs: https://github.com/openshift/openshift-mcp-server` - -⚠️ **SECURITY**: Never display KUBECONFIG path or credential values. - -**Note on Fallback**: If MCP server unavailable but KUBECONFIG set, offer CLI fallback with user confirmation. - -## When to Use This Skill - -**Trigger when:** -- User explicitly invokes `/vm-inventory` command -- User wants to see all VMs or VMs in a specific namespace -- User asks about VM status or health -- User needs to find a VM by name -- User wants details about a specific VM configuration - -**User phrases:** -- "List all VMs" -- "Show VMs in production namespace" -- "What VMs are running?" -- "Get details of VM web-server" -- "Show me the status of database-vm" -- "/vm-inventory" (explicit command) - -**Do NOT use when:** -- User wants to create a VM → Use `/vm-create` skill instead -- User wants to start/stop VMs → Use `/vm-lifecycle-manager` skill instead -- User wants to modify VM configuration → Different operation (not inventory) - -## Workflow - -**CRITICAL EXECUTION PATTERN**: -1. **ALWAYS attempt MCP server tools FIRST** - Try `resources_list` or `resources_get` -2. **If MCP tools fail** - Propose CLI commands (`oc get vm`) with user confirmation -3. **Never skip MCP attempt** - Always try them first - -**Tool Execution Priority**: MCP tools (primary) → CLI commands (fallback with confirmation) - -### Workflow A: List All VMs (Across All Namespaces) - -**Step 1: Query VirtualMachine Resources Using MCP Tool** - -**MCP Tool**: `resources_list` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", allNamespaces=true) - -**Errors:** Tool not found/connection error → Report, offer CLI fallback: `oc get virtualmachines -A -o json` - -**Step 2: Get Resource Details for Running VMs** - -**CRITICAL**: To display complete VM information, query VirtualMachineInstance (VMI) resources: - -**MCP Tool**: `resources_list` (apiVersion="kubevirt.io/v1", kind="VirtualMachineInstance") - -**For each VMI, extract**: -- `.spec.domain.cpu.sockets` and `.spec.domain.memory.guest` - Resources column ("X vCPU, YGi") -- `.status.volumeStatus[].persistentVolumeClaimInfo.capacity.storage` - Storage column (sum all PVC volumes, exclude container disks/cloudinit) -- `.status.guestOSInfo.prettyName` or `.status.guestOSInfo.name` + version - Guest OS column -- `.status.interfaces[0].ipAddress` - IP column (primary interface) -- `.status.nodeName` - Node column -- `.status.conditions[]` - Conditions column (Ready, AgentConnected, LiveMigratable) - -**Stopped VMs**: Use VirtualMachine spec for Resources only; Storage/Guest OS/IP/Conditions show "-" - -**Step 3: Format and Display Results** - -**ALWAYS display in table format** ordered by namespace and status: - -```markdown -## 📋 Virtual Machines (All Namespaces) - -| Namespace | VM Name | Status | Age | Resources | Storage | Guest OS | Node | IP | Conditions | -|-----------|---------|--------|-----|-----------|---------|----------|------|----|------------| -| development | debug-vm | ⚠ Pending | 2d | 2 vCPU, 4Gi | 30Gi | - | - | - | ⚠ Not Ready | -| development | test-vm | ✓ Running | 5d | 2 vCPU, 4Gi | 30Gi | Ubuntu 24.04 | worker-03 | 10.131.0.20 | ✓ Ready, ✓ Live Migration | -| production | database-vm | ✗ Stopped | 30d | 8 vCPU, 16Gi | - | - | - | - | - | -| production | web-server-01 | ✓ Running | 15d | 4 vCPU, 8Gi | 100Gi | RHEL 9.7 | worker-01 | 10.131.0.15 | ✓ Ready, ✓ Agent, ✗ Live Migration | -| production | web-server-02 | ✓ Running | 15d | 4 vCPU, 8Gi | 100Gi | RHEL 9.7 | worker-02 | 10.131.0.16 | ✓ Ready, ✓ Agent, ✗ Live Migration | - -**Summary:** -- **Total VMs**: 5 -- **Running**: 3 -- **Stopped**: 1 -- **Pending**: 1 -``` - -**Table Ordering Rules:** -1. **Primary sort**: Namespace (alphabetical) -2. **Secondary sort**: Status (Running → Pending → Stopped → Failed/Error) -3. **Tertiary sort**: VM Name (alphabetical within same namespace and status) - -**Status Indicators:** -- ✓ Running/Ready -- ✗ Stopped/Halted -- ⚠ Pending/Starting/Terminating -- ❌ Failed/Error - -**Resources Column Format**: MUST show "X vCPU, YGi" (query VMI `.spec.domain.cpu.sockets` and `.spec.domain.memory.guest`), NOT instance type names (e.g., NOT "u1.medium") - -### Workflow B: List VMs in Specific Namespace - -**Step 1: Gather Namespace** - -Ask user for namespace if not provided. - -**Step 2: Query VMs in Namespace Using MCP Tool** - -**MCP Tool**: `resources_list` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", namespace=``) - -**Errors:** Tool fails → Report, offer CLI fallback: `oc get virtualmachines -n -o json` - -**Step 3: Get Resource Details and Display** - -Follow same format rules as Workflow A Step 2-3. Use namespace-specific header: - -```markdown -## 📋 Virtual Machines in '' - -| Name | Status | vCPU | Memory | Age | Node | -|------|--------|------|--------|-----|------| -| web-server-01 | Running | 4 | 8Gi | 15d | worker-01 | -| web-server-02 | Running | 4 | 8Gi | 15d | worker-02 | -| database-vm | Stopped | 8 | 16Gi | 30d | - | - -**Summary**: 3 VMs (2 running, 1 stopped) -``` - -### Workflow C: Get Details of Specific VM - -**Step 1: Gather VM Information** - -Required: VM name, Namespace (ask if not provided) - -**Step 2: Retrieve VM Resource Details Using MCP Tool** - -**MCP Tool**: `resources_get` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", namespace=``, name=``) - -**Errors:** Tool fails → Report, offer CLI fallback: `oc get vm -n -o yaml` - -**Step 3: Interpret Status and Conditions (Optional)** - -**OPTIONAL**: If VM has error status (ErrorUnschedulable, ErrorDataVolumeNotReady, CrashLoopBackOff), consult [troubleshooting/INDEX.md](../../docs/troubleshooting/INDEX.md) using Read tool. Output: "Consulted INDEX.md to interpret status." - -**When to consult**: VM status is Error/Warning or stuck state (CrashLoopBackOff, Terminating) -**When NOT to consult**: VM status is normal (Running, Stopped, Provisioning) - -**Step 4: Display Detailed Information** - -```markdown -## 🖥️ Virtual Machine Details - -### Basic Information -- **Name**: `web-server-01` -- **Namespace**: `production` -- **Status**: Running -- **Created**: 15 days ago - -### Configuration -- **Instance Type**: u1.medium -- **Workload**: Fedora -- **Run Strategy**: Always (auto-restart on crash) - -### Resources -- **vCPU**: 4 cores -- **Memory**: 8Gi -- **Storage**: 50Gi -- **Storage Class**: ocs-storagecluster-ceph-rbd - -### Network -- **Primary**: default (pod network) -- **Secondary**: vlan100 (multus - 192.168.100.5) - -### Volumes -- **rootdisk**: 50Gi (DataVolume/PVC) - -### Current State -- **Phase**: Running -- **Ready**: True -- **Node**: worker-01 -- **Pod IP**: 10.129.2.45 -- **Guest OS Uptime**: 12 days - -### Conditions -- ✓ Ready -- ✓ LiveMigratable -- ✓ AgentConnected - -### Labels -- app: web -- env: production -- tier: frontend -``` - -### Workflow D: Filter VMs by Criteria - -**Step 1: Query VMs with Filters Using MCP Tool** - -**MCP Tool**: `resources_list` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", allNamespaces=true, labelSelector=``) - -**Filtering options**: -- By Labels (via labelSelector): `"app=web"`, `"app=web,env=production"`, `"tier in (frontend,backend)"` -- By Status (post-processing): Filter results by `status.printableStatus` field -- By Resource Size (post-processing): Parse instance type or VMI resource specs - -**Errors:** Tool fails → Report, offer CLI fallback: `oc get virtualmachines -A -l -o json` - -**Step 2: Display Filtered Results** - -Display with explanation: `## 📋 VMs with label 'app=web'` + list/table using Workflow A format - -## Common Issues - -### Issue 1: No VMs Found -**Error**: Empty list | **Causes**: No VMs exist, wrong namespace, insufficient RBAC | **Response**: Report no VMs found, suggest create VM (vm-create), list namespaces, check permissions - -### Issue 2: Permission Denied -**Error**: "Forbidden: User cannot list VirtualMachines" | **Solution**: Verify KUBECONFIG has list/get permissions, contact admin - -### Issue 3: Cluster Connection Error -**Error**: "Unable to connect to cluster" | **Solution**: Verify KUBECONFIG valid, check `oc cluster-info`, verify network, check credentials expiry - -## Output Formatting Guidelines - -**Use consistent status indicators:** -- ✓ Running/Healthy/Ready -- ✗ Stopped/Halted -- ⚠ Warning/Pending/Migrating -- ❌ Critical/Failed/Error - -**Include key information always:** -- VM name and namespace -- Current status -- Resource allocation (vCPU, memory) -- Age/creation time -- Node placement (for running VMs) - -**Organize by namespace** when showing multiple VMs for logical grouping and clear separation. - -**Provide actionable next steps:** How to start stopped VMs, get more details, when to use other skills - -## Integration with Other Skills - -**Before creating a VM** (vm-create): Use vm-inventory to check if VM name exists, verify namespace has capacity -**Before lifecycle operations** (vm-lifecycle-manager): Check current VM status, verify VM exists -**For troubleshooting**: Get VM overview with vm-inventory first, then use vm-troubleshooter for deep diagnostics - -## Dependencies - -### Required MCP Servers -- `openshift-virtualization` - OpenShift MCP server (https://github.com/openshift/openshift-mcp-server) - -### Required MCP Tools (PRIMARY - Always try first) -- `resources_list` - List resources (apiVersion, kind, namespace optional, allNamespaces optional, labelSelector optional) -- `resources_get` - Get resource details (apiVersion, kind, namespace, name) - -### CLI Fallback Commands (Use only if MCP tools fail) -- `oc get virtualmachines` / `oc get vm` - List VirtualMachines -- `oc get vm -n ` - Get specific VM -- `oc get vm -A` - List VMs across all namespaces -- `oc get vm -n ` - List VMs in specific namespace -- `oc get vm -l ` - Filter VMs by label selector - -**Important**: Always attempt MCP tools first. Only use CLI commands after MCP tool failure and with user confirmation. - -### Related Skills -- `vm-create` - Create VMs after checking inventory -- `vm-lifecycle-manager` - Manage VMs discovered in inventory -- `vm-troubleshooter` (planned) - Diagnose problematic VMs from inventory - -### Reference Documentation -- [Troubleshooting INDEX](../../docs/troubleshooting/INDEX.md) - VM status interpretation (optionally consulted when displaying VM details with error states) -- [OpenShift Virtualization Documentation](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt/about_virt/about-virt.html) -- [KubeVirt VirtualMachine API](https://kubevirt.io/api-reference/) -- [Accessing VMs](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt/virtual_machines/virt-accessing-vm-consoles.html) -- [VM Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) - -## Critical: Human-in-the-Loop Requirements - -**Not applicable** - This skill performs read-only operations and does not modify any cluster resources. No user confirmation required. - -**Read-only operations:** -- Listing VirtualMachines across namespaces or in specific namespaces -- Retrieving VM details, status, and resource configurations -- Displaying VM health conditions and resource usage -- Filtering VMs by labels or field selectors -- Viewing VM network, storage, and node placement information - -**No modifications performed:** -- ✓ Does not change VM state (start/stop/restart) -- ✓ Does not modify VM configuration -- ✓ Does not delete VMs or resources -- ✓ Does not consume cluster resources - -## Security Considerations - -- Read-only operations - no modifications to VMs -- Respects Kubernetes RBAC permissions -- Only shows VMs in namespaces user has access to -- KUBECONFIG credentials never exposed in output -- No sensitive VM configuration details displayed by default -- All queries audited in Kubernetes API logs - -## Example Usage - -### Example 1: List all VMs (table format) - -``` -User: "List all VMs" -Agent: [MCP: resources_list(apiVersion="kubevirt.io/v1", kind="VirtualMachine", allNamespaces=true)] - [Queries VMI resources for CPU/memory] - [Displays table format from Workflow A Step 3] -``` - -### Example 2: CLI fallback when MCP unavailable - -``` -User: "List all VMs" -Agent: [MCP tool fails] - ⚠️ MCP tool 'resources_list' not available. Use CLI: `oc get virtualmachines -A`? -User: "yes" -Agent: [Executes: oc get virtualmachines -A -o json] - [Displays table format] -``` - -### Example 3: Get specific VM details - -``` -User: "Show me details of web-server-01 in production" -Agent: [MCP: resources_get(kind="VirtualMachine", namespace="production", name="web-server-01")] - [Displays VM Details format from Workflow C Step 4] -``` - -### Example 4: Filter running VMs - -``` -User: "Show me all running VMs" -Agent: [Lists all VMs, filters by status.printableStatus == "Running"] - ## ✓ Running Virtual Machines - ### production: web-server-01 (4 vCPU, 8Gi, worker-01) | web-server-02 (4 vCPU, 8Gi, worker-02) - ### development: test-vm (2 vCPU, 4Gi, worker-03) - **Total**: 3 running VMs -``` diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/.ai-index/semantic-index.json deleted file mode 100644 index 2512c2a3..00000000 --- a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/.ai-index/semantic-index.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "index_version": "1.0.0", - "last_updated": "2026-02-17", - "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", - - "documents": [ - { - "file": "INDEX.md", - "title": "VM Troubleshooting Guide - Index", - "category": "navigation", - "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], - "use_cases": ["navigation", "tool-reference", "general-guidance"], - "token_estimate": 450, - "skills": ["all"] - }, - { - "file": "scheduling-errors.md", - "title": "VM Scheduling Errors", - "category": "scheduling", - "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], - "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], - "errors_covered": [ - "ErrorUnschedulable - Node Taints", - "ErrorUnschedulable - Insufficient Resources", - "ErrorUnschedulable - Node Selector Mismatch" - ], - "token_estimate": 820, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "storage-errors.md", - "title": "VM Storage Errors", - "category": "storage", - "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], - "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], - "errors_covered": [ - "ErrorDataVolumeNotReady - DataVolume Still Provisioning", - "ErrorDataVolumeNotReady - Storage Class Not Found", - "ErrorDataVolumeNotReady - Insufficient Storage Quota", - "ErrorPvcNotFound", - "Storage Deletion Failures", - "DataVolume Cloning Failures" - ], - "token_estimate": 1350, - "skills": ["vm-creator", "vm-delete", "vm-clone"] - }, - { - "file": "lifecycle-errors.md", - "title": "VM Lifecycle Errors", - "category": "lifecycle", - "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], - "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], - "errors_covered": [ - "VM Stuck in Terminating State", - "VM Won't Start (Non-Scheduling Issues)", - "VM Won't Stop" - ], - "token_estimate": 1150, - "skills": ["vm-delete", "vm-lifecycle-manager"] - }, - { - "file": "runtime-errors.md", - "title": "VM Runtime Errors", - "category": "runtime", - "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], - "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], - "errors_covered": [ - "CrashLoopBackOff" - ], - "token_estimate": 850, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "network-errors.md", - "title": "VM Network Errors", - "category": "networking", - "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], - "use_cases": ["vm-creation", "network-troubleshooting"], - "errors_covered": [ - "Network Attachment Failures" - ], - "token_estimate": 580, - "skills": ["vm-creator"] - } - ], - - "skill_to_docs_mapping": { - "vm-creator": { - "primary_docs": ["scheduling-errors.md", "storage-errors.md"], - "secondary_docs": ["runtime-errors.md", "network-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3650, - "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" - }, - "vm-delete": { - "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 2950, - "description": "VM deletion failures - stuck terminating, storage cleanup" - }, - "vm-clone": { - "primary_docs": ["storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 1800, - "description": "VM cloning failures - DataVolume cloning, storage provisioning" - }, - "vm-lifecycle-manager": { - "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], - "secondary_docs": ["runtime-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3270, - "description": "VM start/stop failures - lifecycle management, scheduling" - }, - "vm-inventory": { - "primary_docs": [], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 450, - "description": "General guidance for status interpretation" - } - }, - - "error_to_docs_mapping": { - "ErrorUnschedulable": "scheduling-errors.md", - "ErrorDataVolumeNotReady": "storage-errors.md", - "ErrorPvcNotFound": "storage-errors.md", - "Terminating": "lifecycle-errors.md", - "CrashLoopBackOff": "runtime-errors.md", - "NetworkAttachmentFailed": "network-errors.md" - }, - - "cross_references": { - "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], - "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], - "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], - "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], - "network-errors.md": ["scheduling-errors.md"] - }, - - "progressive_disclosure_strategy": { - "description": "Load only the minimum documentation needed for current task", - "pattern": "error_type → primary_doc → cross_references (if needed)", - "token_savings": "29% average reduction vs single-file (based on rh-sre data)", - "navigation_overhead_reduction": "85% vs single-file navigation" - } -} diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/INDEX.md deleted file mode 100644 index a2e76963..00000000 --- a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/INDEX.md +++ /dev/null @@ -1,332 +0,0 @@ ---- -title: VM Troubleshooting Guide - Index -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 - - title: OpenShift Virtualization - Virtual Machine Status - url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html - date_accessed: 2026-02-06 - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] -semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] -use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] -last_updated: 2026-02-17 ---- - -# VM Troubleshooting Guide - Index - -## Overview - -This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: - -- **Creation**: Scheduling failures, storage provisioning issues -- **Lifecycle**: Start/stop failures, stuck states -- **Deletion**: Resources stuck in Terminating, storage cleanup failures -- **Cloning**: DataVolume cloning errors, cross-namespace issues -- **Networking**: Secondary network attachment failures -- **Runtime**: CrashLoopBackOff, guest OS boot failures - -This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. - ---- - -## 🗂️ Troubleshooting Categories - -The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: - -### 1. [Scheduling Errors](scheduling-errors.md) -**When to use**: VM fails to schedule on any node - -**Errors covered**: -- ErrorUnschedulable - Node Taints -- ErrorUnschedulable - Insufficient Resources -- ErrorUnschedulable - Node Selector Mismatch - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 2. [Storage Errors](storage-errors.md) -**When to use**: VM has storage provisioning, deletion, or cloning issues - -**Errors covered**: -- ErrorDataVolumeNotReady (all 3 subsections) -- ErrorPvcNotFound -- Storage Deletion Failures -- DataVolume Cloning Failures - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### 3. [Lifecycle Errors](lifecycle-errors.md) -**When to use**: VM has start/stop/termination issues - -**Errors covered**: -- VM Stuck in Terminating State -- VM Won't Start (Non-Scheduling Issues) -- VM Won't Stop - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### 4. [Runtime Errors](runtime-errors.md) -**When to use**: VM repeatedly crashes or fails at runtime - -**Errors covered**: -- CrashLoopBackOff - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 5. [Network Errors](network-errors.md) -**When to use**: VM has secondary network attachment failures - -**Errors covered**: -- Network Attachment Failures (Multus, SR-IOV) - -**Skills that use this**: vm-create - ---- - -## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last - -### Critical Principle: MCP-First Approach - -**All diagnostic procedures in this guide follow the MCP-first pattern**: - -``` -1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) -2. ⚠️ IF FAILS: Ask user permission to use CLI command -3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval -``` - -### Why MCP Tools First? - -- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources -- **Consistency**: Same interface across all operations -- **Better Error Handling**: MCP tools return structured errors -- **Audit Trail**: MCP tool usage is logged and trackable - -### Command Pattern Examples - -Throughout this guide, you'll see diagnostic steps formatted like this: - -**✅ CORRECT Pattern (MCP First)**: - -```markdown -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Extract**: `.metadata.finalizers` from returned JSON - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` -``` - -### Available MCP Tools - -The `openshift-virtualization` MCP server provides these tools: - -**Resource Operations**: -- `resources_get` - Get specific resource (replaces `oc get `) -- `resources_list` - List resources (replaces `oc get `) -- `resources_delete` - Delete resource (replaces `oc delete`) -- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) - -**Pod Operations**: -- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) -- `pods_get` - Get pod details (replaces `oc get pod`) -- `pods_log` - Get pod logs (replaces `oc logs`) -- `pods_exec` - Execute in pod (replaces `oc exec`) -- `pods_delete` - Delete pod (replaces `oc delete pod`) -- `pods_top` - Pod resource usage (replaces `oc top pods`) - -**Events & Monitoring**: -- `events_list` - List events (replaces `oc get events`) -- `nodes_top` - Node resource usage (replaces `oc top nodes`) -- `nodes_log` - Node logs (replaces `oc adm node-logs`) -- `nodes_stats_summary` - Detailed node stats - -**VM Operations** (KubeVirt toolset): -- `vm_create` - Create VMs -- `vm_lifecycle` - Start/stop/restart VMs - -**Namespaces**: -- `namespaces_list` - List namespaces (replaces `oc get namespaces`) -- `projects_list` - List OpenShift projects (replaces `oc get projects`) - -### When CLI Commands Are Required - -Some operations have **NO MCP equivalent** and require CLI: - -- `virtctl` commands (console, VNC access) -- `oc debug node` (node debugging) -- `oc auth can-i` (permission checks) -- `oc adm taint` (node taint management) -- Complex JSON patch operations - -For these, the guide will note: **"CLI Only - No MCP equivalent"** - -### Quick Reference: CLI → MCP Mapping - -| CLI Command | MCP Tool Equivalent | -|-------------|---------------------| -| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | -| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | -| `oc logs -n ` | `pods_log` with name="", namespace="" | -| `oc get events -n ` | `events_list` with namespace="" | -| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | - -**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. - -### How to Read Diagnostic Sections - -Each error section includes: -1. **Symptom** - What you observe -2. **Description** - What's happening -3. **Possible Causes** - Why it's happening -4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback -5. **Solutions** - **MCP tools first**, then CLI fallback -6. **Verification** - **MCP tools first**, then CLI fallback - -**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. - ---- - -## 🔍 Quick Navigation by Skill - -**vm-create**: -- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics -- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning -- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff -- [Network Errors](network-errors.md) - Network attachment failures - -**vm-delete**: -- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state -- [Storage Errors](storage-errors.md) - Storage deletion failures - -**vm-clone**: -- [Storage Errors](storage-errors.md) - DataVolume cloning failures - -**vm-lifecycle-manager**: -- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop -- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling - -**vm-inventory**: -- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed - ---- - -## 📊 Documentation Coverage & Maintenance - -### Current Coverage - -This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: - -- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) -- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures -- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop -- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) -- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems - -**Total errors documented**: 12 error types across 6 categories - ---- - -### Encountering Undocumented Errors - -#### For AI Agents (Claude Code) - -If you encounter an error **not documented** in the categories above: - -1. **Report to user** with all available details (error message, affected resources, namespace) -2. **Provide best-effort diagnostics** using MCP tools: - - `resources_get` to inspect resource status - - `pods_log` to check virt-launcher or compute container logs - - `events_list` to view Kubernetes events -3. **Suggest documentation update**: - ``` - ⚠️ This error is not yet documented in the troubleshooting guide. - - **Error**: - **Affected resource**: / - - I recommend adding this error to the troubleshooting guide: - - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file - - If it's a new error class → Create new category file in docs/troubleshooting/ - - Would you like me to help document this error for future reference? - ``` - -4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs - ---- - -#### For Human Contributors - -To document a new error: - -1. **Determine the category**: - - Scheduling issue → `scheduling-errors.md` - - Storage problem → `storage-errors.md` - - Lifecycle/start/stop → `lifecycle-errors.md` - - Runtime crash → `runtime-errors.md` - - Network issue → `network-errors.md` - - New error class → Create new category file (e.g., `security-errors.md`) - -2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): - - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback - - **Solutions**: MCP Tool approach → Implementation → CLI Fallback - - **Verification**: MCP Tool checks → Expected results → CLI Fallback - -3. **Update semantic index**: - - Edit `.ai-index/semantic-index.json` - - Add error to `error_to_docs_mapping` - - Update relevant skill's `primary_docs` or `secondary_docs` - - Increment token estimates if needed - -4. **Reference in skill files**: - - Update skill's Reference Documentation section - - Ensure skill's Document Consultation steps point to the right category - -5. **Update this INDEX.md**: - - Add error to the appropriate category section (lines 50-120) - - Update "Current Coverage" list above - ---- - -## 📚 Additional Resources - -- [rh-virt README](../../README.md) - MCP server setup and skill documentation -- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources -- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/lifecycle-errors.md deleted file mode 100644 index 6167fcc8..00000000 --- a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/lifecycle-errors.md +++ /dev/null @@ -1,869 +0,0 @@ ---- -title: VM Lifecycle Errors -category: kubevirt -sources: - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 -tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] -semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] -use_cases: [vm-deletion, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Lifecycle Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM lifecycle issues including start/stop failures and stuck termination states. - -**When to use this document**: -- VM stuck in `Terminating` state -- VM won't start (runStrategy is Always but VM never reaches Running) -- VM won't stop (runStrategy is Halted but VM never reaches Stopped) - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### VM Stuck in Terminating State - -**Symptom**: VM shows status `Terminating` but deletion never completes - -**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). - -**Possible Causes**: -- Finalizers blocking deletion -- PVC/DataVolume still attached and preventing cleanup -- VirtualMachineInstance (VMI) not terminating properly -- Custom controllers or operators blocking deletion -- Stuck virt-launcher pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check finalizers on the VM**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` - -**2. Check if VMI still exists**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name containing "virt-launcher-". - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check events for deletion issues**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results where `involvedObject.name` == "" and sort by timestamp. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' -``` - -**5. Check VM deletion timestamp**: - -Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. - -If present, VM is in deletion process. If null, VM is not being deleted. - -**Common Finalizer Patterns**: -- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer -- `foregroundDeletion` - Waits for dependent objects to be deleted -- Custom finalizers from operators - -**Solutions** (Use MCP Tools First): - -1. **Wait for dependent resources** (safest approach): - - Use `resources_list` to check if PVCs, DataVolumes are still deleting - - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) - -2. **Force delete VMI** (if VMI is stuck): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable - requires explicit user permission): - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -3. **Force delete virt-launcher pod** (if pod is stuck): - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - # Find the virt-launcher pod name first - oc get pods -n | grep virt-launcher- - - # Then delete it - oc delete pod virt-launcher--xxx -n --force --grace-period=0 - ``` - -4. **Remove finalizers** (⚠️ dangerous - use only as last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` - 2. Remove items from `.metadata.finalizers` array - 3. Update VM using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch not easily done via MCP): - ```bash - # This operation is complex for MCP - may need CLI - oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Check for protection labels** (vm-delete skill specific): - - Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. - - If value is "true", the vm-delete skill refuses deletion (this is expected behavior). - -**Verification**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error if deletion successful. - -**CLI Fallback**: -```bash -oc get vm -n -# Should return: Error from server (NotFound) -``` - ---- - - ---- - -### VM Won't Start (Non-Scheduling Issues) - -**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state - -**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. - -**Possible Causes**: -- Guest OS kernel panic or boot failure -- Cloud-init configuration errors -- Missing or corrupted disk image -- Insufficient memory for guest OS to boot -- QEMU/KVM virtualization errors -- VirtualMachineInstance (VMI) creation failures -- virt-launcher pod crashes - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VMI (VirtualMachineInstance) status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and overall VMI state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**2. Check VMI conditions for errors**: - -Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. View virt-launcher pod logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -First, get pod name from step 3, then: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -Look for QEMU errors, memory allocation failures, device errors. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**5. Check virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. - -Alternatively, use `pods_get` to get full pod details: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**6. Access VM console to see guest OS boot messages**: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -Look for kernel panic, initramfs errors, filesystem errors. - -**7. Check VM events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep -A 20 "Events:" -``` - -**Common Error Patterns**: - -1. **Guest OS Boot Failure**: - - Console shows kernel panic - - Guest hangs at GRUB or boot loader - - Cloud-init errors during first boot - -2. **Insufficient Memory**: - - Guest OS kills processes due to OOM - - VMI logs show memory allocation errors - -3. **QEMU Crashes**: - - virt-launcher logs show QEMU segfaults - - VMI repeatedly restarts - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for kernel panic, initramfs errors, filesystem errors. - -2. **Check virt-launcher pod logs for QEMU errors**: - - Use `pods_log` from diagnostic step 4 to view logs. - - Look for: - - "qemu-system-x86_64: ..." errors - - Memory allocation failures - - Device errors - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx - ``` - -3. **Increase memory if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check VMI memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -4. **Verify disk image integrity**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check PVC is bound: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - If using container disk, verify image exists and is accessible (check VMI spec). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get pvc -n - ``` - -5. **Check cloud-init configuration** (if applicable): - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. - - Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq - ``` - -6. **Restart VMI** (soft reset): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete VMI (VM controller will recreate it): - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for new VMI to start (use `resources_get` to check status). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n -w - ``` - -7. **Check virtualization extensions** (KVM): - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - - # In debug shell: - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should eventually return `Running`). - -Check VMI is running: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should show `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should eventually return: Running - -oc get vmi -n -# Should show: Running -``` - ---- - -### VM Won't Stop - -**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state - -**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. - -**Possible Causes**: -- Guest OS not responding to ACPI shutdown signal -- virt-launcher pod stuck and not terminating -- VirtualMachineInstance (VMI) deletion blocked by finalizers -- Guest shutdown scripts hanging -- Filesystem sync issues in guest OS - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (might show `Stopping`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**2. Check VMI status and deletion timestamp**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check VMI events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vmi -n | grep -A 10 "Events:" -``` - -**5. Check VMI finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.metadata.finalizers}' -``` - -**6. Check if guest is responsive** (if VMI still exists): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Patterns**: -- VMI shows `deletionTimestamp` but never actually deletes -- virt-launcher pod in `Terminating` state -- VM runStrategy is `Halted` but printableStatus shows `Stopping` - -**Solutions** (Use MCP Tools First): - -1. **Wait for graceful shutdown** (default: 30 seconds): - - Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. - -2. **Force stop by deleting VMI**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - This is the standard way to force-stop a VM. - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n - # Should return: Error from server (NotFound) - ``` - -3. **Force delete VMI with grace period** (if VMI won't delete): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -4. **Force delete virt-launcher pod**: - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find the pod using `pods_list_in_namespace` from diagnostic step 3. - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - ⚠️ **Note**: For force deletion with grace period, use CLI fallback. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') - oc delete pod $POD_NAME -n --force --grace-period=0 - ``` - -5. **Remove VMI finalizers** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VMI using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update VMI using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Patch VM runStrategy directly** (ensure consistency): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` (diagnostic step 1) - 2. Set `.spec.runStrategy` to `"Halted"` - 3. Update VM using `resources_create_or_update` with modified JSON - - **CLI Fallback** (merge patch easier via CLI): - Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Stopped` or `Halted`). - -Verify VMI is gone: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Verify virt-launcher pod is gone: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for pods matching `virt-launcher-`. Should return no results. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Stopped or Halted - -oc get vmi -n -# Should return: Error from server (NotFound) - -oc get pods -n | grep virt-launcher- -# Should return: No resources found -``` - -**Prevention**: -- Ensure guest OS has ACPI support enabled -- Use proper shutdown commands in guest OS -- Avoid forceful stops unless necessary (can corrupt guest filesystem) - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/network-errors.md deleted file mode 100644 index 82fa313c..00000000 --- a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/network-errors.md +++ /dev/null @@ -1,429 +0,0 @@ ---- -title: VM Network Errors -category: kubevirt -sources: - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] -semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] -use_cases: [vm-creation, network-troubleshooting] -related_docs: [INDEX.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Network Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. - -**When to use this document**: -- VM created successfully but secondary networks not attached -- NetworkAttachmentDefinition not found errors -- Multus CNI failures -- SR-IOV device attachment issues - -**Skills that use this**: vm-create - ---- - -### Network Attachment Failures - -**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working - -**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). - -**Possible Causes**: -- NetworkAttachmentDefinition doesn't exist in the namespace -- Multus CNI not installed or not configured on cluster -- Namespace mismatch (NAD in different namespace than VM) -- Interface name conflicts in VM spec -- Bridge/network configuration errors in NAD -- SR-IOV device not available (if using SR-IOV) - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List NetworkAttachmentDefinitions in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "" -} -``` - -Review `.items[].metadata.name` for available NADs. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -n -``` - -**2. List NetworkAttachmentDefinitions in all namespaces**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition" -} -``` - -Omit `namespace` parameter to list across all namespaces. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -A -``` - -**3. Check specific NetworkAttachmentDefinition**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "", - "name": "" -} -``` - -Review `.spec.config` for CNI configuration. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definition -n -o yaml -``` - -**4. Check VM network configuration**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.networks` to see network references. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq -``` - -**5. Check VM domain interfaces**: - -Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq -``` - -**6. Check virt-launcher pod network annotations** (shows actual attachments): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**7. Check Multus is installed**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "openshift-multus" -} -``` - -Should show Multus CNI pods running. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n openshift-multus -``` - -**8. Check for errors in virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n -``` - -**Common Error Messages**: -- `"network-attachment-definition not found"` - NAD doesn't exist in namespace -- `"multus CNI not configured"` - Multus not installed or misconfigured -- `"interface name conflict"` - Duplicate interface names in VM spec -- `"failed to add network"` - CNI plugin error (check NAD config) -- `"no available devices"` - SR-IOV device not available (if using SR-IOV) - -**Solutions** (Use MCP Tools First): - -1. **Verify NetworkAttachmentDefinition exists in correct namespace**: - - Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. - - If NAD is in different namespace, copy it to VM namespace: - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get NAD from source namespace using `resources_get` - 2. Modify `.metadata.namespace` to target namespace - 3. Create NAD in target namespace using `resources_create_or_update` - - **CLI Fallback** (stream processing easier via CLI): - Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" - ```bash - oc get network-attachment-definition -n -o yaml | \ - sed "s/namespace: /namespace: /" | \ - oc apply -f - - ``` - -2. **Create missing NetworkAttachmentDefinition**: - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - Example: Linux bridge network - - **Parameters**: - ```json - { - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "metadata": { - "name": "vlan100", - "namespace": "" - }, - "spec": { - "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" - ```bash - cat < - spec: - config: '{ - "cniVersion": "0.3.1", - "type": "bridge", - "bridge": "br1", - "vlan": 100, - "ipam": { - "type": "host-local", - "subnet": "192.168.100.0/24" - } - }' - EOF - ``` - -3. **Check Multus CNI installation**: - - Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. - - To check cluster network operator: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "config.openshift.io/v1", - "kind": "ClusterOperator" - } - ``` - - Filter for `network` operator. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n openshift-multus - oc get clusteroperators network - ``` - -4. **Fix interface name conflicts** (if VM has duplicate names): - - Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. - - Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. - - **CLI Fallback** (interactive edit easier via CLI): - Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' - oc edit vm -n - ``` - -5. **Validate NAD configuration syntax**: - - Use `resources_get` from diagnostic step 3, extract `.spec.config`. - - Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get network-attachment-definition -n -o jsonpath='{.spec.config}' - ``` - -6. **Check SR-IOV device availability** (if using SR-IOV networks): - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - List SR-IOV network node policies: - - **Parameters**: - ```json - { - "apiVersion": "sriovnetwork.openshift.io/v1", - "kind": "SriovNetworkNodePolicy", - "namespace": "openshift-sriov-network-operator" - } - ``` - - Check SR-IOV device plugin pods: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "openshift-sriov-network-operator" - } - ``` - - Filter for pods with "device-plugin" in name. - - Check available SR-IOV devices on node: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "Node", - "name": "" - } - ``` - - Review `.status.allocatable` for SR-IOV resources. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get sriovnetworknodepolicy -n openshift-sriov-network-operator - oc get pods -n openshift-sriov-network-operator | grep device-plugin - oc describe node | grep -A 10 "Allocatable:" - ``` - -7. **Recreate VM with corrected network configuration** (if needed): - - If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -After remediation, check virt-launcher pod network status: - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -Should show all attached networks with status. Example output: -```json -[ - { - "name": "openshift-sdn", - "interface": "eth0", - "ips": ["10.128.2.10"], - "default": true - }, - { - "name": "vlan100", - "interface": "net1", - "ips": ["192.168.100.5"] - } -] -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**Check from inside VM** (via console): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# In guest OS: -ip addr show -# Should show all network interfaces (eth0, net1, etc.) -``` - -**Common Network Types**: -- **Linux Bridge**: Layer 2 bridge for VLAN networks -- **SR-IOV**: High-performance direct device assignment -- **macvlan**: MAC-based VLAN for container networks -- **OVN-Kubernetes**: OpenShift native overlay network - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/runtime-errors.md deleted file mode 100644 index 804d3cbc..00000000 --- a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/runtime-errors.md +++ /dev/null @@ -1,616 +0,0 @@ ---- -title: VM Runtime Errors -category: kubevirt -sources: - - title: Kubernetes Pod Lifecycle - url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ - date_accessed: 2026-02-17 -tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] -semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] -use_cases: [vm-creation, vm-lifecycle, diagnostics] -related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Runtime Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. - -**When to use this document**: -- VM or virt-launcher pod shows `CrashLoopBackOff` status -- virt-launcher pod repeatedly restarting -- Guest OS kernel panics on boot - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### CrashLoopBackOff - -**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting - -**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. - -**Possible Causes**: -- Guest OS kernel panic on boot -- Insufficient resources (memory/CPU) for guest OS -- Corrupted disk image or filesystem -- QEMU/libvirt crashes due to configuration errors -- Missing or incompatible device drivers in guest -- Resource limits too low for virt-launcher pod -- Virtualization features (KVM) not available on node - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check virt-launcher pod restart count**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# Look at RESTARTS column -``` - -**2. View recent crash logs** (previous container instance): - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "previous": true, - "tail": 100 -} -``` - -Look for QEMU errors, kernel panics, or segfaults. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n virt-launcher--xxx --previous -``` - -**3. Check current virt-launcher logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. - -**CLI Fallback** (if MCP unavailable or all containers needed): -```bash -oc logs -n virt-launcher--xxx --all-containers -``` - -**4. Check VMI conditions for crash details**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Extract `.status.conditions` for crash details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**5. Check pod events for crash reasons**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" -``` - -**6. Check pod resource limits**: - -**MCP Tool**: `pods_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -Extract `.spec.containers[0].resources` for resource limits. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq -``` - -**7. Check node kubelet logs for OOM kills**: - -⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -oc adm node-logs -u kubelet | grep -i oom -``` - -**8. Access guest console** (if VM briefly starts): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Crash Patterns**: - -1. **Guest Kernel Panic**: - - Console logs show kernel panic messages - - Guest crashes immediately after boot - - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" - -2. **OOM (Out of Memory)**: - - Pod killed with reason: `OOMKilled` - - Guest runs out of memory during boot or operation - - virt-launcher logs show memory allocation failures - -3. **QEMU Crash**: - - virt-launcher logs show QEMU segmentation fault - - Symptoms: "qemu-system-x86_64: terminated by signal" - - Configuration incompatibility or QEMU bug - -4. **Disk Image Corruption**: - - Guest cannot boot from disk - - Filesystem errors in guest console - - DataVolume import failed - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for kernel panic or boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for: - - Kernel panic messages - - Initramfs errors - - Filesystem mounting failures - - Missing device errors - -2. **Review virt-launcher crash logs**: - - Use `pods_log` with `previous: true` from diagnostic step 2. - - Look for: - - QEMU command line errors - - Device initialization failures - - Memory allocation errors - - Signal termination (SIGSEGV, SIGABRT) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx --previous - ``` - -3. **Check for OOM (Out of Memory) kills**: - - **MCP Tool**: `pods_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - Extract `.status.containerStatuses[0].lastState.terminated.reason`. - - If returns `"OOMKilled"`: - - Option 1: Increase virt-launcher memory limits - - Option 2: Decrease guest memory allocation - - Option 3: Use smaller instance type - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' - ``` - -4. **Increase resources if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check current memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -5. **Verify disk image integrity**: - - **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters for list**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" - } - ``` - - **Parameters for specific DV**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check `.status.phase` (should be `Succeeded`). - - If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get datavolume -n -o jsonpath='{.status.phase}' - ``` - -6. **Check virtualization (KVM) availability**: - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -7. **Simplify VM configuration** (eliminate variables): - - Try creating minimal VM using vm-create skill with: - - Small instance type - - No secondary networks - - Simple container disk (e.g., Fedora) - - No cloud-init - - If minimal VM works, add features back one by one. - -8. **Recreate VM with different workload** (test disk image): - - If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) - -After remediation, check pod restart count stops increasing: - -**Parameters for pods**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). - -Check VM reaches Running state: - -**Parameters for VM**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# RESTARTS should stabilize - -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Running -``` - -Verify guest is responsive: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# Should show login prompt or OS console -``` - -**Advanced Debugging**: - -**MCP Tool**: `pods_exec` (from openshift-virtualization) - -Check libvirt domain XML: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["virsh", "dumpxml", "1"] -} -``` - -Check QEMU process: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["ps", "aux"] -} -``` - -Filter output for "qemu" process. - -**CLI Fallback** (if MCP unavailable): -```bash -oc exec -n virt-launcher--xxx -- virsh dumpxml 1 -oc exec -n virt-launcher--xxx -- ps aux | grep qemu -``` - -**Prevention**: -- Start with minimal VM configuration and add complexity gradually -- Use recommended instance types for your workload -- Test disk images before deploying to production -- Ensure nodes have adequate resources and KVM support -- Monitor resource usage and set appropriate limits - ---- - -## Workaround Patterns for MCP Tool Limitations - -### General Pattern: Diagnose → Propose → Confirm → Execute - -When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): - -**1. Diagnose** the root cause using Kubernetes commands -```bash -oc describe vm -n -oc get events -n --field-selector involvedObject.name= -``` - -**2. Propose** a manual workaround to the user with clear explanation -- Explain why the issue occurred -- Show what will be changed -- List alternative options - -**3. Confirm** with user before executing (Human-in-the-Loop) -- Wait for explicit "yes" or "apply workaround" -- Never auto-execute modifications - -**4. Execute** the workaround using oc/kubectl -```bash -oc patch vm -n --type=merge -p '...' -``` - -**5. Verify** the fix was successful -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**6. Document** the limitation and suggest filing enhancement request -- Note this is temporary until MCP tool is enhanced -- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues - -### Example: Adding Tolerations Workaround - -**Diagnostic Output**: -```markdown -## ⚠️ VM Scheduling Issue Detected - -**Root Cause**: Node taints prevent VM scheduling - -**Details**: -- Found 3 nodes with taint: `virtualization=true:NoSchedule` -- VM spec does not include matching tolerations -- This prevents VM scheduling on virtualization-dedicated nodes -``` - -**Proposed Workaround**: -```bash -oc patch vm web-server -n vms --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' -``` - -**User Confirmation Required**: -``` -How would you like to proceed? -- "apply workaround" - I'll patch the VM with tolerations -- "manual" - I'll provide instructions for you to apply manually -- "cancel" - Delete the VM and abort creation -``` - -**After Execution**: -```markdown -## ✓ Workaround Applied Successfully - -**Action**: Added tolerations for taint `virtualization=true:NoSchedule` -**New Status**: Stopped (VM can now be scheduled) - -**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. -Future VMs in this cluster will need the same fix until the tool is enhanced. -``` - ---- - -## VM Status Reference - -### Status Values - -| Status | Meaning | Action Required | -|--------|---------|-----------------| -| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | -| `Running` | VM is running | Normal | -| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | -| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | -| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | -| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | -| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | -| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | -| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | -| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | - -### Checking VM Status - -```bash -# Get printable status -oc get vm -n -o jsonpath='{.status.printableStatus}' - -# Get detailed status and conditions -oc get vm -n -o jsonpath='{.status}' | jq - -# Watch status changes in real-time -oc get vm -n -w -``` - ---- - -## Best Practices for Agents - -When implementing diagnostic workflows: - -1. **Always verify VM status** after creation (wait 5-10 seconds first) -2. **Consult this document** when encountering error status values -3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) -4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) -5. **Respect human-in-the-loop** for all VM modifications -6. **Document temporary workarounds** and their limitations clearly -7. **Suggest filing issues** for missing MCP tool features - -### Document Consultation Pattern - -```markdown -**Document Consultation** (REQUIRED): -1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes -2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." -``` - ---- - -## Known MCP Tool Limitations - -### vm_create tool - -**Currently Supported**: -- ✓ Namespace, name (required) -- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) -- ✓ Size hints (small, medium, large) -- ✓ Storage size -- ✓ Autostart flag -- ✓ Networks (Multus NetworkAttachmentDefinitions) -- ✓ Performance family (u1, o1, c1, m1) -- ✓ Instance type, preference - -**Not Currently Supported** (requires workarounds): -- ✗ Tolerations (for node taints) -- ✗ Node selectors -- ✗ Affinity/anti-affinity rules -- ✗ Resource requests/limits (beyond instance type) -- ✗ Custom labels/annotations -- ✗ SSH keys injection -- ✗ Cloud-init user data - -**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. - -**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues - ---- - -## Additional Resources - -- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) -- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) -- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) -- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) - ---- - -[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/scheduling-errors.md deleted file mode 100644 index e47c8c4f..00000000 --- a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/scheduling-errors.md +++ /dev/null @@ -1,417 +0,0 @@ ---- -title: VM Scheduling Errors -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 -tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] -semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] -use_cases: [vm-creation, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Scheduling Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**When to use this document**: -- VM shows status `ErrorUnschedulable` after creation or start attempt -- VM events mention scheduling failures, taints, resources, or node selectors - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -## ErrorUnschedulable - -**Symptom**: VM shows status `ErrorUnschedulable` after creation - -**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**Possible Causes**: - -### 1. Node Taints (Most Common) - -Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM events for scheduling failures**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `involvedObject.name` == "" and look for messages like: -- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= -``` - -**2. Check node taints in the cluster**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' -``` - -**Common Taint Patterns**: -- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule -- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes -- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads - -**Solution - Add Tolerations to VM**: - -The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. - -**Workaround (post-creation using MCP Tools)**: - -**Step 1**: Get current VM spec - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Step 2**: Modify the returned JSON to add tolerations - -Add to `.spec.template.spec.tolerations`: -```json -{ - "tolerations": [ - { - "key": "virtualization", - "operator": "Equal", - "value": "true", - "effect": "NoSchedule" - } - ] -} -``` - -**Step 3**: Update VM with modified spec - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters**: -```json -{ - "resource": "" -} -``` - -Pass the complete modified VM resource as YAML or JSON string. - -**Step 4**: Verify tolerations were added - -Use `resources_get` again and check `.spec.template.spec.tolerations` in response. - -**Step 5**: Check if VM status improved - -Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. - -**CLI Fallback** (if MCP patch is too complex): -```bash -# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' - -# Verify tolerations -oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq - -# Check status -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**Example - Multiple Tolerations**: -```bash -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" - - key: "dedicated" - operator: "Equal" - value: "virt-workloads" - effect: "NoSchedule" -' -``` - -**Toleration Operators**: -- `Equal` - Key and value must match exactly -- `Exists` - Only key must exist (ignores value) - -**Toleration Effects**: -- `NoSchedule` - Don't schedule new pods (existing pods continue) -- `PreferNoSchedule` - Avoid scheduling if possible -- `NoExecute` - Don't schedule AND evict existing pods - -**Alternative Solutions**: -1. **Remove node taints** (if you have cluster-admin access): - ```bash - oc adm taint nodes virtualization=true:NoSchedule- - ``` - -2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits - -3. **File enhancement request** - Request tolerations support in openshift-mcp-server: - https://github.com/openshift/openshift-mcp-server/issues - ---- - -### 2. Insufficient Resources - -Not enough CPU, memory, or storage available on any node. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM resource requests**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' -``` - -**2. Check node resource availability**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. - -Alternatively, use `nodes_top` MCP tool for current resource usage. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe nodes | grep -A 5 "Allocated resources" -``` - -**3. Look for VM events mentioning "Insufficient"**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep "Insufficient" -``` - -**Example Event**: -``` -0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. -``` - -**Solutions** (Use MCP Tools First): - -1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) -2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill -3. **Delete unused VMs** - Use vm-delete skill to free up resources -4. **Check resource quotas**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters for quota**: - ```json - { - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" - } - ``` - - **Parameters for limit range**: - ```json - { - "apiVersion": "v1", - "kind": "LimitRange", - "namespace": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - oc describe limitrange -n - ``` - ---- - -### 3. Node Selector Mismatch - -VM requires specific node labels that don't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM node selector requirements**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.nodeSelector` to see required node labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' -``` - -**2. List available node labels**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.metadata.labels` for available labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes --show-labels -``` - -**3. Check if any nodes match the selector**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node", - "labelSelector": "=" -} -``` - -Should return at least one node with matching labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -l = -``` - -**Solutions** (Use MCP Tools First): - -**Option 1: Remove node selector from VM** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get current VM using `resources_get` (diagnostic step 1) -2. Remove `.spec.template.spec.nodeSelector` field -3. Update VM using `resources_create_or_update` with modified JSON - -**CLI Fallback** (JSON patch easier via CLI): -Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" -```bash -oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' -``` - -**Option 2: Add label to nodes** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get node using `resources_get` -2. Add label to `.metadata.labels` -3. Update node using `resources_create_or_update` - -⚠️ **Note**: Node labeling typically requires cluster admin privileges. - -**CLI Fallback** (simpler via CLI): -Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" -```bash -oc label node = -``` - ---- - -[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/storage-errors.md deleted file mode 100644 index 16ba584d..00000000 --- a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/docs/troubleshooting/storage-errors.md +++ /dev/null @@ -1,1011 +0,0 @@ ---- -title: VM Storage Errors -category: kubevirt -sources: - - title: KubeVirt CDI - DataVolumes - url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ - date_accessed: 2026-02-17 - - title: Kubernetes Persistent Volumes - url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ - date_accessed: 2026-02-17 -tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] -semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] -use_cases: [vm-creation, vm-deletion, vm-cloning] -related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] -last_updated: 2026-02-17 ---- - -# VM Storage Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. - -**When to use this document**: -- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` -- Storage deletion fails after VM deletion -- DataVolume cloning operations fail -- PVC provisioning issues - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### ErrorDataVolumeNotReady - -**Symptom**: VM shows status `ErrorDataVolumeNotReady` - -**Description**: The DataVolume (persistent storage) backing the VM is not ready. - -**Possible Causes**: - -#### 1. DataVolume Still Provisioning - -Storage provisioning takes time, especially for large disks or when importing images. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Get detailed DataVolume information**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` and `.status.conditions` for provisioning details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o json -``` - -**3. Check PVC (PersistentVolumeClaim) bound status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Check `.status.phase` for each PVC (should be `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. - -#### 2. Storage Class Not Found - -The requested storage class doesn't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List available storage classes**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" -} -``` - -Review the list of available storage classes (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -``` - -**2. Check DataVolume's requested storage class**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.spec.pvc.storageClassName` in the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' -``` - -**Solution**: -1. Use a valid storage class from the cluster -2. Recreate VM with correct storage class parameter - -#### 3. Insufficient Storage Quota - -Namespace has insufficient storage quota to provision the PVC. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check resource quotas**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**2. Check storage usage**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase -``` - -**Solution**: -1. Request quota increase from cluster admin -2. Delete unused PVCs to free quota -3. Reduce VM storage size - ---- - -### ErrorPvcNotFound - -**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List PVCs in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review the list of available PVCs (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check VM's PVC references**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' -``` - -**Solution**: -- Wait for DataVolume to create the PVC -- Manually create missing PVC -- Fix VM spec to reference correct PVC name - ---- - - ---- - -### Storage Deletion Failures - -**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace - -**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. - -**Possible Causes**: -- PVC still bound to active PersistentVolume with `Retain` policy -- DataVolume still being referenced by another resource -- CDI (Containerized Data Importer) controller issues -- Storage class retention policy preventing deletion -- Finalizers on PVC/DataVolume blocking cleanup -- PVC still mounted by a pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check PVC status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check specific PVC phase**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Released` or `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o jsonpath='{.status.phase}' -``` - -**3. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**4. Check what's using the PVC**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' -``` - -**5. Check PVC finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. - -**6. Check DataVolume finalizers**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.metadata.finalizers}' -``` - -**7. Check PV reclaim policy**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume" -} -``` - -Filter results for PV where `.spec.claimRef.name` matches ``. - -To get specific PV policy: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume", - "name": "" -} -``` - -Check `.spec.persistentVolumeReclaimPolicy`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pv | grep -oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' -``` - -**Common Finalizer Patterns**: -- `kubernetes.io/pvc-protection` - Protects PVC while in use -- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer - -**Solutions** (Use MCP Tools First): - -1. **Delete DataVolume first, then PVC**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete DataVolume first (often blocks PVC deletion): - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Wait a few seconds, then delete PVC: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete datavolume -n - oc delete pvc -n - ``` - -2. **Check for pods still using PVC**: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "" - } - ``` - - Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. - - Then delete the pods using `pods_delete`: - - **Parameters**: - ```json - { - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' - oc delete pod -n - ``` - -3. **Force delete PVC** (if safe to do so): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete pvc -n --grace-period=0 --force - ``` - -4. **Remove finalizers from PVC** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PVC using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update PVC using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Remove finalizers from DataVolume** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current DataVolume using `resources_get` (diagnostic step 6) - 2. Remove items from `.metadata.finalizers` array - 3. Update DataVolume using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Change PV reclaim policy** (if PV has Retain policy): - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PV policy using `resources_get` (diagnostic step 7) - 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` - 3. Update PV using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" - ```bash - oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' - oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' - ``` - -**Storage Quota Check** (Use MCP Tools First): - -After deletion, verify storage quota is freed: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters for quota check**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.used` to verify storage quota is freed. - -**Parameters for PVC verification**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -oc get pvc -n -``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Confirm PVC is deleted: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Confirm DataVolume is deleted: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -# Should return: Error from server (NotFound) - -oc get datavolume -n -# Should return: Error from server (NotFound) -``` - ---- - -### DataVolume Cloning Failures - -**Symptom**: VM clone created successfully but DataVolume clone operation fails - -**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. - -**Possible Causes**: -- CSI driver doesn't support volume cloning -- Source PVC storage class incompatible with cloning -- Cross-namespace cloning not permitted by storage backend -- Insufficient storage quota in target namespace -- Source PVC not in `Bound` state -- Storage class doesn't have volume cloning enabled -- CDI (Containerized Data Importer) controller issues - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume clone status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and `.metadata.name`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Check DataVolume events for errors**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe datavolume -n -``` - -**3. Check DataVolume phase**: - -Use `resources_get` from step 1, extract `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -``` - -**4. Check if storage class supports cloning**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass", - "name": "" -} -``` - -Review the full YAML output for cloning-related configurations. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -o yaml | grep -A 5 -i clone -``` - -**5. Check CSI driver capabilities**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "CSIDriver" -} -``` - -Review `.items[].metadata.name` for available CSI drivers. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get csidriver -``` - -**6. Check source PVC status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Bound` for cloning to work). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**7. Check target namespace storage quota**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**8. Check CDI controller logs**: - -**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) - -First, list pods in openshift-cnv namespace: - -**Parameters for pods_list_in_namespace**: -```json -{ - "namespace": "openshift-cnv", - "labelSelector": "app.kubernetes.io/component=cdi-deployment" -} -``` - -Then get logs using `pods_log`: - -**Parameters**: -```json -{ - "namespace": "openshift-cnv", - "name": "", - "tail": 100 -} -``` - -**CLI Fallback** (if MCP unavailable or easier via CLI): -```bash -oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') -``` - -**Common Error Messages**: -- `"volume cloning is not supported"` - CSI driver lacks clone capability -- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage -- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace -- `"insufficient quota"` - Target namespace lacks storage quota -- `"source PVC not bound"` - Source PVC must be in Bound state for cloning -- `"StorageClass does not support cloning"` - Storage class configuration issue - -**Solutions** (Use MCP Tools First): - -1. **Check storage class clone support**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" - } - ``` - - For each storage class in `.items[]`, check: - - `.metadata.name` (storage class name) - - `.provisioner` (CSI driver) - - Storage classes using CSI drivers typically support cloning. Look for provisioners like: - - `csi.ovirt.org` (oVirt CSI) - - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) - - `ebs.csi.aws.com` (AWS EBS CSI) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner - ``` - -2. **Verify source PVC is bound**: - - Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pvc -n -o jsonpath='{.status.phase}' - ``` - -3. **Check target namespace quota**: - - Use `resources_list` from diagnostic step 7 to check quota. - - If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - ``` - -4. **Use snapshot-based cloning** (alternative method): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Step 1**: Create VolumeSnapshot of source PVC - - **Parameters**: - ```json - { - "apiVersion": "snapshot.storage.k8s.io/v1", - "kind": "VolumeSnapshot", - "metadata": { - "name": "-snapshot", - "namespace": "" - }, - "spec": { - "source": { - "persistentVolumeClaimName": "" - } - } - } - ``` - - **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) - - **Step 3**: Create new DataVolume from snapshot - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "metadata": { - "name": "-rootdisk", - "namespace": "" - }, - "spec": { - "source": { - "snapshot": { - "name": "-snapshot", - "namespace": "" - } - }, - "storage": { - "resources": { - "requests": { - "storage": "50Gi" - } - }, - "storageClassName": "" - } - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" - ```bash - cat <-snapshot - namespace: - spec: - source: - persistentVolumeClaimName: - EOF - - oc get volumesnapshot -snapshot -n - - cat <-rootdisk - namespace: - spec: - source: - snapshot: - name: -snapshot - namespace: - storage: - resources: - requests: - storage: 50Gi - storageClassName: - EOF - ``` - -5. **Use "new empty storage" option** (vm-clone skill): - - If cloning isn't supported, create VM with empty storage - - Manually copy data if needed - -6. **Cross-namespace cloning workaround**: - - Some storage backends require snapshot for cross-namespace cloning - - Create snapshot in source namespace, restore in target namespace (see solution 4 above) - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Check DataVolume reached Succeeded phase: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should return `Succeeded`). - -Check PVC was created and bound: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "-rootdisk" -} -``` - -Check `.status.phase` (should return `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -# Should return: Succeeded - -oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' -# Should return: Bound -``` - -**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/skills/vm-lifecycle-manager/SKILL.md b/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/skills/vm-lifecycle-manager/SKILL.md deleted file mode 100644 index e41c3f78..00000000 --- a/evaluation/with_skills/rh-virt__vm-lifecycle-manager/environment/skills/vm-lifecycle-manager/SKILL.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -name: vm-lifecycle-manager -description: | - Manage virtual machine lifecycle operations including start, stop, and restart. - - Use when: - - "Start VM [name]" - - "Stop the virtual machine [name]" - - "Restart VM [name]" - - "Power on/off VM [name]" - - This skill handles VM state transitions safely with user confirmation for each action. - - NOT for creating VMs (use vm-create) or deleting VMs (use vm-delete). - -model: inherit -color: blue ---- - -# /vm-lifecycle-manager Skill - -Control virtual machine power state in OpenShift Virtualization using the `vm_lifecycle` tool. - -## Prerequisites - -**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools**: -- `vm_lifecycle` (from openshift-virtualization) - Manage VM power state - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster (>= 4.19) -- OpenShift Virtualization operator installed -- ServiceAccount with RBAC permissions to update VirtualMachine resources - -### Prerequisite Verification - -**Before executing:** - -1. Check `openshift-virtualization` exists in `.mcp.json` → If missing, report setup -2. Verify `KUBECONFIG` is set (presence only, never expose value) → If missing, report - -**Human Notification Protocol:** `❌ Cannot execute vm-lifecycle-manager: MCP server not available. Setup: Add to .mcp.json, set KUBECONFIG, restart Claude Code. Docs: https://github.com/openshift/openshift-mcp-server` - -⚠️ **SECURITY**: Never display KUBECONFIG path or credential values. - -## When to Use This Skill - -**Trigger when:** -- User explicitly invokes `/vm-lifecycle-manager` command -- User requests starting/stopping/restarting a VM -- User wants to change VM power state - -**User phrases:** -- "Start VM web-server in namespace vms" -- "Stop the database VM" -- "Restart test-vm" -- "Power on the VM called api-server" -- "/vm-lifecycle-manager" (explicit command) - -**Do NOT use when:** -- Create VM → `/vm-create` -- List VMs → `/vm-inventory` -- Delete VM → `/vm-delete` - -## Workflow - -### Step 1: Gather Parameters and Confirm - -**Required from user:** VM Name, Namespace, Action (start|stop|restart) - -**Present for confirmation:** - -```markdown -## VM Lifecycle Operation - -| Parameter | Value | Impact | -|-----------|-------|--------| -| VM Name | `` | from user | -| Namespace | `` | from user | -| Action | `` | start: consumes resources / stop: graceful shutdown / restart: brief interruption (~1-2min) | - -Confirm: yes/no -``` - -Proceed automatically unless the user explicitly objects. - -### Step 2: Execute Lifecycle Operation - -**ONLY AFTER user confirmation in Step 1.** - -**For start or stop actions:** - -**MCP Tool**: `vm_lifecycle` (namespace=``, name=``, action=``) - -**For restart action (composite operation):** - -**CRITICAL**: Implement restart as two separate operations to avoid resourceVersion conflicts: - -1. **Stop VM**: `vm_lifecycle` (namespace=``, name=``, action="stop") -2. **Verify stopped**: `resources_get` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", namespace=``, name=``) → Check `status.printableStatus` == "Stopped" -3. **Wait**: 5 seconds for VM to fully stop -4. **Start VM**: `vm_lifecycle` (namespace=``, name=``, action="start") -5. **Verify started**: `resources_get` → Check `status.printableStatus` == "Running" - -**Errors:** -- VM not found → Report, suggest vm-inventory -- Permission denied → Report RBAC error -- Already in desired state → Inform user -- Stop fails during restart → Report, do not proceed to start -- Start fails during restart → Report, VM is stopped -- Transition fails → Report details - -### Step 3: Report Operation Status - -**On Success:** - -```markdown -## ✓ VM Successful - -**VM**: `` | **Namespace**: `` | **Action**: | **RunStrategy**: - -**Impact**: -- **start**: Running, consuming resources (CPU/memory). Access: virtctl console or SSH. RunStrategy: Always (auto-restart on crash) -- **stop**: Stopped, resources freed. State preserved. Start: "Start VM ". RunStrategy: Halted (stays off) -- **restart**: Running after stop+start. Brief interruption (~1-2min). Monitor app logs. RunStrategy: Always - -**Next**: "Show status of VM " or "List VMs in namespace " -``` - -**On Failure:** - -**OPTIONAL**: Read [lifecycle-errors.md](../../docs/troubleshooting/lifecycle-errors.md) for start/stop failures or [scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) for ErrorUnschedulable. Output: "Consulted lifecycle-errors.md for failure." - -**When to consult**: Start/stop failures, stuck transitions, unexpected errors. **NOT**: Already in state, not found, RBAC errors. - -```markdown -## ❌ Lifecycle Operation Failed - -**Error**: - -**Causes**: VM not found | RBAC denied | Already in desired state | VM in transition (wait 30-60s) | Resource constraints (start) - -**Troubleshoot**: -1. vm-inventory to verify VM exists -2. Check RBAC: `oc auth can-i update virtualmachines -n ` -3. View VM status and events -4. Check node capacity (for start operations) -``` - -## Common Issues - -### Issue 1: VM Not Found -**Error**: "VirtualMachine 'xyz' not found in namespace 'abc'" -**Solution**: Verify spelling, check namespace, use vm-inventory, VM may be deleted - -### Issue 2: VM Already in Desired State -**Warning**: "VM is already running" (when attempting start) -**Solution**: Not an error - VM already in desired state. Use `restart` if intended to restart - -### Issue 3: Permission Denied -**Error**: "Forbidden: User cannot update VirtualMachines" -**Solution**: Verify RBAC permissions (update VirtualMachine resources), contact admin - -### Issue 4: VM Stuck in Transitioning State -**Error**: "VM stuck in 'Terminating' or 'Starting'" -**Solution**: Wait 30-60s, check events (`oc describe vm`), use vm-troubleshooter, check virt-launcher pod - -### Issue 5: Insufficient Resources (Start) -**Error**: "Insufficient CPU/memory to start VM" -**Solution**: Check cluster availability, stop other VMs, scale nodes, resize VM to smaller instance type - -### Issue 6: Restart Implementation -**Note**: Restart is implemented as two separate operations (stop → verify → start → verify) -**Reason**: Avoids Kubernetes resourceVersion conflicts when using single restart action -**Behavior**: If stop succeeds but start fails, VM remains stopped. Check VM status with vm-inventory - -## Understanding RunStrategy - -| Action | RunStrategy | Behavior | -|--------|------------|----------| -| start | Always | Runs, auto-restarts on crash | -| stop | Halted | Stops, stays off | -| restart | Always | Stops, starts, auto-restarts | - -## Dependencies - -### Required MCP Servers -- `openshift-virtualization` - OpenShift MCP server with KubeVirt toolset - -### Required MCP Tools -- `vm_lifecycle` - Manage VM power state (start/stop/restart) - -### Related Skills -- `vm-create` - Create VMs -- `vm-inventory` - Check VM status -- `vm-troubleshooter` (planned) - Diagnose startup/shutdown issues - -### Reference Documentation -- [lifecycle-errors.md](../../docs/troubleshooting/lifecycle-errors.md) - Start/stop failures, stuck transitions (consulted on failures) -- [scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) - ErrorUnschedulable, resource constraints (consulted when VM won't start) -- [Troubleshooting INDEX](../../docs/troubleshooting/INDEX.md) - Navigation hub for error categories -- [OpenShift Virt Docs](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt/about_virt/about-virt.html) -- [KubeVirt Lifecycle](https://kubevirt.io/user-guide/virtual_machines/lifecycle/) -- [RunStrategy Docs](https://kubevirt.io/user-guide/virtual_machines/run_strategies/) -- [OpenShift MCP](https://github.com/openshift/openshift-mcp-server) - -## Critical: Human-in-the-Loop Requirements - -**IMPORTANT:** This skill requires explicit user confirmation before executing. You MUST: - -1. **Wait for user confirmation** on specific action (start/stop/restart) before executing `vm_lifecycle` -2. **Do NOT proceed** until user explicitly approves -3. **Present action clearly** with VM name, namespace, intended state change -4. **Never auto-execute** - these operations impact running services -5. **Warn about consequences** - stopping interrupts services, starting consumes resources, restarting causes downtime - -If user says "no" or wants to reconsider, do not proceed. - -**Why**: start (consumes resources), stop (interrupts services), restart (brief downtime). User should verify correct VM and understand impact. - -## Security Considerations - -- Requires RBAC permissions (update VirtualMachine resources) -- Stop operations attempt graceful shutdown; guest OS/app data consistency depends on workload -- Start operations consume resources (quota limits apply) -- All operations audited in Kubernetes API logs -- KUBECONFIG credentials never exposed -- ServiceAccount permissions limit controllable VMs - -## Example Usage - -### Example 1: Start Stopped VM - -``` -User: "Start VM web-server in namespace vms" -Agent: [vm-lifecycle-manager skill] -## VM Lifecycle Operation -| VM Name | `web-server` | Namespace | `vms` | Action | `start` | consumes resources | -Confirm: yes/no -User: "yes" -Agent: [vm_lifecycle(namespace="vms", name="web-server", action="start")] -## ✓ VM Start Successful -VM: `web-server` | Namespace: `vms` | RunStrategy: Always -Impact: Running, consuming resources. Access: virtctl console or SSH -``` - -### Example 2: Stop with User Reconsidering - -``` -User: "Stop the database VM in production namespace" -Agent: [vm-lifecycle-manager skill] -| VM | `database` | Namespace | `production` | Action | `stop` | graceful shutdown | -Confirm: yes/no -User: "wait, no, don't stop it" -Agent: Operation cancelled. VM 'database' in namespace 'production' remains running. -``` - -### Example 3: Restart VM - -``` -User: "Restart test-vm in namespace dev" -Agent: [vm-lifecycle-manager skill] -| VM | `test-vm` | Namespace | `dev` | Action | `restart` | brief interruption ~1-2min | -User: "yes" -Agent: [Composite restart operation] - 1. vm_lifecycle(namespace="dev", name="test-vm", action="stop") - 2. Verify stopped: resources_get → status.printableStatus="Stopped" - 3. Wait 5s - 4. vm_lifecycle(namespace="dev", name="test-vm", action="start") - 5. Verify started: resources_get → status.printableStatus="Running" -## ✓ VM Restart Successful -VM: `test-vm` | Namespace: `dev` | RunStrategy: Always -Impact: Running after stop+start. Brief interruption. Monitor app logs. -``` - -### Example 4: VM Already in Desired State - -``` -User: "Start web-server in namespace vms" -Agent: [vm-lifecycle-manager skill] - [vm_lifecycle(action="start")] -## ℹ️ VM Already Running -VM: `web-server` | Namespace: `vms` | Status: Running -Result: No action taken - VM already in desired state. -To restart: "Restart VM web-server in namespace vms" -``` - -### Example 5: Batch Operations - -``` -User: "Stop VMs web-01, web-02, web-03 in namespace production" -Agent: [vm-lifecycle-manager skill - batch mode] -## Batch Lifecycle Operation -Stopping 3 VMs in 'production': web-01, web-02, web-03 -Impact: All 3 VMs will shut down, services interrupted. -Confirm: yes/no -User: "yes" -Agent: [Executes vm_lifecycle for each VM sequentially] -## ✓ Batch Stop Successful -- web-01: Stopped -- web-02: Stopped -- web-03: Stopped -All VMs stopped. Resources freed. -``` diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/.ai-index/semantic-index.json deleted file mode 100644 index 2512c2a3..00000000 --- a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/.ai-index/semantic-index.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "index_version": "1.0.0", - "last_updated": "2026-02-17", - "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", - - "documents": [ - { - "file": "INDEX.md", - "title": "VM Troubleshooting Guide - Index", - "category": "navigation", - "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], - "use_cases": ["navigation", "tool-reference", "general-guidance"], - "token_estimate": 450, - "skills": ["all"] - }, - { - "file": "scheduling-errors.md", - "title": "VM Scheduling Errors", - "category": "scheduling", - "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], - "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], - "errors_covered": [ - "ErrorUnschedulable - Node Taints", - "ErrorUnschedulable - Insufficient Resources", - "ErrorUnschedulable - Node Selector Mismatch" - ], - "token_estimate": 820, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "storage-errors.md", - "title": "VM Storage Errors", - "category": "storage", - "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], - "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], - "errors_covered": [ - "ErrorDataVolumeNotReady - DataVolume Still Provisioning", - "ErrorDataVolumeNotReady - Storage Class Not Found", - "ErrorDataVolumeNotReady - Insufficient Storage Quota", - "ErrorPvcNotFound", - "Storage Deletion Failures", - "DataVolume Cloning Failures" - ], - "token_estimate": 1350, - "skills": ["vm-creator", "vm-delete", "vm-clone"] - }, - { - "file": "lifecycle-errors.md", - "title": "VM Lifecycle Errors", - "category": "lifecycle", - "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], - "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], - "errors_covered": [ - "VM Stuck in Terminating State", - "VM Won't Start (Non-Scheduling Issues)", - "VM Won't Stop" - ], - "token_estimate": 1150, - "skills": ["vm-delete", "vm-lifecycle-manager"] - }, - { - "file": "runtime-errors.md", - "title": "VM Runtime Errors", - "category": "runtime", - "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], - "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], - "errors_covered": [ - "CrashLoopBackOff" - ], - "token_estimate": 850, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "network-errors.md", - "title": "VM Network Errors", - "category": "networking", - "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], - "use_cases": ["vm-creation", "network-troubleshooting"], - "errors_covered": [ - "Network Attachment Failures" - ], - "token_estimate": 580, - "skills": ["vm-creator"] - } - ], - - "skill_to_docs_mapping": { - "vm-creator": { - "primary_docs": ["scheduling-errors.md", "storage-errors.md"], - "secondary_docs": ["runtime-errors.md", "network-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3650, - "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" - }, - "vm-delete": { - "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 2950, - "description": "VM deletion failures - stuck terminating, storage cleanup" - }, - "vm-clone": { - "primary_docs": ["storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 1800, - "description": "VM cloning failures - DataVolume cloning, storage provisioning" - }, - "vm-lifecycle-manager": { - "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], - "secondary_docs": ["runtime-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3270, - "description": "VM start/stop failures - lifecycle management, scheduling" - }, - "vm-inventory": { - "primary_docs": [], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 450, - "description": "General guidance for status interpretation" - } - }, - - "error_to_docs_mapping": { - "ErrorUnschedulable": "scheduling-errors.md", - "ErrorDataVolumeNotReady": "storage-errors.md", - "ErrorPvcNotFound": "storage-errors.md", - "Terminating": "lifecycle-errors.md", - "CrashLoopBackOff": "runtime-errors.md", - "NetworkAttachmentFailed": "network-errors.md" - }, - - "cross_references": { - "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], - "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], - "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], - "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], - "network-errors.md": ["scheduling-errors.md"] - }, - - "progressive_disclosure_strategy": { - "description": "Load only the minimum documentation needed for current task", - "pattern": "error_type → primary_doc → cross_references (if needed)", - "token_savings": "29% average reduction vs single-file (based on rh-sre data)", - "navigation_overhead_reduction": "85% vs single-file navigation" - } -} diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/INDEX.md deleted file mode 100644 index a2e76963..00000000 --- a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/INDEX.md +++ /dev/null @@ -1,332 +0,0 @@ ---- -title: VM Troubleshooting Guide - Index -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 - - title: OpenShift Virtualization - Virtual Machine Status - url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html - date_accessed: 2026-02-06 - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] -semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] -use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] -last_updated: 2026-02-17 ---- - -# VM Troubleshooting Guide - Index - -## Overview - -This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: - -- **Creation**: Scheduling failures, storage provisioning issues -- **Lifecycle**: Start/stop failures, stuck states -- **Deletion**: Resources stuck in Terminating, storage cleanup failures -- **Cloning**: DataVolume cloning errors, cross-namespace issues -- **Networking**: Secondary network attachment failures -- **Runtime**: CrashLoopBackOff, guest OS boot failures - -This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. - ---- - -## 🗂️ Troubleshooting Categories - -The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: - -### 1. [Scheduling Errors](scheduling-errors.md) -**When to use**: VM fails to schedule on any node - -**Errors covered**: -- ErrorUnschedulable - Node Taints -- ErrorUnschedulable - Insufficient Resources -- ErrorUnschedulable - Node Selector Mismatch - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 2. [Storage Errors](storage-errors.md) -**When to use**: VM has storage provisioning, deletion, or cloning issues - -**Errors covered**: -- ErrorDataVolumeNotReady (all 3 subsections) -- ErrorPvcNotFound -- Storage Deletion Failures -- DataVolume Cloning Failures - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### 3. [Lifecycle Errors](lifecycle-errors.md) -**When to use**: VM has start/stop/termination issues - -**Errors covered**: -- VM Stuck in Terminating State -- VM Won't Start (Non-Scheduling Issues) -- VM Won't Stop - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### 4. [Runtime Errors](runtime-errors.md) -**When to use**: VM repeatedly crashes or fails at runtime - -**Errors covered**: -- CrashLoopBackOff - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 5. [Network Errors](network-errors.md) -**When to use**: VM has secondary network attachment failures - -**Errors covered**: -- Network Attachment Failures (Multus, SR-IOV) - -**Skills that use this**: vm-create - ---- - -## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last - -### Critical Principle: MCP-First Approach - -**All diagnostic procedures in this guide follow the MCP-first pattern**: - -``` -1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) -2. ⚠️ IF FAILS: Ask user permission to use CLI command -3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval -``` - -### Why MCP Tools First? - -- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources -- **Consistency**: Same interface across all operations -- **Better Error Handling**: MCP tools return structured errors -- **Audit Trail**: MCP tool usage is logged and trackable - -### Command Pattern Examples - -Throughout this guide, you'll see diagnostic steps formatted like this: - -**✅ CORRECT Pattern (MCP First)**: - -```markdown -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Extract**: `.metadata.finalizers` from returned JSON - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` -``` - -### Available MCP Tools - -The `openshift-virtualization` MCP server provides these tools: - -**Resource Operations**: -- `resources_get` - Get specific resource (replaces `oc get `) -- `resources_list` - List resources (replaces `oc get `) -- `resources_delete` - Delete resource (replaces `oc delete`) -- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) - -**Pod Operations**: -- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) -- `pods_get` - Get pod details (replaces `oc get pod`) -- `pods_log` - Get pod logs (replaces `oc logs`) -- `pods_exec` - Execute in pod (replaces `oc exec`) -- `pods_delete` - Delete pod (replaces `oc delete pod`) -- `pods_top` - Pod resource usage (replaces `oc top pods`) - -**Events & Monitoring**: -- `events_list` - List events (replaces `oc get events`) -- `nodes_top` - Node resource usage (replaces `oc top nodes`) -- `nodes_log` - Node logs (replaces `oc adm node-logs`) -- `nodes_stats_summary` - Detailed node stats - -**VM Operations** (KubeVirt toolset): -- `vm_create` - Create VMs -- `vm_lifecycle` - Start/stop/restart VMs - -**Namespaces**: -- `namespaces_list` - List namespaces (replaces `oc get namespaces`) -- `projects_list` - List OpenShift projects (replaces `oc get projects`) - -### When CLI Commands Are Required - -Some operations have **NO MCP equivalent** and require CLI: - -- `virtctl` commands (console, VNC access) -- `oc debug node` (node debugging) -- `oc auth can-i` (permission checks) -- `oc adm taint` (node taint management) -- Complex JSON patch operations - -For these, the guide will note: **"CLI Only - No MCP equivalent"** - -### Quick Reference: CLI → MCP Mapping - -| CLI Command | MCP Tool Equivalent | -|-------------|---------------------| -| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | -| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | -| `oc logs -n ` | `pods_log` with name="", namespace="" | -| `oc get events -n ` | `events_list` with namespace="" | -| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | - -**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. - -### How to Read Diagnostic Sections - -Each error section includes: -1. **Symptom** - What you observe -2. **Description** - What's happening -3. **Possible Causes** - Why it's happening -4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback -5. **Solutions** - **MCP tools first**, then CLI fallback -6. **Verification** - **MCP tools first**, then CLI fallback - -**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. - ---- - -## 🔍 Quick Navigation by Skill - -**vm-create**: -- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics -- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning -- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff -- [Network Errors](network-errors.md) - Network attachment failures - -**vm-delete**: -- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state -- [Storage Errors](storage-errors.md) - Storage deletion failures - -**vm-clone**: -- [Storage Errors](storage-errors.md) - DataVolume cloning failures - -**vm-lifecycle-manager**: -- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop -- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling - -**vm-inventory**: -- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed - ---- - -## 📊 Documentation Coverage & Maintenance - -### Current Coverage - -This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: - -- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) -- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures -- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop -- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) -- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems - -**Total errors documented**: 12 error types across 6 categories - ---- - -### Encountering Undocumented Errors - -#### For AI Agents (Claude Code) - -If you encounter an error **not documented** in the categories above: - -1. **Report to user** with all available details (error message, affected resources, namespace) -2. **Provide best-effort diagnostics** using MCP tools: - - `resources_get` to inspect resource status - - `pods_log` to check virt-launcher or compute container logs - - `events_list` to view Kubernetes events -3. **Suggest documentation update**: - ``` - ⚠️ This error is not yet documented in the troubleshooting guide. - - **Error**: - **Affected resource**: / - - I recommend adding this error to the troubleshooting guide: - - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file - - If it's a new error class → Create new category file in docs/troubleshooting/ - - Would you like me to help document this error for future reference? - ``` - -4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs - ---- - -#### For Human Contributors - -To document a new error: - -1. **Determine the category**: - - Scheduling issue → `scheduling-errors.md` - - Storage problem → `storage-errors.md` - - Lifecycle/start/stop → `lifecycle-errors.md` - - Runtime crash → `runtime-errors.md` - - Network issue → `network-errors.md` - - New error class → Create new category file (e.g., `security-errors.md`) - -2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): - - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback - - **Solutions**: MCP Tool approach → Implementation → CLI Fallback - - **Verification**: MCP Tool checks → Expected results → CLI Fallback - -3. **Update semantic index**: - - Edit `.ai-index/semantic-index.json` - - Add error to `error_to_docs_mapping` - - Update relevant skill's `primary_docs` or `secondary_docs` - - Increment token estimates if needed - -4. **Reference in skill files**: - - Update skill's Reference Documentation section - - Ensure skill's Document Consultation steps point to the right category - -5. **Update this INDEX.md**: - - Add error to the appropriate category section (lines 50-120) - - Update "Current Coverage" list above - ---- - -## 📚 Additional Resources - -- [rh-virt README](../../README.md) - MCP server setup and skill documentation -- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources -- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/lifecycle-errors.md deleted file mode 100644 index 6167fcc8..00000000 --- a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/lifecycle-errors.md +++ /dev/null @@ -1,869 +0,0 @@ ---- -title: VM Lifecycle Errors -category: kubevirt -sources: - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 -tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] -semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] -use_cases: [vm-deletion, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Lifecycle Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM lifecycle issues including start/stop failures and stuck termination states. - -**When to use this document**: -- VM stuck in `Terminating` state -- VM won't start (runStrategy is Always but VM never reaches Running) -- VM won't stop (runStrategy is Halted but VM never reaches Stopped) - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### VM Stuck in Terminating State - -**Symptom**: VM shows status `Terminating` but deletion never completes - -**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). - -**Possible Causes**: -- Finalizers blocking deletion -- PVC/DataVolume still attached and preventing cleanup -- VirtualMachineInstance (VMI) not terminating properly -- Custom controllers or operators blocking deletion -- Stuck virt-launcher pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check finalizers on the VM**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` - -**2. Check if VMI still exists**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name containing "virt-launcher-". - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check events for deletion issues**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results where `involvedObject.name` == "" and sort by timestamp. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' -``` - -**5. Check VM deletion timestamp**: - -Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. - -If present, VM is in deletion process. If null, VM is not being deleted. - -**Common Finalizer Patterns**: -- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer -- `foregroundDeletion` - Waits for dependent objects to be deleted -- Custom finalizers from operators - -**Solutions** (Use MCP Tools First): - -1. **Wait for dependent resources** (safest approach): - - Use `resources_list` to check if PVCs, DataVolumes are still deleting - - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) - -2. **Force delete VMI** (if VMI is stuck): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable - requires explicit user permission): - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -3. **Force delete virt-launcher pod** (if pod is stuck): - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - # Find the virt-launcher pod name first - oc get pods -n | grep virt-launcher- - - # Then delete it - oc delete pod virt-launcher--xxx -n --force --grace-period=0 - ``` - -4. **Remove finalizers** (⚠️ dangerous - use only as last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` - 2. Remove items from `.metadata.finalizers` array - 3. Update VM using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch not easily done via MCP): - ```bash - # This operation is complex for MCP - may need CLI - oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Check for protection labels** (vm-delete skill specific): - - Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. - - If value is "true", the vm-delete skill refuses deletion (this is expected behavior). - -**Verification**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error if deletion successful. - -**CLI Fallback**: -```bash -oc get vm -n -# Should return: Error from server (NotFound) -``` - ---- - - ---- - -### VM Won't Start (Non-Scheduling Issues) - -**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state - -**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. - -**Possible Causes**: -- Guest OS kernel panic or boot failure -- Cloud-init configuration errors -- Missing or corrupted disk image -- Insufficient memory for guest OS to boot -- QEMU/KVM virtualization errors -- VirtualMachineInstance (VMI) creation failures -- virt-launcher pod crashes - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VMI (VirtualMachineInstance) status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and overall VMI state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**2. Check VMI conditions for errors**: - -Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. View virt-launcher pod logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -First, get pod name from step 3, then: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -Look for QEMU errors, memory allocation failures, device errors. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**5. Check virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. - -Alternatively, use `pods_get` to get full pod details: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**6. Access VM console to see guest OS boot messages**: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -Look for kernel panic, initramfs errors, filesystem errors. - -**7. Check VM events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep -A 20 "Events:" -``` - -**Common Error Patterns**: - -1. **Guest OS Boot Failure**: - - Console shows kernel panic - - Guest hangs at GRUB or boot loader - - Cloud-init errors during first boot - -2. **Insufficient Memory**: - - Guest OS kills processes due to OOM - - VMI logs show memory allocation errors - -3. **QEMU Crashes**: - - virt-launcher logs show QEMU segfaults - - VMI repeatedly restarts - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for kernel panic, initramfs errors, filesystem errors. - -2. **Check virt-launcher pod logs for QEMU errors**: - - Use `pods_log` from diagnostic step 4 to view logs. - - Look for: - - "qemu-system-x86_64: ..." errors - - Memory allocation failures - - Device errors - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx - ``` - -3. **Increase memory if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check VMI memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -4. **Verify disk image integrity**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check PVC is bound: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - If using container disk, verify image exists and is accessible (check VMI spec). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get pvc -n - ``` - -5. **Check cloud-init configuration** (if applicable): - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. - - Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq - ``` - -6. **Restart VMI** (soft reset): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete VMI (VM controller will recreate it): - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for new VMI to start (use `resources_get` to check status). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n -w - ``` - -7. **Check virtualization extensions** (KVM): - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - - # In debug shell: - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should eventually return `Running`). - -Check VMI is running: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should show `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should eventually return: Running - -oc get vmi -n -# Should show: Running -``` - ---- - -### VM Won't Stop - -**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state - -**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. - -**Possible Causes**: -- Guest OS not responding to ACPI shutdown signal -- virt-launcher pod stuck and not terminating -- VirtualMachineInstance (VMI) deletion blocked by finalizers -- Guest shutdown scripts hanging -- Filesystem sync issues in guest OS - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (might show `Stopping`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**2. Check VMI status and deletion timestamp**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check VMI events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vmi -n | grep -A 10 "Events:" -``` - -**5. Check VMI finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.metadata.finalizers}' -``` - -**6. Check if guest is responsive** (if VMI still exists): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Patterns**: -- VMI shows `deletionTimestamp` but never actually deletes -- virt-launcher pod in `Terminating` state -- VM runStrategy is `Halted` but printableStatus shows `Stopping` - -**Solutions** (Use MCP Tools First): - -1. **Wait for graceful shutdown** (default: 30 seconds): - - Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. - -2. **Force stop by deleting VMI**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - This is the standard way to force-stop a VM. - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n - # Should return: Error from server (NotFound) - ``` - -3. **Force delete VMI with grace period** (if VMI won't delete): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -4. **Force delete virt-launcher pod**: - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find the pod using `pods_list_in_namespace` from diagnostic step 3. - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - ⚠️ **Note**: For force deletion with grace period, use CLI fallback. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') - oc delete pod $POD_NAME -n --force --grace-period=0 - ``` - -5. **Remove VMI finalizers** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VMI using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update VMI using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Patch VM runStrategy directly** (ensure consistency): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` (diagnostic step 1) - 2. Set `.spec.runStrategy` to `"Halted"` - 3. Update VM using `resources_create_or_update` with modified JSON - - **CLI Fallback** (merge patch easier via CLI): - Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Stopped` or `Halted`). - -Verify VMI is gone: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Verify virt-launcher pod is gone: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for pods matching `virt-launcher-`. Should return no results. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Stopped or Halted - -oc get vmi -n -# Should return: Error from server (NotFound) - -oc get pods -n | grep virt-launcher- -# Should return: No resources found -``` - -**Prevention**: -- Ensure guest OS has ACPI support enabled -- Use proper shutdown commands in guest OS -- Avoid forceful stops unless necessary (can corrupt guest filesystem) - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/network-errors.md deleted file mode 100644 index 82fa313c..00000000 --- a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/network-errors.md +++ /dev/null @@ -1,429 +0,0 @@ ---- -title: VM Network Errors -category: kubevirt -sources: - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] -semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] -use_cases: [vm-creation, network-troubleshooting] -related_docs: [INDEX.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Network Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. - -**When to use this document**: -- VM created successfully but secondary networks not attached -- NetworkAttachmentDefinition not found errors -- Multus CNI failures -- SR-IOV device attachment issues - -**Skills that use this**: vm-create - ---- - -### Network Attachment Failures - -**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working - -**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). - -**Possible Causes**: -- NetworkAttachmentDefinition doesn't exist in the namespace -- Multus CNI not installed or not configured on cluster -- Namespace mismatch (NAD in different namespace than VM) -- Interface name conflicts in VM spec -- Bridge/network configuration errors in NAD -- SR-IOV device not available (if using SR-IOV) - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List NetworkAttachmentDefinitions in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "" -} -``` - -Review `.items[].metadata.name` for available NADs. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -n -``` - -**2. List NetworkAttachmentDefinitions in all namespaces**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition" -} -``` - -Omit `namespace` parameter to list across all namespaces. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -A -``` - -**3. Check specific NetworkAttachmentDefinition**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "", - "name": "" -} -``` - -Review `.spec.config` for CNI configuration. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definition -n -o yaml -``` - -**4. Check VM network configuration**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.networks` to see network references. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq -``` - -**5. Check VM domain interfaces**: - -Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq -``` - -**6. Check virt-launcher pod network annotations** (shows actual attachments): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**7. Check Multus is installed**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "openshift-multus" -} -``` - -Should show Multus CNI pods running. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n openshift-multus -``` - -**8. Check for errors in virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n -``` - -**Common Error Messages**: -- `"network-attachment-definition not found"` - NAD doesn't exist in namespace -- `"multus CNI not configured"` - Multus not installed or misconfigured -- `"interface name conflict"` - Duplicate interface names in VM spec -- `"failed to add network"` - CNI plugin error (check NAD config) -- `"no available devices"` - SR-IOV device not available (if using SR-IOV) - -**Solutions** (Use MCP Tools First): - -1. **Verify NetworkAttachmentDefinition exists in correct namespace**: - - Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. - - If NAD is in different namespace, copy it to VM namespace: - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get NAD from source namespace using `resources_get` - 2. Modify `.metadata.namespace` to target namespace - 3. Create NAD in target namespace using `resources_create_or_update` - - **CLI Fallback** (stream processing easier via CLI): - Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" - ```bash - oc get network-attachment-definition -n -o yaml | \ - sed "s/namespace: /namespace: /" | \ - oc apply -f - - ``` - -2. **Create missing NetworkAttachmentDefinition**: - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - Example: Linux bridge network - - **Parameters**: - ```json - { - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "metadata": { - "name": "vlan100", - "namespace": "" - }, - "spec": { - "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" - ```bash - cat < - spec: - config: '{ - "cniVersion": "0.3.1", - "type": "bridge", - "bridge": "br1", - "vlan": 100, - "ipam": { - "type": "host-local", - "subnet": "192.168.100.0/24" - } - }' - EOF - ``` - -3. **Check Multus CNI installation**: - - Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. - - To check cluster network operator: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "config.openshift.io/v1", - "kind": "ClusterOperator" - } - ``` - - Filter for `network` operator. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n openshift-multus - oc get clusteroperators network - ``` - -4. **Fix interface name conflicts** (if VM has duplicate names): - - Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. - - Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. - - **CLI Fallback** (interactive edit easier via CLI): - Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' - oc edit vm -n - ``` - -5. **Validate NAD configuration syntax**: - - Use `resources_get` from diagnostic step 3, extract `.spec.config`. - - Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get network-attachment-definition -n -o jsonpath='{.spec.config}' - ``` - -6. **Check SR-IOV device availability** (if using SR-IOV networks): - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - List SR-IOV network node policies: - - **Parameters**: - ```json - { - "apiVersion": "sriovnetwork.openshift.io/v1", - "kind": "SriovNetworkNodePolicy", - "namespace": "openshift-sriov-network-operator" - } - ``` - - Check SR-IOV device plugin pods: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "openshift-sriov-network-operator" - } - ``` - - Filter for pods with "device-plugin" in name. - - Check available SR-IOV devices on node: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "Node", - "name": "" - } - ``` - - Review `.status.allocatable` for SR-IOV resources. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get sriovnetworknodepolicy -n openshift-sriov-network-operator - oc get pods -n openshift-sriov-network-operator | grep device-plugin - oc describe node | grep -A 10 "Allocatable:" - ``` - -7. **Recreate VM with corrected network configuration** (if needed): - - If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -After remediation, check virt-launcher pod network status: - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -Should show all attached networks with status. Example output: -```json -[ - { - "name": "openshift-sdn", - "interface": "eth0", - "ips": ["10.128.2.10"], - "default": true - }, - { - "name": "vlan100", - "interface": "net1", - "ips": ["192.168.100.5"] - } -] -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**Check from inside VM** (via console): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# In guest OS: -ip addr show -# Should show all network interfaces (eth0, net1, etc.) -``` - -**Common Network Types**: -- **Linux Bridge**: Layer 2 bridge for VLAN networks -- **SR-IOV**: High-performance direct device assignment -- **macvlan**: MAC-based VLAN for container networks -- **OVN-Kubernetes**: OpenShift native overlay network - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/runtime-errors.md deleted file mode 100644 index 804d3cbc..00000000 --- a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/runtime-errors.md +++ /dev/null @@ -1,616 +0,0 @@ ---- -title: VM Runtime Errors -category: kubevirt -sources: - - title: Kubernetes Pod Lifecycle - url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ - date_accessed: 2026-02-17 -tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] -semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] -use_cases: [vm-creation, vm-lifecycle, diagnostics] -related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Runtime Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. - -**When to use this document**: -- VM or virt-launcher pod shows `CrashLoopBackOff` status -- virt-launcher pod repeatedly restarting -- Guest OS kernel panics on boot - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### CrashLoopBackOff - -**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting - -**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. - -**Possible Causes**: -- Guest OS kernel panic on boot -- Insufficient resources (memory/CPU) for guest OS -- Corrupted disk image or filesystem -- QEMU/libvirt crashes due to configuration errors -- Missing or incompatible device drivers in guest -- Resource limits too low for virt-launcher pod -- Virtualization features (KVM) not available on node - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check virt-launcher pod restart count**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# Look at RESTARTS column -``` - -**2. View recent crash logs** (previous container instance): - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "previous": true, - "tail": 100 -} -``` - -Look for QEMU errors, kernel panics, or segfaults. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n virt-launcher--xxx --previous -``` - -**3. Check current virt-launcher logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. - -**CLI Fallback** (if MCP unavailable or all containers needed): -```bash -oc logs -n virt-launcher--xxx --all-containers -``` - -**4. Check VMI conditions for crash details**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Extract `.status.conditions` for crash details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**5. Check pod events for crash reasons**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" -``` - -**6. Check pod resource limits**: - -**MCP Tool**: `pods_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -Extract `.spec.containers[0].resources` for resource limits. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq -``` - -**7. Check node kubelet logs for OOM kills**: - -⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -oc adm node-logs -u kubelet | grep -i oom -``` - -**8. Access guest console** (if VM briefly starts): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Crash Patterns**: - -1. **Guest Kernel Panic**: - - Console logs show kernel panic messages - - Guest crashes immediately after boot - - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" - -2. **OOM (Out of Memory)**: - - Pod killed with reason: `OOMKilled` - - Guest runs out of memory during boot or operation - - virt-launcher logs show memory allocation failures - -3. **QEMU Crash**: - - virt-launcher logs show QEMU segmentation fault - - Symptoms: "qemu-system-x86_64: terminated by signal" - - Configuration incompatibility or QEMU bug - -4. **Disk Image Corruption**: - - Guest cannot boot from disk - - Filesystem errors in guest console - - DataVolume import failed - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for kernel panic or boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for: - - Kernel panic messages - - Initramfs errors - - Filesystem mounting failures - - Missing device errors - -2. **Review virt-launcher crash logs**: - - Use `pods_log` with `previous: true` from diagnostic step 2. - - Look for: - - QEMU command line errors - - Device initialization failures - - Memory allocation errors - - Signal termination (SIGSEGV, SIGABRT) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx --previous - ``` - -3. **Check for OOM (Out of Memory) kills**: - - **MCP Tool**: `pods_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - Extract `.status.containerStatuses[0].lastState.terminated.reason`. - - If returns `"OOMKilled"`: - - Option 1: Increase virt-launcher memory limits - - Option 2: Decrease guest memory allocation - - Option 3: Use smaller instance type - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' - ``` - -4. **Increase resources if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check current memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -5. **Verify disk image integrity**: - - **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters for list**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" - } - ``` - - **Parameters for specific DV**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check `.status.phase` (should be `Succeeded`). - - If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get datavolume -n -o jsonpath='{.status.phase}' - ``` - -6. **Check virtualization (KVM) availability**: - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -7. **Simplify VM configuration** (eliminate variables): - - Try creating minimal VM using vm-create skill with: - - Small instance type - - No secondary networks - - Simple container disk (e.g., Fedora) - - No cloud-init - - If minimal VM works, add features back one by one. - -8. **Recreate VM with different workload** (test disk image): - - If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) - -After remediation, check pod restart count stops increasing: - -**Parameters for pods**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). - -Check VM reaches Running state: - -**Parameters for VM**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# RESTARTS should stabilize - -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Running -``` - -Verify guest is responsive: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# Should show login prompt or OS console -``` - -**Advanced Debugging**: - -**MCP Tool**: `pods_exec` (from openshift-virtualization) - -Check libvirt domain XML: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["virsh", "dumpxml", "1"] -} -``` - -Check QEMU process: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["ps", "aux"] -} -``` - -Filter output for "qemu" process. - -**CLI Fallback** (if MCP unavailable): -```bash -oc exec -n virt-launcher--xxx -- virsh dumpxml 1 -oc exec -n virt-launcher--xxx -- ps aux | grep qemu -``` - -**Prevention**: -- Start with minimal VM configuration and add complexity gradually -- Use recommended instance types for your workload -- Test disk images before deploying to production -- Ensure nodes have adequate resources and KVM support -- Monitor resource usage and set appropriate limits - ---- - -## Workaround Patterns for MCP Tool Limitations - -### General Pattern: Diagnose → Propose → Confirm → Execute - -When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): - -**1. Diagnose** the root cause using Kubernetes commands -```bash -oc describe vm -n -oc get events -n --field-selector involvedObject.name= -``` - -**2. Propose** a manual workaround to the user with clear explanation -- Explain why the issue occurred -- Show what will be changed -- List alternative options - -**3. Confirm** with user before executing (Human-in-the-Loop) -- Wait for explicit "yes" or "apply workaround" -- Never auto-execute modifications - -**4. Execute** the workaround using oc/kubectl -```bash -oc patch vm -n --type=merge -p '...' -``` - -**5. Verify** the fix was successful -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**6. Document** the limitation and suggest filing enhancement request -- Note this is temporary until MCP tool is enhanced -- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues - -### Example: Adding Tolerations Workaround - -**Diagnostic Output**: -```markdown -## ⚠️ VM Scheduling Issue Detected - -**Root Cause**: Node taints prevent VM scheduling - -**Details**: -- Found 3 nodes with taint: `virtualization=true:NoSchedule` -- VM spec does not include matching tolerations -- This prevents VM scheduling on virtualization-dedicated nodes -``` - -**Proposed Workaround**: -```bash -oc patch vm web-server -n vms --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' -``` - -**User Confirmation Required**: -``` -How would you like to proceed? -- "apply workaround" - I'll patch the VM with tolerations -- "manual" - I'll provide instructions for you to apply manually -- "cancel" - Delete the VM and abort creation -``` - -**After Execution**: -```markdown -## ✓ Workaround Applied Successfully - -**Action**: Added tolerations for taint `virtualization=true:NoSchedule` -**New Status**: Stopped (VM can now be scheduled) - -**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. -Future VMs in this cluster will need the same fix until the tool is enhanced. -``` - ---- - -## VM Status Reference - -### Status Values - -| Status | Meaning | Action Required | -|--------|---------|-----------------| -| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | -| `Running` | VM is running | Normal | -| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | -| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | -| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | -| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | -| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | -| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | -| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | -| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | - -### Checking VM Status - -```bash -# Get printable status -oc get vm -n -o jsonpath='{.status.printableStatus}' - -# Get detailed status and conditions -oc get vm -n -o jsonpath='{.status}' | jq - -# Watch status changes in real-time -oc get vm -n -w -``` - ---- - -## Best Practices for Agents - -When implementing diagnostic workflows: - -1. **Always verify VM status** after creation (wait 5-10 seconds first) -2. **Consult this document** when encountering error status values -3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) -4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) -5. **Respect human-in-the-loop** for all VM modifications -6. **Document temporary workarounds** and their limitations clearly -7. **Suggest filing issues** for missing MCP tool features - -### Document Consultation Pattern - -```markdown -**Document Consultation** (REQUIRED): -1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes -2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." -``` - ---- - -## Known MCP Tool Limitations - -### vm_create tool - -**Currently Supported**: -- ✓ Namespace, name (required) -- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) -- ✓ Size hints (small, medium, large) -- ✓ Storage size -- ✓ Autostart flag -- ✓ Networks (Multus NetworkAttachmentDefinitions) -- ✓ Performance family (u1, o1, c1, m1) -- ✓ Instance type, preference - -**Not Currently Supported** (requires workarounds): -- ✗ Tolerations (for node taints) -- ✗ Node selectors -- ✗ Affinity/anti-affinity rules -- ✗ Resource requests/limits (beyond instance type) -- ✗ Custom labels/annotations -- ✗ SSH keys injection -- ✗ Cloud-init user data - -**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. - -**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues - ---- - -## Additional Resources - -- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) -- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) -- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) -- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) - ---- - -[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/scheduling-errors.md deleted file mode 100644 index e47c8c4f..00000000 --- a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/scheduling-errors.md +++ /dev/null @@ -1,417 +0,0 @@ ---- -title: VM Scheduling Errors -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 -tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] -semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] -use_cases: [vm-creation, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Scheduling Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**When to use this document**: -- VM shows status `ErrorUnschedulable` after creation or start attempt -- VM events mention scheduling failures, taints, resources, or node selectors - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -## ErrorUnschedulable - -**Symptom**: VM shows status `ErrorUnschedulable` after creation - -**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**Possible Causes**: - -### 1. Node Taints (Most Common) - -Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM events for scheduling failures**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `involvedObject.name` == "" and look for messages like: -- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= -``` - -**2. Check node taints in the cluster**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' -``` - -**Common Taint Patterns**: -- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule -- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes -- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads - -**Solution - Add Tolerations to VM**: - -The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. - -**Workaround (post-creation using MCP Tools)**: - -**Step 1**: Get current VM spec - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Step 2**: Modify the returned JSON to add tolerations - -Add to `.spec.template.spec.tolerations`: -```json -{ - "tolerations": [ - { - "key": "virtualization", - "operator": "Equal", - "value": "true", - "effect": "NoSchedule" - } - ] -} -``` - -**Step 3**: Update VM with modified spec - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters**: -```json -{ - "resource": "" -} -``` - -Pass the complete modified VM resource as YAML or JSON string. - -**Step 4**: Verify tolerations were added - -Use `resources_get` again and check `.spec.template.spec.tolerations` in response. - -**Step 5**: Check if VM status improved - -Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. - -**CLI Fallback** (if MCP patch is too complex): -```bash -# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' - -# Verify tolerations -oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq - -# Check status -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**Example - Multiple Tolerations**: -```bash -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" - - key: "dedicated" - operator: "Equal" - value: "virt-workloads" - effect: "NoSchedule" -' -``` - -**Toleration Operators**: -- `Equal` - Key and value must match exactly -- `Exists` - Only key must exist (ignores value) - -**Toleration Effects**: -- `NoSchedule` - Don't schedule new pods (existing pods continue) -- `PreferNoSchedule` - Avoid scheduling if possible -- `NoExecute` - Don't schedule AND evict existing pods - -**Alternative Solutions**: -1. **Remove node taints** (if you have cluster-admin access): - ```bash - oc adm taint nodes virtualization=true:NoSchedule- - ``` - -2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits - -3. **File enhancement request** - Request tolerations support in openshift-mcp-server: - https://github.com/openshift/openshift-mcp-server/issues - ---- - -### 2. Insufficient Resources - -Not enough CPU, memory, or storage available on any node. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM resource requests**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' -``` - -**2. Check node resource availability**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. - -Alternatively, use `nodes_top` MCP tool for current resource usage. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe nodes | grep -A 5 "Allocated resources" -``` - -**3. Look for VM events mentioning "Insufficient"**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep "Insufficient" -``` - -**Example Event**: -``` -0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. -``` - -**Solutions** (Use MCP Tools First): - -1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) -2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill -3. **Delete unused VMs** - Use vm-delete skill to free up resources -4. **Check resource quotas**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters for quota**: - ```json - { - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" - } - ``` - - **Parameters for limit range**: - ```json - { - "apiVersion": "v1", - "kind": "LimitRange", - "namespace": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - oc describe limitrange -n - ``` - ---- - -### 3. Node Selector Mismatch - -VM requires specific node labels that don't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM node selector requirements**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.nodeSelector` to see required node labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' -``` - -**2. List available node labels**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.metadata.labels` for available labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes --show-labels -``` - -**3. Check if any nodes match the selector**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node", - "labelSelector": "=" -} -``` - -Should return at least one node with matching labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -l = -``` - -**Solutions** (Use MCP Tools First): - -**Option 1: Remove node selector from VM** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get current VM using `resources_get` (diagnostic step 1) -2. Remove `.spec.template.spec.nodeSelector` field -3. Update VM using `resources_create_or_update` with modified JSON - -**CLI Fallback** (JSON patch easier via CLI): -Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" -```bash -oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' -``` - -**Option 2: Add label to nodes** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get node using `resources_get` -2. Add label to `.metadata.labels` -3. Update node using `resources_create_or_update` - -⚠️ **Note**: Node labeling typically requires cluster admin privileges. - -**CLI Fallback** (simpler via CLI): -Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" -```bash -oc label node = -``` - ---- - -[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/storage-errors.md deleted file mode 100644 index 16ba584d..00000000 --- a/evaluation/with_skills/rh-virt__vm-rebalance/environment/docs/troubleshooting/storage-errors.md +++ /dev/null @@ -1,1011 +0,0 @@ ---- -title: VM Storage Errors -category: kubevirt -sources: - - title: KubeVirt CDI - DataVolumes - url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ - date_accessed: 2026-02-17 - - title: Kubernetes Persistent Volumes - url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ - date_accessed: 2026-02-17 -tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] -semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] -use_cases: [vm-creation, vm-deletion, vm-cloning] -related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] -last_updated: 2026-02-17 ---- - -# VM Storage Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. - -**When to use this document**: -- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` -- Storage deletion fails after VM deletion -- DataVolume cloning operations fail -- PVC provisioning issues - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### ErrorDataVolumeNotReady - -**Symptom**: VM shows status `ErrorDataVolumeNotReady` - -**Description**: The DataVolume (persistent storage) backing the VM is not ready. - -**Possible Causes**: - -#### 1. DataVolume Still Provisioning - -Storage provisioning takes time, especially for large disks or when importing images. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Get detailed DataVolume information**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` and `.status.conditions` for provisioning details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o json -``` - -**3. Check PVC (PersistentVolumeClaim) bound status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Check `.status.phase` for each PVC (should be `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. - -#### 2. Storage Class Not Found - -The requested storage class doesn't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List available storage classes**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" -} -``` - -Review the list of available storage classes (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -``` - -**2. Check DataVolume's requested storage class**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.spec.pvc.storageClassName` in the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' -``` - -**Solution**: -1. Use a valid storage class from the cluster -2. Recreate VM with correct storage class parameter - -#### 3. Insufficient Storage Quota - -Namespace has insufficient storage quota to provision the PVC. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check resource quotas**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**2. Check storage usage**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase -``` - -**Solution**: -1. Request quota increase from cluster admin -2. Delete unused PVCs to free quota -3. Reduce VM storage size - ---- - -### ErrorPvcNotFound - -**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List PVCs in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review the list of available PVCs (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check VM's PVC references**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' -``` - -**Solution**: -- Wait for DataVolume to create the PVC -- Manually create missing PVC -- Fix VM spec to reference correct PVC name - ---- - - ---- - -### Storage Deletion Failures - -**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace - -**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. - -**Possible Causes**: -- PVC still bound to active PersistentVolume with `Retain` policy -- DataVolume still being referenced by another resource -- CDI (Containerized Data Importer) controller issues -- Storage class retention policy preventing deletion -- Finalizers on PVC/DataVolume blocking cleanup -- PVC still mounted by a pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check PVC status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check specific PVC phase**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Released` or `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o jsonpath='{.status.phase}' -``` - -**3. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**4. Check what's using the PVC**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' -``` - -**5. Check PVC finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. - -**6. Check DataVolume finalizers**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.metadata.finalizers}' -``` - -**7. Check PV reclaim policy**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume" -} -``` - -Filter results for PV where `.spec.claimRef.name` matches ``. - -To get specific PV policy: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume", - "name": "" -} -``` - -Check `.spec.persistentVolumeReclaimPolicy`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pv | grep -oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' -``` - -**Common Finalizer Patterns**: -- `kubernetes.io/pvc-protection` - Protects PVC while in use -- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer - -**Solutions** (Use MCP Tools First): - -1. **Delete DataVolume first, then PVC**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete DataVolume first (often blocks PVC deletion): - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Wait a few seconds, then delete PVC: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete datavolume -n - oc delete pvc -n - ``` - -2. **Check for pods still using PVC**: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "" - } - ``` - - Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. - - Then delete the pods using `pods_delete`: - - **Parameters**: - ```json - { - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' - oc delete pod -n - ``` - -3. **Force delete PVC** (if safe to do so): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete pvc -n --grace-period=0 --force - ``` - -4. **Remove finalizers from PVC** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PVC using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update PVC using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Remove finalizers from DataVolume** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current DataVolume using `resources_get` (diagnostic step 6) - 2. Remove items from `.metadata.finalizers` array - 3. Update DataVolume using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Change PV reclaim policy** (if PV has Retain policy): - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PV policy using `resources_get` (diagnostic step 7) - 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` - 3. Update PV using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" - ```bash - oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' - oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' - ``` - -**Storage Quota Check** (Use MCP Tools First): - -After deletion, verify storage quota is freed: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters for quota check**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.used` to verify storage quota is freed. - -**Parameters for PVC verification**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -oc get pvc -n -``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Confirm PVC is deleted: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Confirm DataVolume is deleted: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -# Should return: Error from server (NotFound) - -oc get datavolume -n -# Should return: Error from server (NotFound) -``` - ---- - -### DataVolume Cloning Failures - -**Symptom**: VM clone created successfully but DataVolume clone operation fails - -**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. - -**Possible Causes**: -- CSI driver doesn't support volume cloning -- Source PVC storage class incompatible with cloning -- Cross-namespace cloning not permitted by storage backend -- Insufficient storage quota in target namespace -- Source PVC not in `Bound` state -- Storage class doesn't have volume cloning enabled -- CDI (Containerized Data Importer) controller issues - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume clone status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and `.metadata.name`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Check DataVolume events for errors**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe datavolume -n -``` - -**3. Check DataVolume phase**: - -Use `resources_get` from step 1, extract `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -``` - -**4. Check if storage class supports cloning**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass", - "name": "" -} -``` - -Review the full YAML output for cloning-related configurations. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -o yaml | grep -A 5 -i clone -``` - -**5. Check CSI driver capabilities**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "CSIDriver" -} -``` - -Review `.items[].metadata.name` for available CSI drivers. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get csidriver -``` - -**6. Check source PVC status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Bound` for cloning to work). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**7. Check target namespace storage quota**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**8. Check CDI controller logs**: - -**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) - -First, list pods in openshift-cnv namespace: - -**Parameters for pods_list_in_namespace**: -```json -{ - "namespace": "openshift-cnv", - "labelSelector": "app.kubernetes.io/component=cdi-deployment" -} -``` - -Then get logs using `pods_log`: - -**Parameters**: -```json -{ - "namespace": "openshift-cnv", - "name": "", - "tail": 100 -} -``` - -**CLI Fallback** (if MCP unavailable or easier via CLI): -```bash -oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') -``` - -**Common Error Messages**: -- `"volume cloning is not supported"` - CSI driver lacks clone capability -- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage -- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace -- `"insufficient quota"` - Target namespace lacks storage quota -- `"source PVC not bound"` - Source PVC must be in Bound state for cloning -- `"StorageClass does not support cloning"` - Storage class configuration issue - -**Solutions** (Use MCP Tools First): - -1. **Check storage class clone support**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" - } - ``` - - For each storage class in `.items[]`, check: - - `.metadata.name` (storage class name) - - `.provisioner` (CSI driver) - - Storage classes using CSI drivers typically support cloning. Look for provisioners like: - - `csi.ovirt.org` (oVirt CSI) - - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) - - `ebs.csi.aws.com` (AWS EBS CSI) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner - ``` - -2. **Verify source PVC is bound**: - - Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pvc -n -o jsonpath='{.status.phase}' - ``` - -3. **Check target namespace quota**: - - Use `resources_list` from diagnostic step 7 to check quota. - - If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - ``` - -4. **Use snapshot-based cloning** (alternative method): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Step 1**: Create VolumeSnapshot of source PVC - - **Parameters**: - ```json - { - "apiVersion": "snapshot.storage.k8s.io/v1", - "kind": "VolumeSnapshot", - "metadata": { - "name": "-snapshot", - "namespace": "" - }, - "spec": { - "source": { - "persistentVolumeClaimName": "" - } - } - } - ``` - - **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) - - **Step 3**: Create new DataVolume from snapshot - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "metadata": { - "name": "-rootdisk", - "namespace": "" - }, - "spec": { - "source": { - "snapshot": { - "name": "-snapshot", - "namespace": "" - } - }, - "storage": { - "resources": { - "requests": { - "storage": "50Gi" - } - }, - "storageClassName": "" - } - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" - ```bash - cat <-snapshot - namespace: - spec: - source: - persistentVolumeClaimName: - EOF - - oc get volumesnapshot -snapshot -n - - cat <-rootdisk - namespace: - spec: - source: - snapshot: - name: -snapshot - namespace: - storage: - resources: - requests: - storage: 50Gi - storageClassName: - EOF - ``` - -5. **Use "new empty storage" option** (vm-clone skill): - - If cloning isn't supported, create VM with empty storage - - Manually copy data if needed - -6. **Cross-namespace cloning workaround**: - - Some storage backends require snapshot for cross-namespace cloning - - Create snapshot in source namespace, restore in target namespace (see solution 4 above) - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Check DataVolume reached Succeeded phase: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should return `Succeeded`). - -Check PVC was created and bound: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "-rootdisk" -} -``` - -Check `.status.phase` (should return `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -# Should return: Succeeded - -oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' -# Should return: Bound -``` - -**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/REBALANCE_AUTOMATIC.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/REBALANCE_AUTOMATIC.md deleted file mode 100644 index 936a4f3b..00000000 --- a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/REBALANCE_AUTOMATIC.md +++ /dev/null @@ -1,760 +0,0 @@ -# Automatic Rebalancing Strategy - -**Status**: ✅ PRODUCTION READY - -**Purpose**: AI-driven rebalancing where user explains high-level goals (CPU balance, memory optimization, drain node, etc.) and AI generates optimal rebalance plan. User can modify or approve plan before execution. - ---- - -## When to Use Automatic Mode - -Use this mode when the user wants: -- AI to analyze cluster and propose optimal rebalancing -- High-level goal specification (balance CPU, optimize memory, drain node) -- Expert recommendations with ability to customize -- Multi-objective optimization (CPU AND memory simultaneously) -- Intelligent rebalance planning without manual VM-by-node decisions - -**User Request Patterns:** -- "Rebalance VMs based on CPU load" -- "Optimize cluster for CPU and memory" -- "Drain worker-02 for maintenance" -- "Automatically balance the cluster" -- "Help me redistribute VMs to improve performance" -- "Optimize VM placement" - -**Do NOT use Automatic mode when:** -- User specifies exact VM→node mappings → Use Manual mode -- User only wants to see available VMs → Use `/vm-inventory` skill - ---- - -## Workflow - -### Step 1: Gather Cluster State and Determine Optimization Goal - -**1.1 Collect Cluster Information** - -**List all VMs across namespaces:** - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine" -} -``` - -Extract for each VM: -- Name, namespace -- Current node placement (from VirtualMachineInstance if running) -- Resource requests (CPU, memory) -- Storage type (RWX vs RWO) - determines live vs cold migration capability - -**List all nodes:** - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -Extract for each node: -- Name, status (Ready/NotReady) -- Capacity and allocatable resources -- Current utilization -- Taints and labels -- Schedulable status (not cordoned) - -**Gather resource usage metrics:** - -**MCP Tool**: `nodes_top` (from openshift-virtualization) - -**Parameters**: None (lists all nodes) - -Extract current CPU and memory utilization for each node. - -**MCP Tool**: `pods_top` (from openshift-virtualization) - -**Parameters**: -```json -{ - "all_namespaces": true, - "label_selector": "kubevirt.io=virt-launcher" -} -``` - -Extract current CPU and memory usage for each VM. - ---- - -**1.2 Determine Optimization Goal from User Request** - -**Analyze user's language to infer goal:** - -| User Phrase | Optimization Goal | Metrics to Optimize | -|-------------|-------------------|---------------------| -| "balance CPU", "CPU load" | Balance CPU utilization | Minimize CPU variance across nodes | -| "optimize memory", "memory pressure" | Balance memory utilization | Minimize memory variance across nodes | -| "balance cluster", "rebalance", "optimize" | Multi-objective (CPU + memory) | Minimize both CPU and memory variance | -| "drain worker-02", "evacuate node-X" | Drain specific node | Migrate all VMs off target node | -| "optimize performance" | Performance optimization | Balance resources + avoid hotspots | -| "distribute VMs evenly" | VM count distribution | Equal number of VMs per node | - -**If goal is ambiguous**, ask user to clarify: - -``` -I can optimize the cluster for several goals: -1. **CPU load balancing** - Distribute CPU usage evenly across nodes -2. **Memory load balancing** - Distribute memory usage evenly across nodes -3. **Both CPU and memory** - Multi-objective optimization -4. **Drain specific node** - Move all VMs off a node for maintenance -5. **VM count distribution** - Equal number of VMs per node - -Which optimization goal would you like me to pursue? -``` - -**WAIT for user response** before proceeding. - ---- - -**1.3 Support Multi-Objective Optimization** - -When user requests multiple goals (e.g., "balance CPU and memory"): - -**Approach:** -1. **Calculate scoring function** combining all objectives -2. **Weight objectives** (can ask user for priorities or use defaults) -3. **Find rebalance plan** that optimizes combined score - -**Example Scoring:** -``` -Score = (0.5 × CPU_variance_reduction) + (0.5 × Memory_variance_reduction) -``` - -**User can adjust weights** if AI proposes alternative approach: - -``` -I can optimize for: -- Equal priority: CPU 50%, Memory 50% -- CPU-focused: CPU 70%, Memory 30% -- Memory-focused: CPU 30%, Memory 70% - -Would you like to adjust priorities, or proceed with equal weighting? -``` - ---- - -### Step 2: Analyze and Generate Optimal Migration Plan - -**2.1 Identify Migration Candidates** - -For each optimization goal: - -**CPU Balancing:** -- Identify overloaded nodes (>80% CPU) -- Identify underloaded nodes (<50% CPU) -- Select VMs to migrate from overloaded to underloaded nodes - -**Memory Balancing:** -- Identify nodes with high memory pressure (>85%) -- Identify nodes with low memory usage (<50%) -- Select VMs to migrate for better distribution - -**Node Drain:** -- Select ALL VMs currently on target node -- Find suitable destination nodes with capacity - -**Performance Optimization:** -- Identify VMs with high resource variance (bursty workloads) -- Distribute high-performance VMs across different nodes -- Avoid co-locating resource-intensive VMs - -**2.2 Apply Constraints and Validation** - -For each candidate migration, check: - -**Storage Compatibility** (see SKILL.md - Common Validation Logic): -- RWX storage → Live migration possible -- RWO storage → Cold migration required -- Stopped VM → Cold migration required - -**Target Node Capacity:** -- Verify target has sufficient CPU and memory -- Account for VM resource requests -- Ensure node is Ready and schedulable - -**Taints and Tolerations:** -- Check target node for taints -- Verify VM has matching tolerations -- If mismatch, skip that target or propose adding tolerations - -**Concurrency Limits** (see [references/performance-tuning.md](./references/performance-tuning.md)): -- Cluster limit: 5 concurrent migrations (default) -- Per-node limit: 2 outbound migrations (default) -- Plan migration batches respecting limits - -**Network Bandwidth:** -- Avoid saturating network with too many concurrent large VM migrations -- Consider VM memory size when scheduling concurrent migrations - -**2.3 Optimize Migration Plan** - -**Migration Ordering Strategy:** - -1. **Smallest VMs first** - Faster migrations, higher success rate -2. **Live migrations before cold** - Minimize total downtime -3. **Group by source node** - Efficient for node draining -4. **Respect dependencies** - Avoid migrating related VMs simultaneously (e.g., database + app tier) - -**Expected Improvement Calculation:** - -**Before migration:** -``` -CPU variance = StdDev([worker-01: 85%, worker-02: 78%, worker-03: 42%, worker-04: 38%]) - = 22.1% -``` - -**After migration:** -``` -CPU variance = StdDev([worker-01: 65%, worker-02: 58%, worker-03: 62%, worker-04: 55%]) - = 4.2% -``` - -**Improvement:** -``` -Variance reduction: 22.1% → 4.2% = 81% improvement -``` - ---- - -### Step 3: Present Plan to User with Modification Options - -**CRITICAL**: Use the **Common Plan Visualization** format defined in SKILL.md. - -**Reference**: [SKILL.md - Common Plan Visualization](./SKILL.md#common-plan-visualization) - -**Present the plan with:** -1. **Optimization Goal** statement -2. **VM Rebalance Plan** table (from SKILL.md common format) -3. **Node State: Before → After** table (from SKILL.md common format) -4. **Key Improvement** summary line -5. **Migration Summary** (for batch operations) -6. **Risks & Considerations** (optional) -7. **User options** (approve / modify / explain / cancel) - -**Additional context for automatic mode:** -- State the optimization goal clearly (CPU balance, memory balance, multi-objective, node drain, etc.) -- Include expected improvements with metrics (variance reduction, capacity headroom, etc.) -- Offer modification options (skip, change target, reorder) -- Allow user to explain reasoning for specific migrations - -**CRITICAL: WAIT for user response before executing.** - ---- - -**3.1 Handle User Modifications** - -If user selects **"modify"**, enter interactive modification mode: - -**Example Interaction:** - -``` -User: "modify" - -Agent: "Which migrations would you like to modify?" - -Options: -- "skip <#>" - Remove migration from plan (e.g., "skip 5") -- "change target <#> to " - Change destination (e.g., "change target 1 to worker-04") -- "reorder" - Change execution order -- "done" - Finish modifications and execute - -User: "skip 5" - -Agent: "Migration #5 (app-vm-05) removed from plan. Updated plan:" - -## Updated Rebalancing Plan - -**Proposed Migrations:** (Now 4 migrations) - -| # | VM | Current Node | Target Node | Type | Impact | -|---|----|--------------|-------------|------|--------| -| 1 | database-01 | worker-01 | worker-03 | live | <1s | -| 2 | web-server-02 | worker-01 | worker-04 | live | <1s | -| 3 | cache-vm | worker-02 | worker-04 | live | <1s | -| 4 | api-gateway | worker-02 | worker-03 | live | <1s | - -**Updated Expected Improvements:** -- CPU variance: 22.1% → 5.8% (74% improvement) -- Memory variance: 11.8% → 8.1% (31% improvement) - -**Ready to execute?** (approve / modify more / cancel) - -User: "approve" - -Agent: [Proceeds to Step 4 - Execution] -``` - ---- - -**3.2 AI Proposes Better Approach (If Detected)** - -If AI detects a potentially better strategy, propose alternative: - -``` -💡 **Alternative Approach Detected** - -I notice that migrating app-vm-05 via cold migration will cause 40s downtime. However, I found: -- app-vm-05 is currently stopped (not running) -- Moving it now via cold migration has **zero additional downtime** since it's already offline - -**Alternative proposal:** -Include app-vm-05 in the plan (no additional impact vs current state) - -Would you like to: -- **accept alternative** - Include app-vm-05 in plan -- **keep original** - Proceed with current plan -- **explain more** - Tell me more about this alternative -``` - -**User has final word** - If user prefers original plan, execute original plan. - ---- - -### Step 4: Validate All VMs Before Execution - -**BEFORE executing any migration**, validate ALL VMs in the plan: - -**For each VM**, perform **Common Validation Logic from SKILL.md**: - -1. **Verify VM exists** (see SKILL.md - Validation 1) -2. **Check current location** (see SKILL.md - Validation 2) -3. **Validate storage compatibility** (see SKILL.md - Validation 3) -4. **Verify target node exists** (see SKILL.md - Validation 4) - -**If any VM fails validation:** -- Remove from rebalance plan -- Warn user: "Migration #X (vm-name) failed validation: [reason]. Proceeding with remaining migrations." -- Continue with other migrations - -**Reference**: [SKILL.md - Common Validation Logic](./SKILL.md#common-validation-logic) - ---- - -### Step 5: Execute Migrations with Progress Reporting - -**5.1 Group Migrations by Type** - -**Live Migrations** (execute first): -- Can run concurrently (up to cluster limits) -- Lower risk, zero downtime -- Follow live migration workflow from REBALANCE_MANUAL.md - -**Cold Migrations** (execute after live migrations): -- Run sequentially (to prevent cascading failures) -- Higher risk, has downtime -- Follow cold migration workflow from REBALANCE_MANUAL.md - -**5.2 Respect Concurrency Limits** - -**Cluster-wide limit**: 5 concurrent migrations (default) -**Per-node limit**: 2 outbound migrations per source node (default) - -**Monitor current migrations:** - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstanceMigration" -} -``` - -Count migrations where `.status.phase` is NOT "Succeeded" or "Failed". - -**Wait if at limit** before starting new migrations. - -**Reference**: [references/performance-tuning.md](./references/performance-tuning.md#concurrency-limits-tuning) - ---- - -**5.3 Execute Each Migration** - -**For Live Migrations:** - -**Create VirtualMachineInstanceMigration:** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters**: -```json -{ - "resource": "apiVersion: kubevirt.io/v1\nkind: VirtualMachineInstanceMigration\nmetadata:\n name: migrate--\n namespace: \nspec:\n vmiName: " -} -``` - -**Monitor migration progress:** - -Poll using `resources_get` for VirtualMachineInstanceMigration, checking `.status.phase`: -- Pending → Scheduling → PreparingTarget → Running → Succeeded - -**For Cold Migrations:** - -Follow cold migration workflow from REBALANCE_MANUAL.md: -1. Stop VM using `vm_lifecycle` (action: stop) and wait for completion -2. Re-read VM using `resources_get` for fresh resourceVersion -3. Update VM nodeAffinity to target node -4. Start VM using `vm_lifecycle` (action: start) -5. Verify VM reached target node - -**Reference**: [REBALANCE_MANUAL.md - Cold Migration Workflow](./REBALANCE_MANUAL.md) - ---- - -**5.4 Report Progress Incrementally** - -After each migration completes: - -```markdown -## Automatic Rebalancing in Progress - -**Status:** 2/4 migrations complete - -✓ **database-01**: Migrated to worker-03 (live, 42s) -✓ **web-server-02**: Migrated to worker-04 (live, 38s) -⏳ **cache-vm**: Migrating to worker-04 (live, in progress - 15s elapsed) -⏸️ **api-gateway**: Pending (waiting for cache-vm to complete) - -**Estimated time remaining:** 2-3 minutes -``` - -Update after each completion/start. - ---- - -**5.5 Error Handling During Execution** - -**On first failure:** - -1. **Stop remaining migrations** (do not continue blindly) -2. **Report detailed status**: - -```markdown -## ⚠️ Rebalancing Paused - Migration Failed - -**Status:** 2/4 successful, 1 failed, 1 not attempted - -**Successful:** -- ✓ database-01: Migrated to worker-03 (live, 42s) -- ✓ web-server-02: Migrated to worker-04 (live, 38s) - -**Failed:** -- ❌ cache-vm: Migration timeout (VM memory write rate too high) - -**Not Attempted:** -- ⏸️ api-gateway: Skipped due to previous failure - -**Troubleshooting:** - -Migration timeout typically occurs when: -- VM has high memory write rate (database, caching workload) -- Network bandwidth insufficient for transfer speed -- VM memory size very large (>32GB) - -**Recommendations:** -1. Reduce workload on cache-vm and retry -2. Use cold migration for cache-vm (will have ~30-60s downtime) -3. Check network bandwidth availability - -**How would you like to proceed?** -- **retry** - Retry failed migration with same settings -- **retry cold** - Retry using cold migration instead -- **skip** - Skip cache-vm and continue with api-gateway -- **abort** - Stop rebalancing, leave cluster in current state -``` - -3. **Wait for user decision** before proceeding - -**Reference**: [SKILL.md - Common Error Handling](./SKILL.md#common-error-handling) - ---- - -### Step 6: Report Final Results - -**On Complete Success:** - -```markdown -## ✓ Automatic Rebalancing Complete - -**All migrations successful!** - -**Executed Migrations:** - -| VM | From | To | Type | Duration | Status | -|----|------|----|----|----------|--------| -| database-01 | worker-01 | worker-03 | live | 42s | ✓ Success | -| web-server-02 | worker-01 | worker-04 | live | 38s | ✓ Success | -| cache-vm | worker-02 | worker-04 | live | 35s | ✓ Success | -| api-gateway | worker-02 | worker-03 | live | 41s | ✓ Success | - -**Cluster State: Before → After** - -| Node | CPU Before | CPU After | Change | Memory Before | Memory After | Change | -|------|------------|-----------|--------|---------------|--------------|--------| -| worker-01 | 85% | 68% | -17% ✓ | 72% | 59% | -13% ✓ | -| worker-02 | 78% | 58% | -20% ✓ | 65% | 52% | -13% ✓ | -| worker-03 | 42% | 62% | +20% | 48% | 61% | +13% | -| worker-04 | 38% | 55% | +17% | 51% | 63% | +12% | - -**Improvements Achieved:** -- ✓ **CPU load balanced**: All nodes within 10% variance (was 22.1%) -- ✓ **Memory balanced**: All nodes within 8% variance (was 11.8%) -- ✓ **No node exceeding 70% utilization** (was 85% max) -- ✓ **Cluster capacity headroom**: 41% average (was 28%) -- ✓ **Total execution time**: 2 minutes 36 seconds - -**Next Steps:** -- Monitor cluster for 24-48 hours to ensure sustained improvement -- Consider removing nodeAffinity constraints (if added) for long-term flexibility -- Use `/vm-inventory` to verify all VMs are healthy - -Cluster is now optimally balanced. No further action needed. -``` - -**On Partial Success:** - -Display similar format but include: -- Which migrations succeeded -- Which failed (with error details and troubleshooting) -- Which were not attempted (and why) -- Current cluster state vs target -- Recommendations for completing rebalancing - ---- - -## Advanced Features - -### Intelligent Workload Analysis - -**Categorize VMs by workload type** (see [references/production-considerations.md](./references/production-considerations.md)): - -- **Database** (high dirty page rate) → Schedule during low-activity window, consider cold migration -- **Web servers** (low dirty page rate) → Safe for concurrent live migration -- **Caching** (very high dirty page rate) → Migrate during idle or use cold migration -- **Batch processing** → Migrate during job idle periods - -**Use workload characteristics** to optimize migration scheduling. - -### Network Bandwidth Awareness - -**Monitor network saturation:** - -**MCP Tool**: `nodes_stats_summary` (from openshift-virtualization) - -**Parameters**: -```json -{ - "name": "" -} -``` - -Review `.network.interfaces[].rxBytes` and `.network.interfaces[].txBytes`. - -**If saturation detected** (>80% utilization): -- Reduce concurrent migrations -- Set bandwidth limits per migration -- Suggest dedicated migration network - -**Reference**: [references/live-migration-best-practices.md#dedicated-migration-network](./references/live-migration-best-practices.md#dedicated-migration-network-production-best-practice) - -### Multi-Constraint Optimization - -**Consider additional constraints:** -- **Anti-affinity rules**: Don't co-locate VMs with same label -- **Topology spread**: Distribute VMs across zones/racks -- **Resource quotas**: Respect namespace limits -- **Custom scheduling**: Honor existing nodeSelector/tolerations - -**If conflicts detected**, explain to user and suggest resolution. - ---- - -## Human-in-the-Loop Requirements - -**CRITICAL: This mode requires user approval at multiple points.** - -### 1. Goal Clarification (if ambiguous) -- Present optimization options -- Wait for user to select goal -- Do NOT proceed with assumptions - -### 2. Plan Approval (MANDATORY) -- Display complete rebalance plan -- Show expected impact and improvements -- Offer modification options -- **REQUIRE explicit approval** ("approve", "yes", "execute") -- **NEVER execute without approval** - -### 3. Plan Modification (if requested) -- Allow user to skip migrations -- Allow changing target nodes -- Allow reordering -- Recalculate expected improvements -- Present updated plan for approval - -### 4. Error Handling (on failure) -- Stop execution immediately -- Report failure details -- Ask user how to proceed -- **Do NOT continue without user decision** - -### 5. Alternative Proposals (if AI detects better approach) -- Present alternative with rationale -- Show comparison vs original plan -- **User chooses** - execute user's preference - -**Rationale**: User maintains control over cluster changes while benefiting from AI's analytical capabilities. - ---- - -## Safety Considerations - -**Automatic mode involves batch operations with higher complexity:** - -**Risks:** -- ⚠️ Multiple concurrent migrations can saturate network -- ⚠️ Batch failures can compound -- ⚠️ Wrong optimization goal can worsen cluster state -- ⚠️ User must understand multi-VM impact - -**Mitigations:** -- ✓ Require explicit plan approval before execution -- ✓ Allow plan modification before execution -- ✓ Stop on first failure (no blind continuation) -- ✓ Respect cluster concurrency limits -- ✓ Validate all VMs before starting any migration -- ✓ Detailed progress reporting with errors -- ✓ Clear troubleshooting guidance - -**Best Practices:** -- Start with dev/test clusters to learn behavior -- Use during maintenance windows for production -- Monitor network bandwidth during migrations -- Have rollback plan for critical workloads -- Consider snapshots before risky migrations (use `/vm-snapshot-create`) - -**Reference**: [references/anti-patterns.md](./references/anti-patterns.md) - ---- - -## Example Usage - -### Example 1: Multi-Objective Optimization - -``` -User: "Rebalance VMs to optimize both CPU and memory" - -Agent: [Loads REBALANCE_AUTOMATIC.md] - [Gathers cluster state: VMs, nodes, metrics] - [Determines goal: Multi-objective (CPU + memory)] - [Analyzes and generates plan] - -## Automatic Rebalancing Plan - -**Optimization Goal:** Balance CPU and memory load across cluster - -[Displays plan with 5 migrations] -[Shows current state, proposed migrations, expected improvements] - -**How would you like to proceed?** -Options: approve | modify | explain <#> | cancel - -User: "approve" - -Agent: [Validates all VMs] - [Executes migrations with progress reporting] - -## ✓ Automatic Rebalancing Complete - -All 5 migrations successful. -- CPU variance: 22.1% → 4.2% (81% improvement) -- Memory variance: 11.8% → 7.3% (38% improvement) - -Cluster is now balanced. -``` - -### Example 2: With User Modification - -``` -User: "Balance cluster CPU load" - -Agent: [Generates plan with 4 migrations] - -## Automatic Rebalancing Plan - -[Shows plan including cold migration of app-vm-05] - -User: "modify - skip 4" - -Agent: "Migration #4 (app-vm-05) removed from plan." - -[Shows updated plan with 3 migrations] -[Recalculates expected improvements] - -**Ready to execute?** (approve / modify more / cancel) - -User: "approve" - -Agent: [Executes 3 migrations] - -## ✓ Rebalancing Complete - -3/3 migrations successful. -CPU variance: 22.1% → 6.8% (69% improvement) -``` - -### Example 3: AI Proposes Alternative - -``` -User: "Drain worker-02 for maintenance" - -Agent: [Generates plan to migrate all 5 VMs from worker-02] - -💡 **Alternative Approach Detected** - -I notice 2 VMs on worker-02 use RWO storage (cold migration required). -Current plan has ~80s total downtime (2 VMs × 40s each). - -**Alternative:** -Migrate VMs sequentially instead of concurrently to reduce risk: -- Same total time -- Lower network impact -- Easier to abort if issues - -Would you like to: accept alternative | keep original | explain more - -User: "accept alternative" - -Agent: [Updates plan to sequential execution] - [Presents updated plan for approval] -``` - ---- - -**Sources:** -- [Live Migration - KubeVirt User Guide](https://kubevirt.io/user-guide/compute/live_migration/) -- [Node Assignment - KubeVirt User Guide](https://kubevirt.io/user-guide/compute/node_assignment/) -- [Kubernetes Descheduler](https://github.com/kubernetes-sigs/descheduler) -- [Best Practices for Virtual Machine Deployments](https://learn.microsoft.com/en-us/azure/openshift/best-practices-openshift-virtualization) - -**Last Updated**: 2026-02-24 -**Status**: Production Ready diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/REBALANCE_MANUAL.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/REBALANCE_MANUAL.md deleted file mode 100644 index f1e9d2c6..00000000 --- a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/REBALANCE_MANUAL.md +++ /dev/null @@ -1,848 +0,0 @@ -# Manual Rebalancing Strategy - -**Status**: ✅ FULLY IMPLEMENTED - -**Purpose**: Execute VM migrations where the user specifies which VM(s) to move and the target node(s). - -## When to Use Manual Mode - -Use this mode when the user specifies: -- Exact VM name(s) to migrate -- Target node(s) for placement -- Migration type (live or cold) - -**User Request Patterns:** -- "Move VM database-01 to worker-03" -- "Live migrate web-server from worker-01 to worker-05" -- "Cold migrate app-vm to worker-02" -- "Migrate VMs web-01, web-02, web-03 to worker-05" - -## Manual Rebalancing Workflow - -### Step 1: Gather Migration Parameters - -**Collect the following information from the user:** - -**Required Parameters:** -1. **VM Name(s)** - Name of virtual machine(s) to migrate - - Example: "database-01" or ["web-01", "web-02", "web-03"] - -2. **Source Namespace** - Namespace where VM(s) exist - - Example: "production", "vms", "dev-environment" - -3. **Target Node** (optional for live migration, required for cold migration) - - Example: "worker-03", "worker-node-1.example.com" - - For live migration: Can be omitted (scheduler chooses) - - For cold migration: Required (set via nodeAffinity) - -4. **Migration Type** - Choose migration strategy - - `live` - Zero downtime, VM stays running (requires RWX storage) - - `cold` - Brief downtime, stop → move → start (always works) - -**If user doesn't specify migration type, determine automatically:** -- Check VM's PVC access mode using `resources_get` -- If PVC has ReadWriteMany (RWX) → Suggest live migration -- If PVC has ReadWriteOnce (RWO) → Require cold migration -- If uncertain → Ask user to choose - -### Step 2: Validate Migration Feasibility - -**CRITICAL**: Before proceeding with migration execution, perform the **Common Validation Logic** defined in SKILL.md. - -**Reference**: See [SKILL.md - Common Validation Logic](./SKILL.md#common-validation-logic) for complete validation steps. - -**The common validation performs these checks** (defined in SKILL.md): -1. **Validation 1**: Verify VM Exists -2. **Validation 2**: Check Current VM Location -3. **Validation 3**: Validate Storage Compatibility (RWX vs RWO) -4. **Validation 4**: Verify Target Node Exists - -**Only proceed to Step 3 after all validations pass.** - -### Step 3: Present Rebalance Plan for Confirmation - -**After validation, present the rebalance plan to the user:** - -**For Live Migration:** - -```markdown -## VM Rebalance Plan - -**Please review and confirm the migration:** - -| Parameter | Value | Details | -|-----------|-------|---------| -| VM Name | `` | [from user input] | -| Namespace | `` | [from user input] | -| Current Node | `` | [detected from VMI status] | -| Target Node | `` | [from user input or "Scheduler decides"] | -| Migration Type | `live` | Zero downtime, VM stays running | -| Storage Type | `RWX (ReadWriteMany)` | Live migration supported | -| Current Status | `Running` | [from VMI phase] | - -**Migration Strategy: Live Migration** - -**What will happen:** -1. Create VirtualMachineInstanceMigration resource -2. KubeVirt will: - - Create new virt-launcher pod on target node - - Transfer VM memory and state (live) - - Switch network traffic to new pod - - Terminate old virt-launcher pod -3. VM continues running throughout (brief network pause <1s) - -**Impact:** -- ✓ Zero downtime (VM stays running) -- ✓ Applications remain accessible -- ⚠️ Brief network pause during cutover (<1 second) -- ⚠️ Requires network bandwidth for memory transfer -- ⚠️ Migration duration depends on VM memory size - -**Estimated Duration:** ~30-60 seconds for typical VM (depends on memory size) - -**Reference**: See [references/live-migration-best-practices.md](./references/live-migration-best-practices.md) for configuration parameters, storage requirements, and network optimization - -**Rollback:** If migration fails, VM continues running on current node - -Confirm this migration or tell me what to change: -- yes - Proceed with migration -- no - Cancel migration -- modify - Change parameters -``` - -**For Cold Migration:** - -```markdown -## VM Rebalance Plan - -| Parameter | Value | Details | -|-----------|-------|---------| -| VM Name | `` | [from user input] | -| Namespace | `` | [from user input] | -| Current Node | `` | [detected] | -| Target Node | `` | [from user input] | -| Migration Type | `cold` | VM will stop and restart | -| Storage Type | `RWO (ReadWriteOnce)` | Live migration not supported | -| Current Status | `Running` | Will be stopped then restarted | - -**Migration Strategy: Cold Migration** - -**What will happen:** -1. Update VM spec with **required nodeAffinity** (forces placement on target node) -2. Stop VM using vm_lifecycle -3. VM pod terminates on current node -4. Start VM using vm_lifecycle -5. Kubernetes scheduler places VM on target node (due to nodeAffinity) -6. **After migration:** You'll choose whether to remove the affinity for long-term flexibility - -**Impact:** -- ⚠️ **Downtime**: ~30-60 seconds while VM restarts -- ⚠️ Applications will be interrupted -- ⚠️ Network connections will be dropped -- ✓ Works with any storage type (RWO or RWX) -- ✓ Less resource-intensive than live migration -- ℹ️ **Long-term**: After migration, you can remove node affinity to allow flexible rescheduling if target node fails - -**Estimated Downtime:** ~30-60 seconds - -**Rollback:** Can start VM again if migration fails - -**Note on Node Affinity:** -We use nodeAffinity (not nodeSelector) for better long-term resilience. After migration, you'll have the option to remove it, allowing the VM to reschedule to other nodes if fails in the future. - -**Reference**: See [references/production-considerations.md](./references/production-considerations.md) for workload categorization and downtime planning strategies - -Confirm this migration or tell me what to change: -- yes - Proceed with cold migration -- no - Cancel -- modify - Change parameters -``` - -**WAIT for user confirmation before proceeding to Step 4.** Do NOT continue until user explicitly confirms with "yes". - -### Step 4a: Execute Live Migration (if migration type = live) - -**ONLY PROCEED AFTER USER CONFIRMATION IN STEP 3.** - -**Create a VirtualMachineInstanceMigration resource to trigger live migration:** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters**: -- `resource`: (JSON object as string) - REQUIRED - - Complete VirtualMachineInstanceMigration resource definition as a JSON-formatted string - - Must follow KubeVirt API specification - -**Resource Structure** (for live migration): - -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstanceMigration", - "metadata": { - "name": "migration--", - "namespace": "" - }, - "spec": { - "vmiName": "" - } -} -``` - -**Example tool invocation:** -```json -{ - "resource": "{\"apiVersion\":\"kubevirt.io/v1\",\"kind\":\"VirtualMachineInstanceMigration\",\"metadata\":{\"name\":\"migration-database-01-20260223\",\"namespace\":\"production\"},\"spec\":{\"vmiName\":\"database-01\"}}" -} -``` - -**Note**: The `resource` parameter accepts the resource definition as a JSON-formatted string. The MCP tool will parse and apply this resource to the cluster. - -**Note on Target Node Selection:** -- KubeVirt scheduler automatically selects target node -- To influence target node, update VM's nodeAffinity BEFORE creating migration -- For manual target node selection, combine with nodeAffinity update first - -**Expected Output**: VirtualMachineInstanceMigration resource created successfully - -**Error Handling:** -- If creation fails → Check RBAC permissions, report error to user -- If VMI not found → Verify VM is running, report error -- If VMI not migratable → Check storage access mode, suggest cold migration - -#### Monitor Migration Progress - -**After creating migration, monitor progress:** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters** (to monitor migration status): -- `apiVersion`: "kubevirt.io/v1" - REQUIRED -- `kind`: "VirtualMachineInstanceMigration" - REQUIRED -- `name`: "migration--" - REQUIRED -- `namespace`: "" - REQUIRED - -**Poll every 5-10 seconds until migration completes or fails.** - -**Timeout**: Stop polling after 10 minutes (600 seconds) and report timeout error to user. Most migrations complete within 1-5 minutes depending on VM memory size. - -**Reference**: See [references/performance-tuning.md](./references/performance-tuning.md) for timeout configuration and migration optimization strategies - -**Expected Output**: VMIM resource with status field - -**Extract Information:** -- `status.phase` - Migration phase: - - `Scheduling` - Finding target node - - `PreparingTarget` - Setting up target pod - - `TargetReady` - Target pod ready - - `Running` - Transferring memory - - `Succeeded` - Migration completed - - `Failed` - Migration failed -- `status.migrationState.completed` - Boolean, true when done -- `status.migrationState.targetNode` - Destination node -- `status.migrationState.sourceNode` - Origin node -- `status.migrationState.startTimestamp` - When migration began -- `status.migrationState.endTimestamp` - When migration completed - -**When status.phase = "Succeeded":** -- Migration completed successfully -- Proceed to Step 5 (Report Results) - -**When status.phase = "Failed":** -- Extract failure reason from status -- Consult troubleshooting documentation (see Step 5 failure handling) -- Report detailed error to user - -### Step 4b: Execute Cold Migration (if migration type = cold) - -**ONLY PROCEED AFTER USER CONFIRMATION IN STEP 3.** - -**Cold migration workflow: Stop VM → Re-read VM → Update node placement → Start VM** - -#### Sub-step 4b.1: Stop the VM - -**MCP Tool**: `vm_lifecycle` (from openshift-virtualization) - -**Parameters**: -- `namespace`: "" - REQUIRED -- `name`: "" - REQUIRED -- `action`: "stop" - REQUIRED - -**Expected Output**: VM stopped successfully, VMI terminates - -**Wait for VM to fully stop:** -1. Wait 10 seconds -2. Check VM status using `resources_get` (VirtualMachine) -3. If `status.printableStatus` is not "Stopped", wait another 10 seconds and check again -4. Repeat until VM is fully stopped - -**Error Handling:** -- If stop fails → Report error, check if VM is already stopped -- If VM stuck in Terminating after 60 seconds → Report to user - -#### Sub-step 4b.2: Re-read VM for Fresh ResourceVersion - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -- `apiVersion`: "kubevirt.io/v1" - REQUIRED -- `kind`: "VirtualMachine" - REQUIRED -- `name`: "" - REQUIRED -- `namespace`: "" - REQUIRED - -**Why**: `vm_lifecycle` modified the VM. Re-reading gets fresh resourceVersion to prevent conflicts when updating nodeAffinity. - -**Use this fresh VM spec for nodeAffinity update in next step.** - -#### Sub-step 4b.3: Update VM nodeAffinity - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters**: -- `resource`: (JSON object as string) - REQUIRED - - Complete VirtualMachine resource from Sub-step 4b.2 with updated nodeAffinity - -**Resource Modification** (add required nodeAffinity to VM spec): - -Preserve all existing VM fields and only modify the affinity section. - -**Critical Affinity Structure:** -```json -{ - "spec": { - "template": { - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.io/hostname", - "operator": "In", - "values": [""] - } - ] - } - ] - } - } - } - } - } - } -} -``` - -**Why nodeAffinity instead of nodeSelector?** -- Provides flexibility for long-term resilience -- If target node fails later, user can remove affinity to allow rescheduling -- More powerful than nodeSelector (supports multiple nodes, preferences) - -**Expected Output**: VirtualMachine resource updated successfully - -**Error Handling:** -- If update fails → Check RBAC permissions, explain to user with exact error -- If VM not found → Report error with VM name and namespace -- If conflicts with existing affinity → Ask user: "VM already has node affinity rules. Should I replace them with new affinity to ?" - -#### Sub-step 4b.4: Start the VM - -**MCP Tool**: `vm_lifecycle` (from openshift-virtualization) - -**Parameters**: -- `namespace`: "" - REQUIRED -- `name`: "" - REQUIRED -- `action`: "start" - REQUIRED - - Example: `"start"` (power on the VM) - -**Expected Output**: VM starts successfully - -**After starting, verify VM scheduled on target node:** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters** (get VMI to verify node placement): -- `apiVersion`: "kubevirt.io/v1" - REQUIRED -- `kind`: "VirtualMachineInstance" - REQUIRED -- `name`: "" - REQUIRED -- `namespace`: "" - REQUIRED - -**Expected Output**: VMI resource with node placement - -**Extract Information:** -- `status.nodeName` - Should match target node -- `status.phase` - Should be "Running" - -**Verification:** -- If `status.nodeName` matches target node → Success, proceed to Step 4b.5 -- If `status.nodeName` does NOT match target node → Migration failed, report error with explanation - -**Error Handling with Clear Explanations:** -- If VM fails to start → Explain: "VM failed to start on . This usually means the node doesn't have enough resources (CPU/memory) or has scheduling constraints preventing this VM." - - Consult [scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) - - Suggest: Check node capacity, verify node is Ready, check for taints -- If VM scheduled on wrong node → Explain: "VM started but Kubernetes placed it on instead of . This means couldn't accommodate the VM due to resource constraints, taints, or other scheduling rules." - - Report actual vs expected node - - Ask user if they want to retry with different target node -- If VM stuck in Scheduling phase → Explain: "VM cannot be scheduled. The target node likely lacks sufficient CPU/memory resources, or has taints that prevent scheduling." - - Show node constraints - - Suggest checking node details using `resources_get` with kind="Node" - -#### Sub-step 4b.5: Post-Migration Cleanup (Optional - Long-term Resilience) - -**After successful cold migration, ask user about removing node affinity for long-term flexibility.** - -**Present cleanup option to user:** - -```markdown -## ✓ Cold Migration Successful - VM Running on Target Node - -**VM Details:** -- **Name**: `` -- **Namespace**: `` -- **Current Node**: `` ✓ - -**Important: Long-term Resilience** - -The VM now has a **required node affinity** to ``. This means: -- ✓ VM will stay on `` (as you requested) -- ⚠️ If `` fails or needs maintenance later, the VM **cannot reschedule** to other nodes - -**Recommendation:** Remove the node affinity to allow flexible rescheduling in the future. - -Would you like to remove the node affinity now? -- **yes** - Remove affinity (VM can reschedule to any healthy node if fails) -- **no** - Keep affinity (VM stays pinned to permanently) -- **later** - Keep for now, I'll remove it manually when needed -``` - -**WAIT for user decision.** - -**If user says "yes" (remove affinity):** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters** (get current VM spec): -- `apiVersion`: "kubevirt.io/v1" - REQUIRED -- `kind`: "VirtualMachine" - REQUIRED -- `name`: "" - REQUIRED -- `namespace`: "" - REQUIRED - -**Expected Output**: Full VirtualMachine resource with current affinity - -**Modify the VM spec to remove nodeAffinity:** - -Extract the full VM resource, then remove `spec.template.spec.affinity.nodeAffinity` (or set to null/empty). - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters**: -- `resource`: (JSON object as string) - REQUIRED - - Complete VirtualMachine resource with affinity removed as a JSON-formatted string - -**Important**: -- Preserve all other VM spec fields -- Only remove the affinity section -- VM continues running on current node (doesn't trigger immediate reschedule) -- Future restarts will allow flexible scheduling - -**Expected Output**: VirtualMachine resource updated successfully - -**Error Handling:** -- If update fails → Explain: "Failed to remove node affinity. RBAC permissions issue or API error. VM will remain pinned to . You can remove manually using `resources_create_or_update` with the VM spec after removing the affinity field." - -**If user says "no" or "later" (keep affinity):** -- Report: "Node affinity kept. VM will remain on . To remove manually later, use `resources_create_or_update` with the VM spec after removing the affinity field." -- Proceed to Step 5 - -### Step 5: Report Migration Results - -**On Success (Live Migration):** - -```markdown -## ✓ Live Migration Successful - -**VM Details:** -- **Name**: `` -- **Namespace**: `` -- **Migration Type**: Live (zero downtime) - -**Migration Summary:** -- **Source Node**: `` -- **Target Node**: `` -- **Duration**: seconds -- **Status**: Succeeded - -**Impact:** -- ✓ VM remained running throughout migration -- ✓ Applications stayed accessible -- ✓ Brief network pause during cutover (<1 second) - -**Current VM Status:** -- **Running on**: `` -- **Phase**: Running -- **Ready**: True - -### Next Steps - -**Verify application health:** -Use `/vm-inventory` to check VM status - -**View migration details:** -Use `resources_get` with kind="VirtualMachineInstanceMigration" and name="migration--" - -**Cleanup:** -The VirtualMachineInstanceMigration resource can be deleted if no longer needed. -``` - -**On Success (Cold Migration) - If affinity was REMOVED:** - -```markdown -## ✓ Cold Migration Successful - -**VM Details:** -- **Name**: `` -- **Namespace**: `` -- **Migration Type**: Cold (with downtime) - -**Migration Summary:** -- **Source Node**: `` -- **Target Node**: `` -- **Downtime**: ~ seconds -- **Status**: Succeeded - -**Steps Completed:** -1. ✓ Updated VM with required nodeAffinity to `` -2. ✓ Stopped VM on `` -3. ✓ Started VM on `` -4. ✓ Verified VM running on target node -5. ✓ Removed node affinity for flexible rescheduling - -**Current VM Status:** -- **Running on**: `` -- **Phase**: Running -- **Ready**: True -- **Node Affinity**: None (can reschedule to any node if needed) - -**Long-term Resilience:** -✓ VM can now reschedule to other nodes if `` fails or needs maintenance. - -### Next Steps - -**Verify application health:** -Use `/vm-inventory` to check VM status - -**Test application connectivity:** -VM has restarted, verify services are healthy. -``` - -**On Success (Cold Migration) - If affinity was KEPT:** - -```markdown -## ✓ Cold Migration Successful - -**VM Details:** -- **Name**: `` -- **Namespace**: `` -- **Migration Type**: Cold (with downtime) - -**Migration Summary:** -- **Source Node**: `` -- **Target Node**: `` -- **Downtime**: ~ seconds -- **Status**: Succeeded - -**Steps Completed:** -1. ✓ Updated VM with required nodeAffinity to `` -2. ✓ Stopped VM on `` -3. ✓ Started VM on `` -4. ✓ Verified VM running on target node -5. ℹ️ Kept node affinity (as requested) - -**Current VM Status:** -- **Running on**: `` -- **Phase**: Running -- **Ready**: True -- **Node Affinity**: Required on `` (VM will stay on this node) - -**Important:** -⚠️ VM is pinned to ``. If this node fails, the VM cannot reschedule to other nodes. - -**To remove affinity later:** -Use `resources_create_or_update` with the VM spec after removing the `spec.template.spec.affinity` field - -### Next Steps - -**Verify application health:** -Use `/vm-inventory` to check VM status - -**Test application connectivity:** -VM has restarted, verify services are healthy. -``` - -**On Failure (with Troubleshooting):** - -**OPTIONAL**: If migration fails, consult documentation for common failure scenarios. - -**Document Consultation** (OPTIONAL - when migration fails): -1. **Action**: Read relevant troubleshooting guides to understand VM migration failure scenarios: - - [scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) - For ErrorUnschedulable, node taints, resource constraints - - [lifecycle-errors.md](../../docs/troubleshooting/lifecycle-errors.md) - For VM start/stop failures during cold migration - - [storage-errors.md](../../docs/troubleshooting/storage-errors.md) - For PVC access mode issues affecting live migration -2. **Output to user**: "I consulted [scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) to understand potential causes for the migration failure." - -**When to consult**: -- Live migration fails (check storage, network, resource constraints) -- Cold migration fails during start (likely scheduling or resource issues) -- VM stuck in ErrorUnschedulable state -- Unexpected error messages from migration tools - -**When NOT to consult**: -- RBAC permission errors (clear cause) -- VM not found errors (clear cause) -- PVC access mode incompatibility (already validated in Step 2) - -```markdown -## ❌ Migration Failed - -**Error**: - -**VM Details:** -- **Name**: `` -- **Namespace**: `` -- **Migration Type**: -- **Source Node**: `` -- **Target Node**: `` - -**Common Causes:** - -**For Live Migration Failures:** -- **Insufficient resources** - Target node lacks CPU/memory for VM -- **Network bandwidth** - Slow network prevents migration convergence -- **PVC access mode** - Storage is not ReadWriteMany (RWX) -- **Timeout exceeded** - Migration took longer than allowed (default 150s/GiB) -- **High memory write rate** - VM writes to memory faster than transfer rate - -**For Cold Migration Failures:** -- **Node not schedulable** - Target node is cordoned, has taints, or lacks resources -- **NodeSelector conflict** - VM has other scheduling constraints preventing placement -- **VM failed to start** - Check scheduling errors on target node -- **Storage issues** - PVC not accessible from target node - -**Troubleshooting Steps:** - -1. **Check node availability:** - Use `resources_list` with kind="Node" to verify target node is Ready and schedulable - -2. **Check VM events:** - Use `resources_get` with kind="VirtualMachine" to see VM status and conditions - -3. **Check migration status** (for live migration): - Use `resources_list` with kind="VirtualMachineInstanceMigration" to see migration resource status - -4. **Verify storage access:** - Use `resources_get` with kind="PersistentVolumeClaim" to check PVC access modes - -5. **Check resource capacity:** - Use `nodes_top` or `resources_get` with kind="Node" to verify available CPU/memory - -**Recommended Actions:** - -- **If timeout**: Retry with slower workload or enable auto-converge -- **If resources**: Choose different target node with more capacity -- **If storage**: Use cold migration instead (works with RWO) -- **If node unavailable**: Verify node is Ready and untainted - -**Rollback (if migration succeeded but VM has issues on target node):** - -To move VM back to original node, perform reverse migration: -- **For live migration**: Create new migration from current node → original node -- **For cold migration**: Update nodeAffinity to original node, restart VM -- Use same migration type as original migration - -Example: "Migrate from back to " - -Would you like help troubleshooting this error? -``` - -## Batch VM Migration - -**For migrating multiple VMs in a single operation:** - -**User Request:** "Migrate VMs web-01, web-02, web-03 to worker-05" - -**Safety Limits:** -- ⚠️ **Batches >5 VMs**: Warn user about potential network saturation and suggest staged execution -- ⚠️ **Batches >10 VMs**: Strongly recommend splitting into multiple batches (e.g., 5 VMs at a time) -- ⚠️ **Multiple cold migrations**: Warn about cumulative downtime impact - -**Reference**: See [references/anti-patterns.md](./references/anti-patterns.md) for common mistakes when performing batch migrations and network saturation issues - -**Workflow:** -1. Validate all VMs exist and are migratable -2. Determine migration type for each (based on storage) -3. **Check batch size** - If >5 VMs, warn user and offer to stage migrations -4. Present batch rebalance plan: - ```markdown - ## Batch Rebalance Plan - - Migrating 3 VMs to worker-05: - - | VM Name | Current Node | Migration Type | Impact | - |---------|--------------|----------------|--------| - | web-01 | worker-01 | live | <1s pause | - | web-02 | worker-02 | live | <1s pause | - | web-03 | worker-03 | cold | ~40s downtime | - - Total: 2 live migrations + 1 cold migration - - Proceed with batch migration? - ``` -4. Execute migrations sequentially (avoid saturating network) -5. Report results for each VM - -## Example Usage - -### Example 1: Successful Live Migration - -``` -User: "Live migrate VM database-01 from production namespace to worker-03" - -Agent: [Validates VM exists, checks storage is RWX, verifies target node] - -## VM Rebalance Plan - -Please review and confirm the migration: - -| Parameter | Value | Details | -|-----------|-------|---------| -| VM Name | `database-01` | from user input | -| Namespace | `production` | from user input | -| Current Node | `worker-01` | detected from VMI status | -| Target Node | `worker-03` | from user input | -| Migration Type | `live` | Zero downtime, VM stays running | -| Storage Type | `RWX (ReadWriteMany)` | Live migration supported | - -**What will happen:** -1. Create VirtualMachineInstanceMigration resource -2. VM memory and state transferred live to worker-03 -3. Brief network pause during cutover (<1 second) - -**Impact:** Zero downtime, brief network pause - -Confirm this migration? - -User: "yes" - -Agent: [Creates VirtualMachineInstanceMigration resource] - [Monitors migration status] - -## ✓ Live Migration Successful - -**VM Details:** -- Name: `database-01` -- Namespace: `production` -- Migration Type: Live (zero downtime) - -**Migration Summary:** -- Source Node: `worker-01` -- Target Node: `worker-03` -- Duration: 45 seconds -- Status: Succeeded - -VM is now running on `worker-03`. -``` - -### Example 2: Successful Cold Migration - -``` -User: "Move VM web-server to worker-05 using cold migration" - -Agent: [Validates VM exists, checks current node, verifies target node] - -## VM Rebalance Plan - -| Parameter | Value | Details | -|-----------|-------|---------| -| VM Name | `web-server` | from user input | -| Namespace | `vms` | detected/default | -| Current Node | `worker-02` | detected | -| Target Node | `worker-05` | from user input | -| Migration Type | `cold` | VM will stop and restart | -| Storage Type | `RWO (ReadWriteOnce)` | Live migration not supported | - -**What will happen:** -1. Update VM nodeAffinity to target worker-05 -2. Stop VM (applications will be interrupted) -3. Start VM (Kubernetes places on worker-05) - -**Impact:** Downtime ~30-60 seconds - -Confirm this migration? - -User: "yes" - -Agent: [Updates VM nodeAffinity] - [Calls vm_lifecycle stop] - [Calls vm_lifecycle start] - [Verifies VM on target node] - -## ✓ Cold Migration Successful - -**VM Details:** -- Name: `web-server` -- Namespace: `vms` -- Migration Type: Cold (with downtime) - -**Migration Summary:** -- Source Node: `worker-02` -- Target Node: `worker-05` -- Downtime: ~40 seconds -- Status: Succeeded - -VM is now running on `worker-05`. Verify application health. -``` - -### Example 3: Migration Failure - Storage Incompatible - -``` -User: "Live migrate app-vm to worker-03" - -Agent: [Validates VM] - [Checks PVC access mode - discovers RWO] - -## ❌ Live Migration Not Supported - -**VM Details:** -- Name: `app-vm` -- Namespace: `default` -- Storage Type: `RWO (ReadWriteOnce)` - -**Issue:** Live migration requires ReadWriteMany (RWX) storage. This VM uses ReadWriteOnce (RWO), which only allows attachment to one node at a time. - -**Recommended Solution:** Use cold migration instead. - -Would you like to proceed with cold migration? (Brief downtime ~30-60 seconds) - -User: "yes, do cold migration" - -Agent: [Switches to cold migration workflow] - [Continues with cold migration process...] -``` - ---- - -**Reference Documentation:** - -**Internal Skill Documentation**: -- [references/live-migration-best-practices.md](./references/live-migration-best-practices.md) - Configuration, requirements, dedicated networks -- [references/performance-tuning.md](./references/performance-tuning.md) - Right-sizing, overcommit, bandwidth tuning -- [references/anti-patterns.md](./references/anti-patterns.md) - Common mistakes to avoid -- [references/production-considerations.md](./references/production-considerations.md) - HA strategies, capacity planning - -**Official KubeVirt Documentation**: -- [Live Migration - KubeVirt User Guide](https://kubevirt.io/user-guide/compute/live_migration/) -- [Node Assignment - KubeVirt User Guide](https://kubevirt.io/user-guide/compute/node_assignment/) -- [VirtualMachineInstanceMigration API](https://kubevirt.io/api-reference/main/definitions.html#_v1_virtualmachineinstancemigration) diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/SKILL.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/SKILL.md deleted file mode 100644 index 4ee0bb9c..00000000 --- a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/SKILL.md +++ /dev/null @@ -1,391 +0,0 @@ ---- -name: vm-rebalance -description: | - Orchestrate VM migrations across cluster nodes for load balancing, maintenance, and resource optimization. - - Use when: - - "Move VM database-01 to worker-03" - - "Rebalance VMs to optimize CPU load" - - "Drain worker-02 for maintenance" - - "Automatically rebalance the cluster" - - Supports Manual (user-driven) and Automatic (AI-driven) modes. - - NOT for creating VMs (use vm-create) or lifecycle only (use vm-lifecycle-manager). - -model: inherit -color: yellow ---- - -# /vm-rebalance Skill - -Orchestrate VM migrations across OpenShift cluster nodes for load balancing, maintenance, and resource optimization. Supports manual and automatic rebalancing with live migration (zero downtime) and cold migration (brief downtime) strategies. - -**Implementation**: Uses KubeVirt's VirtualMachineInstanceMigration API for live migrations and node affinity for cold migrations. - -## Prerequisites - -**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools**: -- `resources_list` - List VMs and nodes -- `resources_get` - Get VM and node details -- `resources_create_or_update` - Create migrations and update VM specs -- `vm_lifecycle` - Start/stop VMs for cold migration -- `nodes_top` - Monitor node resource usage -- `pods_top` - Monitor VM resource consumption - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file - -**Required Cluster Setup**: -- OpenShift cluster (>= 4.17) -- OpenShift Virtualization operator installed -- ServiceAccount with permissions: get/list/update for VMs, create for VirtualMachineInstanceMigration -- For live migration: RWX storage and sufficient network bandwidth - -### Prerequisite Verification - -**Before executing:** - -1. Check `openshift-virtualization` exists in `.mcp.json` → If missing, report setup -2. Verify `KUBECONFIG` is set (presence only, never expose value) → If missing, report -3. For live migration: Check PVC access mode is ReadWriteMany (RWX) via `resources_get` - -**Human Notification Protocol:** `❌ Cannot execute vm-rebalance: MCP server not available. Setup: Add to .mcp.json, set KUBECONFIG, restart Claude Code. Docs: https://github.com/openshift/openshift-mcp-server` - -⚠️ **SECURITY**: Never display KUBECONFIG path or credential values. - -## When to Use This Skill - -**Trigger when:** -- User explicitly invokes `/vm-rebalance` -- User requests moving VM(s) to specific node(s) -- User wants to drain node for maintenance -- User requests load balancing or resource optimization - -**User phrases:** -- "Move VM database-01 to worker-03" -- "Live migrate web-server to worker-05" -- "Drain worker-02 for maintenance" -- "Balance CPU load across nodes" -- "Automatically rebalance the cluster" - -**Do NOT use when:** -- Creating VMs → `/vm-create` -- Start/stop only → `/vm-lifecycle-manager` -- Cloning VMs → `/vm-clone` -- Deleting VMs → `/vm-delete` - -## Workflow - -### Step 1: Determine Rebalancing Mode - -**Manual Mode**: User specifies VM name(s) and target node(s). Example: "Move VM database-01 to worker-03" - -**Automatic Mode**: User requests AI-driven rebalancing. Example: "Rebalance VMs based on CPU" - -### Step 2: Load Strategy File and Execute - -**For Manual Mode:** - -**Document Consultation** (REQUIRED - Execute FIRST): -1. Read [REBALANCE_MANUAL.md](./REBALANCE_MANUAL.md) using Read tool -2. Output: "I consulted [REBALANCE_MANUAL.md](./REBALANCE_MANUAL.md) to understand the manual migration workflow." -3. **Then execute**: Follow workflow in REBALANCE_MANUAL.md - ---- - -**For Automatic Mode:** - -**Document Consultation** (REQUIRED - Execute FIRST): -1. Read [REBALANCE_AUTOMATIC.md](./REBALANCE_AUTOMATIC.md) using Read tool -2. Output: "I consulted [REBALANCE_AUTOMATIC.md](./REBALANCE_AUTOMATIC.md) to understand the automatic rebalancing workflow." -3. **Then execute**: Follow workflow in REBALANCE_AUTOMATIC.md - -## Common Validation Logic - -**Shared by ALL migration strategies. Execute before any VM migration:** - -### Validation 1: Verify VM Exists - -**MCP Tool**: `resources_get` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", name=``, namespace=``) - -**Extract**: `spec.template.spec.volumes[].persistentVolumeClaim.claimName`, `status.ready` - -**Errors**: VM not found → Use vm-inventory | Namespace not found → Verify name | Permission denied → Check RBAC - -### Validation 2: Check Current VM Location - -**MCP Tool** (if VM running): `resources_get` (apiVersion="kubevirt.io/v1", kind="VirtualMachineInstance", name=``, namespace=``) - -**Extract**: `status.nodeName`, `status.phase` - -**Validation**: If already on target → "VM already on target node. No migration needed." - -### Validation 3: Validate Storage Compatibility - -**MCP Tool**: `resources_get` (apiVersion="v1", kind="PersistentVolumeClaim", name=``, namespace=``) - -**Extract**: `spec.accessModes` -- ReadWriteMany (RWX) → Live migration supported -- ReadWriteOnce (RWO) → Live migration NOT supported - -**Error for live migration**: If RWO → "Cannot live migrate. Use cold migration (brief downtime ~30-60s)." - -**Reference**: [references/live-migration-best-practices.md](./references/live-migration-best-practices.md) - -### Validation 4: Verify Target Node Exists - -**MCP Tool**: `resources_list` (apiVersion="v1", kind="Node") - -**Validation**: Verify target exists, `status.conditions[]` shows Ready=True, not cordoned - -**Errors**: Not found → "Node doesn't exist" | Not Ready → "Choose different target" | Cordoned → "Uncordon or choose different target" - -**Reference**: [../../docs/troubleshooting/scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) - -## Node Selection for Automatic Rebalancing - -**Applies to Automatic Mode only.** - -**Use** `resources_list` **(apiVersion="v1", kind="Node")** - -Filter where ALL true: -1. `metadata.labels["kubevirt.io/schedulable"] == "true"` -2. `status.capacity["devices.kubevirt.io/kvm"]` > "0" -3. No `node-role.kubernetes.io/control-plane` or `node-role.kubernetes.io/master` label - -**If no nodes**: "No suitable nodes. Check OpenShift Virtualization operator and hardware virtualization support." - -**Note**: Ignore custom taints. Use official KubeVirt labels. - -## Common Migration Types - -**Live Migration**: Zero downtime, <1s pause during cutover. Requires RWX storage. Memory transferred while VM runs. - -**Cold Migration**: Brief downtime (~30-60s). Works with any storage. Stop VM → Update placement → Start on target. - -**Reference**: [references/live-migration-best-practices.md](./references/live-migration-best-practices.md) - -## Common Plan Visualization - -**ALL strategies MUST use this standardized format for consistency.** - -### Information Relevance Principle - -Show only what matters: -- ✅ Include: Deviations from defaults, user-specified criteria, non-obvious context -- ❌ Exclude: Standard procedures, default settings, info already visible in tables - -### Standard Plan Format - -**Table 1: VM Rebalance Plan** - -```markdown -## 📋 VM Rebalance Plan - -| VM | Instance Type | Current Node | → | New Node | Type | Downtime | Notes | -|----|---------------|--------------|---|----------|------|----------|-------| -| vm-1 | u1.xlarge | worker-01 | → | worker-03 | Live | <1s pause | ContainerDisk | -| vm-2 | u1.2xmedium | worker-01 | → | worker-02 | Cold | ~40s | RWO storage | -| vm-3 | u1.medium | worker-02 | - | *stays* | - | - | Already balanced | -``` - -**Column Definitions**: VM name | Instance type | Current node | Movement indicator | Target node (or *stays*) | Migration type (Live/Cold/-) | Downtime (<1s/~30-60s/-) | Brief explanation - -**Table 2: Node State Before → After** - -```markdown -## 📊 Node State: Before → After - -| Node | VMs Now | CPU Now | Memory Now | → | VMs After | CPU After | Memory After | Change | -|------|---------|---------|------------|---|-----------|-----------|--------------|--------| -| worker-01 | 5 | 85% | 72% | → | 3 | 68% | 59% | ✓ Reduced load | -| worker-02 | 2 | 42% | 48% | → | 3 | 58% | 61% | ← Receiving VMs | -| worker-03 | 3 | 38% | 51% | → | 4 | 55% | 63% | ← Receiving VMs | -``` - -**CRITICAL - Capacity Calculation Method:** - -CPU/Memory percentages MUST be calculated based on **allocated capacity**, not actual runtime usage: - -**CPU Percentage Calculation**: -1. Get node total CPU capacity from `resources_get` Node → `status.capacity.cpu` (e.g., "32" = 32 vCPUs) -2. For each VM on node, get allocated vCPUs from VMI → `spec.domain.cpu.sockets × spec.domain.cpu.cores × spec.domain.cpu.threads` -3. Sum all VM vCPUs on the node -4. Calculate: (Sum of VM vCPUs / Node CPU capacity) × 100 - -**Memory Percentage Calculation**: -1. Get node total memory capacity from `resources_get` Node → `status.capacity.memory` (e.g., "128Gi") -2. For each VM on node, get allocated memory from VMI → `spec.domain.memory.guest` -3. Sum all VM memory allocations on the node (convert to same units) -4. Calculate: (Sum of VM memory / Node memory capacity) × 100 - -**Example**: Node with 32 vCPUs hosting VMs with 2+4+8+4+2 = 20 vCPUs → CPU = 62.5% (20/32), NOT the actual runtime usage which might be 0% if VMs are idle. - -**Rationale**: Shows **capacity planning** (how much is reserved) rather than runtime utilization, which is more useful for rebalancing decisions. - -**Overcommit Detection and Warning**: - -If any node's CPU or Memory percentage **exceeds 100%** after rebalancing: - -```markdown -⚠️ **OVERCOMMIT WARNING** - -**Node(s) will be overcommitted after this rebalance:** -- **worker-02**: CPU 125% (40 vCPUs allocated / 32 vCPUs capacity) - **25% overcommit** -- **worker-03**: Memory 110% (88Gi allocated / 80Gi capacity) - **10% overcommit** - -**Impact:** -- **CPU overcommit**: VMs may experience CPU throttling and reduced performance when all VMs are active simultaneously -- **Memory overcommit**: Risk of VM eviction or OOM (Out of Memory) if total memory demand exceeds node capacity - -**Recommendations:** -- Consider distributing VMs across more nodes to avoid overcommit -- Review VM instance types to ensure they match actual workload requirements -- Monitor node resource usage closely after rebalancing - -**Proceed with overcommit?** (yes/cancel) -``` - -**When NOT to warn**: If percentages ≤ 100%, overcommit is not present. Omit this warning section. - -**After tables, include:** - -**Key Improvement**: `"Distribution from 1 node to 4 nodes hosting VMs"` or `"CPU variance reduced from 22% to 4% (81% improvement)"` - -**Rebalance Summary** (batch operations): -```markdown -- Total VMs: 5 | Live: 4 | Cold: 1 | Staying: 2 -- Total Downtime: ~40s | Duration: 1-2min (parallel) -``` - -**Execution Mode**: `**Parallel** (default) - all VMs rebalance simultaneously` OR `**Sequential** (user requested)` - -**Terminology Standards**: -- ✅ "VM Rebalance Plan", "Rebalancing", "Live/Cold migration", "Current Node/New Node", "VMs Now/VMs After" -- ❌ "VM Migration Plan" (reserved for future migration skill) - -## Common Error Handling - -### Error 1: Live Migration Fails - Storage Not RWX -**Symptom**: "Cannot live migrate: PVC access mode is ReadWriteOnce" -**Solution**: Use cold migration OR convert PVC to RWX -**Reference**: [../../docs/troubleshooting/storage-errors.md](../../docs/troubleshooting/storage-errors.md) - -### Error 2: VM Stuck ErrorUnschedulable After Cold Migration -**Symptom**: "VM cannot be scheduled: ErrorUnschedulable" -**Solution**: Check node capacity (`nodes_top`), verify no blocking taints (`resources_get` Node), add tolerations, choose different target, remove nodeSelector -**Reference**: [../../docs/troubleshooting/scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) - -### Error 3: Live Migration Times Out -**Symptom**: "Migration exceeded timeout: 150s per GiB" -**Solution**: Retry migration, reduce VM workload, use cold migration, increase timeout in HyperConverged CR -**Reference**: [references/performance-tuning.md](./references/performance-tuning.md) - -### Error 4: Migration Rejected - Cluster Limit Reached -**Symptom**: "Migration rejected: cluster limit reached (5 concurrent)" -**Solution**: Wait for migrations to complete (`resources_list` VirtualMachineInstanceMigration), retry, migrate sequentially, increase limit -**Reference**: [references/performance-tuning.md](./references/performance-tuning.md) - -### Error 5: RBAC Permission Denied -**Symptom**: "Forbidden: User cannot create VirtualMachineInstanceMigration" -**Solution**: Verify RBAC permissions (`create` on VirtualMachineInstanceMigration, `update` on VirtualMachine), contact admin - -### Error 6: Network Saturation -**Symptom**: Multiple migrations slow/fail, high network utilization -**Solution**: Reduce concurrent migrations, set bandwidth limit, use dedicated migration network -**Reference**: [references/performance-tuning.md](./references/performance-tuning.md) - -### Error 7: Resource Version Conflict During Cold Migration -**Symptom**: "Apply failed: conflict with 'kubernetes-mcp-server' using .spec.runStrategy" -**Solution**: After `vm_lifecycle` stop, re-read VM using `resources_get` before updating nodeAffinity (gets fresh resourceVersion) -**Workflow**: Stop → Wait → Re-read → Update nodeAffinity → Start -**Reference**: [REBALANCE_MANUAL.md - Sub-step 4b.2.5](./REBALANCE_MANUAL.md) - -## Dependencies - -### Required MCP Servers -- `openshift-virtualization` - OpenShift MCP server (https://github.com/openshift/openshift-mcp-server) - -### Required MCP Tools -- `resources_list`, `resources_get`, `resources_create_or_update`, `vm_lifecycle`, `nodes_top`, `pods_top`, `nodes_stats_summary` - -### Related Skills -- `vm-inventory` - List VMs and check placement -- `vm-lifecycle-manager` - Simple start/stop -- `vm-create` - Create VMs with placement -- `vm-snapshot-create` - Backup before risky migrations - -### Reference Documentation - -**Skill Strategy Files**: -- [REBALANCE_MANUAL.md](./REBALANCE_MANUAL.md) - User-driven migration -- [REBALANCE_AUTOMATIC.md](./REBALANCE_AUTOMATIC.md) - AI-driven rebalancing - -**Performance and Best Practices**: -- [references/live-migration-best-practices.md](./references/live-migration-best-practices.md) - Configuration, requirements, networks -- [references/performance-tuning.md](./references/performance-tuning.md) - Right-sizing, overcommit, bandwidth -- [references/anti-patterns.md](./references/anti-patterns.md) - Common mistakes -- [references/production-considerations.md](./references/production-considerations.md) - HA, capacity, security - -**Troubleshooting**: -- [../../docs/troubleshooting/INDEX.md](../../docs/troubleshooting/INDEX.md) - Master index -- [../../docs/troubleshooting/scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) - ErrorUnschedulable, taints -- [../../docs/troubleshooting/storage-errors.md](../../docs/troubleshooting/storage-errors.md) - PVC access modes -- [../../docs/troubleshooting/lifecycle-errors.md](../../docs/troubleshooting/lifecycle-errors.md) - VM start/stop - -**Official Documentation**: -- [OpenShift Virt - Live Migration](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt-live-migration) -- [OpenShift Virt - Node Placement](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt-node-placement) -- [KubeVirt - Live Migration](https://kubevirt.io/user-guide/compute/live_migration/) -- [KubeVirt - Node Assignment](https://kubevirt.io/user-guide/compute/node_assignment/) -- [VirtualMachineInstanceMigration API](https://kubevirt.io/api-reference/main/definitions.html#_v1_virtualmachineinstancemigration) - -## Critical: Human-in-the-Loop Requirements - -**IMPORTANT**: This skill performs VM migrations affecting placement and availability. You MUST: - -1. **Before Initiating Migration** - - Present complete rebalance plan (VM, nodes, type, impact) - - Explain downtime (live = <1s pause, cold = 30-60s) - - Show current vs target placement - - Ask: "Confirm this migration?" - - Wait for explicit confirmation - -2. **Never Auto-Execute** - - **NEVER migrate without confirmation** - - **NEVER assume live vs cold** - ask or infer from storage - - **NEVER skip impact explanation** - - **NEVER proceed if validation fails** - -3. **For Batch Operations** - - Present all VMs to migrate - - Show total impact (e.g., "3 VMs, 2 live + 1 cold") - - Confirm entire batch before starting - - Report progress for each - - Stop on first failure - -**Why**: Live migration (brief pause, bandwidth, performance impact), Cold migration (downtime, dropped connections), Wrong node (performance degradation), Batch (network saturation) - -**Rationale**: Prevents unintended disruption; maintains user control. - -## Security Considerations - -- **RBAC Enforcement**: Requires specific permissions (create/update/list) -- **Node Access**: Respects node taints and RBAC policies -- **Storage Security**: Data remains encrypted if using encrypted storage classes -- **Network Isolation**: Migrations respect NetworkPolicies -- **Audit Trail**: All operations logged in Kubernetes API audit logs -- **KUBECONFIG Security**: Credentials never exposed -- **Resource Quotas**: Respects namespace quotas -- **Tenant Isolation**: Cannot migrate across namespaces without RBAC - ---- - -**Strategy Implementation**: ✅ REBALANCE_MANUAL.md | ✅ REBALANCE_AUTOMATIC.md - -**Reference Documentation**: ✅ live-migration-best-practices.md | ✅ performance-tuning.md | ✅ anti-patterns.md | ✅ production-considerations.md - -**Last Updated**: 2026-02-24 | **OpenShift Virtualization**: 4.17, 4.18, 4.19, 4.20 diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/anti-patterns.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/anti-patterns.md deleted file mode 100644 index f1321fdc..00000000 --- a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/anti-patterns.md +++ /dev/null @@ -1,869 +0,0 @@ -# Anti-Patterns: What NOT to Do - -**Purpose**: Common mistakes, anti-patterns, and pitfalls to avoid when rebalancing VMs in OpenShift Virtualization. - -**When to consult this document**: Before planning rebalancing operations, when troubleshooting failures, or when designing cluster architecture. - ---- - -## Official Sources - -This document is compiled from official Red Hat documentation and community best practices: - -- [Best Practices for Virtual Machine Deployments on OpenShift Virtualization](https://learn.microsoft.com/en-us/azure/openshift/best-practices-openshift-virtualization) - Microsoft Azure Red Hat OpenShift (2026-02-16) -- [Best Practices to Deploy VMs in Red Hat OpenShift Virtualization](https://docs.netapp.com/us-en/netapp-solutions-virtualization/openshift/os-osv-bpg.html) - NetApp Solutions -- [OpenShift Virtualization Best Practices](https://www.tigera.io/learn/guides/kubernetes-networking/openshift-virtualization/) - Tigera -- [Troubleshooting OpenShift Virtualization](https://access.redhat.com/articles/6256861) - Red Hat Customer Portal - ---- - -## Storage Anti-Patterns - -### ❌ Anti-Pattern 1: Using RWO Storage for Live Migration - -**What NOT to Do:** -```yaml -# BAD: VM using ReadWriteOnce storage -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: vm-disk -spec: - accessModes: - - ReadWriteOnce # Cannot live migrate! - storageClassName: gp3 -``` - -**Why It Fails:** - -From Red Hat documentation: -> "Live migration requires the use of a shared storage solution that provides ReadWriteMany (RWX) access mode. The VM disks should be backed by storage option that provides RWX access mode." - -**Error Message:** -``` -cannot migrate VMI: PVC vm-disk is not shared, live migration requires -that all PVCs must be shared (using ReadWriteMany access mode) -``` - -**Correct Approach:** - -**Before Planning Live Migration**, verify storage using MCP tools: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.spec.accessModes` includes `"ReadWriteMany"`. - -**If RWO storage**, use **cold migration** instead (see REBALANCE_MANUAL.md). - -**Storage Types Supporting RWX:** -- ✅ NFS (ontap-nas driver) -- ✅ SMB/CIFS (ontap-nas driver) -- ✅ iSCSI/FC (ontap-san driver, **raw block mode only**) -- ❌ AWS EBS gp3 (RWO only) -- ❌ Local storage (RWO only) - ---- - -### ❌ Anti-Pattern 2: Not Setting SVM Volume Limits - -**What NOT to Do:** - -Deploy Trident without configuring SVM (Storage Virtual Machine) volume limits, allowing unchecked resource consumption. - -**Why It's Dangerous:** - -From NetApp documentation: -> "Set volume limits to prevent Trident from consuming all storage" - -**Impact:** -- Trident creates unlimited volumes -- Storage backend exhausted -- Other workloads starved of storage -- Production outages - -**Correct Approach:** - -Configure limits at multiple levels: - -**1. SVM-level volume limit:** -```bash -vserver modify -vserver -max-volumes -``` - -**2. Storage limits on SVM:** -```bash -vserver create -vserver vserver_name -aggregate aggregate_name -storage-limit value -``` - -**3. Trident backend parameters:** -- `limitVolumeSize`: Maximum volume size created by Trident (e.g., "100Gi") -- `limitVolumePoolSize`: Maximum FlexVol size for economy drivers (e.g., "500Gi") - ---- - -### ❌ Anti-Pattern 3: Enabling showmount Without Justification - -**What NOT to Do:** - -Leave `showmount` enabled on NFS SVMs, exposing volume information to unauthorized clients. - -**Why It's a Security Risk:** - -From NetApp documentation: -> "Disable showmount to prevent unauthorized volume discovery" - -**Correct Approach:** - -Disable showmount unless specifically required: - -```bash -vserver nfs modify -vserver -showmount disabled -``` - -Implement separate export policies for infrastructure vs application nodes for granular access control. - ---- - -## Scheduling and Node Placement Anti-Patterns - -### ❌ Anti-Pattern 4: Excessive Affinity Rules - -**What NOT to Do:** - -Apply too many complex affinity, anti-affinity, node selector, and toleration rules to VMs. - -**Why It's Problematic:** - -From community best practices: -> "Too many rules make scheduling slow and hard to reason about." - -From Red Hat documentation: -> "Affinity rules only apply during scheduling. OpenShift Container Platform does not reschedule running workloads if the constraints are no longer met." - -**Impact:** -- Slow VM scheduling (scheduler overhead) -- Impossible-to-satisfy constraints (VM stuck in Pending) -- Difficult troubleshooting (complex rule interactions) -- No automatic rebalancing when constraints violated - -**Correct Approach:** - -**Keep rules simple and minimal:** - -```yaml -# GOOD: Simple, clear node selector -spec: - template: - spec: - nodeSelector: - workload-type: virtualization -``` - -```yaml -# BAD: Too many overlapping constraints -spec: - template: - spec: - nodeSelector: - node-role.kubernetes.io/worker: "" - workload-type: virtualization - zone: us-east-1a - instance-type: m5.4xlarge - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: [web, database, cache] - topologyKey: kubernetes.io/hostname - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: tier - operator: In - values: [frontend] - topologyKey: failure-domain.beta.kubernetes.io/zone - tolerations: - - key: dedicated - operator: Equal - value: virtualization - effect: NoSchedule - - key: high-performance - operator: Exists - effect: NoSchedule -``` - -**Recommendation:** -- Use **one** primary constraint (nodeSelector OR affinity) -- Add tolerations only when nodes have taints -- Avoid mixing required and preferred affinity rules -- Document the intent of each rule - ---- - -### ❌ Anti-Pattern 5: Not Planning for Node Failures - -**What NOT to Do:** - -Rely on automatic VM failover without configuring machine health checks. - -**Why It Fails:** - -From Red Hat documentation: -> "If a node fails and machine health checks are not deployed on your cluster, virtual machines (VMs) with RunStrategy: Always configured are not automatically relocated to healthy nodes. To trigger VM failover, you must manually delete the Node object." - -**Impact:** -- VMs remain assigned to failed node -- Manual intervention required for recovery -- Extended downtime during node failures - -**Correct Approach:** - -**1. Deploy Machine Health Checks:** - -Configure cluster-level machine health checks to detect and remediate node failures automatically. - -**2. Use RunStrategy: Always for HA VMs:** - -```yaml -spec: - runStrategy: Always # Ensures VM restarts after node recovery -``` - -**3. Implement VM Replication:** - -For critical VMs, create replicas with anti-affinity rules to ensure distribution across different nodes/zones. - -**4. Monitor Node Health:** - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -Filter for nodes where `.status.conditions[]` shows `Ready=False` or other unhealthy states. - ---- - -## Resource Management Anti-Patterns - -### ❌ Anti-Pattern 6: Exceeding CPU Overcommit Limits - -**What NOT to Do:** - -Configure CPU overcommit ratio >1.8x physical cores. - -**Why It's Dangerous:** - -From Red Hat documentation: -> "CPU over-commitment ratio must not exceed 1.8x of the number of physical cores while memory usage may not exceed 0.9x of the physical memory available in a cluster. CPU over-commitment leads to throttling, causing slowness of all workloads on the impacted node." - -**Impact:** -- CPU throttling across ALL VMs on node -- Unpredictable performance degradation -- Cascading slowness affecting entire cluster -- User-facing application latency - -**Correct Approach:** - -**Check Current Overcommit:** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "hco.kubevirt.io/v1beta1", - "kind": "HyperConverged", - "namespace": "openshift-cnv", - "name": "kubevirt-hyperconverged" -} -``` - -Review `.spec.resourceRequirements.vmiCPUAllocationRatio`. - -**Safe Limits:** -- **Production**: 1.0-1.2x (no/minimal overcommit) -- **Dev/Test**: 1.2-1.5x (moderate overcommit) -- **Absolute Maximum**: 1.8x (with careful monitoring) - -**Never Exceed**: 1.8x CPU or 0.9x memory limits. - ---- - -### ❌ Anti-Pattern 7: Applying Strict Resource Limits to VMs - -**What NOT to Do:** - -Set both resource requests **and** limits on VMs without specific governance requirements. - -**Why It's Problematic:** - -From Microsoft Azure Red Hat OpenShift guidance: -> "Avoid strict resource limits: Set only guest memory for VMs; avoid strict resource limits unless required for governance." - -**Impact:** -- CPU throttling even when node has spare capacity -- Reduced VM performance -- Wasted cluster resources -- Difficult troubleshooting (invisible throttling) - -**Correct Approach:** - -**Set requests only:** - -```yaml -# GOOD: Requests only (allows bursting) -spec: - template: - spec: - domain: - resources: - requests: - memory: 16Gi - cpu: 4 -``` - -```yaml -# BAD: Requests + limits (strict throttling) -spec: - template: - spec: - domain: - resources: - requests: - memory: 16Gi - cpu: 4 - limits: # Avoid unless required - memory: 16Gi - cpu: 4 -``` - -**Only set limits when:** -- Governance policies mandate strict resource boundaries -- Multi-tenant environments require isolation -- Preventing one VM from starving others - ---- - -### ❌ Anti-Pattern 8: Relying on On-Premises Sizing References - -**What NOT to Do:** - -Size VMs in OpenShift Virtualization based on on-premises VM sizes without testing. - -**Why It Fails:** - -From Microsoft Azure Red Hat OpenShift guidance: -> "Avoid relying solely on on-premises sizing references; benchmark your own workloads to inform right sizing." - -**Impact:** -- Overprovisioned VMs (wasted resources) -- Underprovisioned VMs (performance issues) -- Unexpected architectural overhead (VMs != native pods) -- Incorrect migration time estimates - -**Correct Approach:** - -**1. Benchmark workloads in OpenShift Virtualization:** -- Deploy test VMs with various sizes -- Run representative workload tests -- Measure actual performance vs requirements - -**2. Account for architectural overhead:** - -Expect 4-56% performance overhead vs bare metal (see performance-tuning.md for details). - -**3. Monitor and adjust:** - -**MCP Tool**: `pods_top` (from openshift-virtualization) - -**Parameters**: -```json -{ - "all_namespaces": true, - "label_selector": "kubevirt.io=virt-launcher" -} -``` - -Track actual resource usage and resize VMs accordingly. - ---- - -## Network Anti-Patterns - -### ❌ Anti-Pattern 9: Using OVN-Kubernetes with Linux Bridge on Default Interface - -**What NOT to Do:** - -Attempt to attach a Linux bridge or bonding device to the host's default interface when using OVN-Kubernetes CNI. - -**Why It Fails:** - -From Red Hat documentation: -> "If your OpenShift Container Platform cluster uses OVN-Kubernetes as the default CNI provider, you cannot attach a Linux bridge or bonding device to a host's default interface." - -**Impact:** -- Network configuration failures -- VM networking broken -- Migration network setup fails - -**Correct Approach:** - -**Option 1: Use secondary network interface** - -Attach Linux bridge to a different physical interface (not the default). - -**Option 2: Switch to OpenShift SDN CNI** - -If Linux bridge on default interface is required, reconfigure cluster to use OpenShift SDN instead of OVN-Kubernetes. - -**Option 3: Use OVS bridge instead** - -For migration networks, use Open vSwitch bridge (compatible with OVN-Kubernetes). - ---- - -### ❌ Anti-Pattern 10: Ignoring MTU Mismatches - -**What NOT to Do:** - -Mix network types with different default MTUs without explicit configuration. - -**Why It's Problematic:** - -From Red Hat documentation: -> "When a virtual machine interface is connected to an OVS bridge, the default MTU is 1400, but when connected to a Linux bridge, the default MTU is 1500." - -**Impact:** -- Packet fragmentation -- Reduced network performance -- Subtle communication failures -- Migration slowdowns - -**Correct Approach:** - -**Explicitly set MTU in NetworkAttachmentDefinition:** - -```json -{ - "cniVersion": "0.3.1", - "name": "migration-bridge", - "type": "macvlan", - "master": "eth1", - "mode": "bridge", - "mtu": 9000, # Explicit MTU setting - "ipam": {...} -} -``` - -**Validate MTU consistency across all interfaces involved in migration.** - ---- - -## Architecture and Platform Anti-Patterns - -### ❌ Anti-Pattern 11: Using RHEL Compute Nodes - -**What NOT to Do:** - -Deploy OpenShift Virtualization on Red Hat Enterprise Linux (RHEL) compute nodes. - -**Why It Fails:** - -From Red Hat documentation: -> "OpenShift Virtualization requires Red Hat Enterprise Linux CoreOS (RHCOS) compute nodes. Even though it is possible to deploy Red Hat Enterprise Linux (RHEL) compute nodes, they are incompatible with OpenShift Virtualization." - -**Impact:** -- VM scheduling failures -- Unsupported configuration -- Migration failures -- No Red Hat support - -**Correct Approach:** - -**Verify all nodes are RHCOS:** - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node, check `.status.nodeInfo.osImage` contains "CoreOS". - -**If RHEL nodes detected:** -- Replace with RHCOS nodes -- Do NOT schedule VMs on RHEL nodes -- Remove RHEL nodes from cluster before deploying virtualization workloads - ---- - -### ❌ Anti-Pattern 12: Placing Master Nodes on Same VMware Host - -**What NOT to Do:** - -In VMware-based deployments, place multiple OpenShift master nodes on the same VMware ESXi host. - -**Why It's Dangerous:** - -From VMware best practices: -> "Critical best practices include: distributing the 3 virtual master nodes across different VMware hosts, placing each master node on a separate datastore, and avoiding hosting master nodes on datastores with high I/O workloads." - -**Impact:** -- Single point of failure (host failure kills multiple masters) -- etcd performance degradation (etcd is latency-sensitive) -- Cluster control plane outage -- Violates high-availability principles - -**Correct Approach:** - -**1. Distribute master nodes across different VMware hosts** - -Use VM anti-affinity rules to enforce separation. - -**2. Use separate datastores for each master node** - -Prevents storage failure from affecting multiple masters. - -**3. Avoid high I/O datastores for master nodes** - -etcd is sensitive to disk latency; use low-latency storage. - ---- - -### ❌ Anti-Pattern 13: Ignoring etcd Latency Sensitivity - -**What NOT to Do:** - -Place etcd (control plane) on high-latency storage or overloaded nodes. - -**Why It's Critical:** - -From best practices: -> "The etcd component hosted on control-plane nodes is usually the component most sensitive to latency issues." - -**Impact:** -- Cluster control plane slowness -- API server timeouts -- Failed VM operations -- Cluster instability - -**Correct Approach:** - -**1. Use low-latency storage for control plane nodes:** -- SSD-backed storage (not HDD) -- Local NVMe if available -- Avoid shared storage with high I/O contention - -**2. Monitor etcd latency:** - -Prometheus metrics: `etcd_disk_wal_fsync_duration_seconds` - -**Target**: <10ms for WAL fsync - -**3. Isolate control plane from VM workloads:** - -Use taints on master nodes to prevent VM scheduling. - ---- - -## Migration Operation Anti-Patterns - -### ❌ Anti-Pattern 14: Not Reducing VM Workload During Migration - -**What NOT to Do:** - -Attempt live migration of write-intensive VMs (databases, caches) under full load. - -**Why It's Problematic:** - -High memory write rate (dirty page rate) can exceed network transfer rate, preventing migration convergence. - -**Impact:** -- Migration timeouts -- Failed migrations -- Extended migration duration -- Network saturation - -**Correct Approach:** - -**Before migrating write-intensive VMs:** - -1. **Schedule migration during low-activity window** (off-hours, maintenance window) - -2. **Temporarily reduce workload:** - - Stop non-critical background processes - - Scale down application traffic - - Pause batch jobs - -3. **Consider cold migration instead** for extremely write-heavy workloads (guaranteed completion) - -4. **Increase timeouts if load cannot be reduced:** - -Modify `.spec.liveMigrationConfig.completionTimeoutPerGiB` in HyperConverged CR (see performance-tuning.md). - ---- - -### ❌ Anti-Pattern 15: Parallel Migrations Without Dedicated Network - -**What NOT to Do:** - -Run many concurrent migrations on shared application network without bandwidth limits. - -**Why It's Dangerous:** - -- Saturates network bandwidth -- Degrades application performance -- Migration failures due to slow transfers -- Cascading performance impact - -**Impact Observed:** - -From search results: -> "Network saturation risk with concurrent migrations" - -**Correct Approach:** - -**Option 1: Use dedicated migration network** (see live-migration-best-practices.md) - -**Option 2: Limit concurrent migrations:** - -Modify `.spec.liveMigrationConfig.parallelMigrationsPerCluster` in HyperConverged CR: - -```yaml -spec: - liveMigrationConfig: - parallelMigrationsPerCluster: 3 # Conservative limit - bandwidthPerMigration: 64Mi # Bandwidth cap per migration -``` - -**Option 3: Migrate sequentially** - -Migrate VMs one at a time instead of batch operations. - ---- - -### ❌ Anti-Pattern 16: Not Validating Migration Prerequisites - -**What NOT to Do:** - -Attempt migration without verifying storage, network, and capacity prerequisites. - -**Why It Fails:** - -Common failures: -- RWO storage → "PVC is not shared" error -- VM not running → "cannot migrate stopped VM" -- Node at capacity → ErrorUnschedulable -- Network issues → Migration timeout - -**Correct Approach:** - -**Always run pre-migration validation** (see live-migration-best-practices.md for complete checklist): - -1. ✅ Verify PVC access modes (RWX required) -2. ✅ Check VM is running (VMI exists) -3. ✅ Validate target node capacity -4. ✅ Confirm virt-handler pods healthy -5. ✅ Check cluster migration limits - -**Use Common Validation Logic from SKILL.md before every migration.** - ---- - -## Production Deployment Anti-Patterns - -### ❌ Anti-Pattern 17: Deploying to Production Without Testing - -**What NOT to Do:** - -Deploy VMs directly to production without dev/test validation. - -**Why It's Risky:** - -From best practices: -> "Begin with non-critical or dev/test workloads before moving production systems - this phased approach allows teams to gain hands-on experience while minimizing risk." - -**Impact:** -- Unexpected performance issues -- Migration failures affecting production -- Learning curve impacts critical systems -- Difficult rollback - -**Correct Approach:** - -**Phased Rollout:** - -**Phase 1: Development/Test** -- Deploy test VMs -- Validate performance and functionality -- Test migration workflows -- Gain operational experience - -**Phase 2: Non-Critical Production** -- Migrate non-critical workloads -- Monitor performance and stability -- Refine sizing and configurations -- Build confidence - -**Phase 3: Critical Production** -- Migrate critical workloads -- Ensure HA and DR configured -- 24/7 monitoring in place -- Rollback plan ready - ---- - -### ❌ Anti-Pattern 18: Starting Big Instead of Small - -**What NOT to Do:** - -Provision large VM fleet from day one without iterative growth. - -**Why It's Problematic:** - -From best practices: -> "When starting with OpenShift Virtualization, it's essential to start small and scale up as needed to avoid over-provisioning and wasting resources." - -**Impact:** -- Overprovisioned cluster (wasted costs) -- Underutilized resources -- Difficult rightsizing later -- Commitment to suboptimal architecture - -**Correct Approach:** - -**Start small:** -1. Deploy 5-10 VMs initially -2. Monitor resource usage patterns -3. Adjust sizing based on actual metrics -4. Gradually add VMs as needs grow - -**Validate assumptions:** -- Test architectural overhead -- Measure actual performance -- Refine resource allocation -- Iterate on configuration - ---- - -### ❌ Anti-Pattern 19: Not Monitoring After Rebalancing - -**What NOT to Do:** - -Execute rebalancing operations and assume everything is optimal without validation. - -**Why It's Risky:** - -- May not achieve intended load distribution -- Hidden performance degradation -- VMs scheduled suboptimally -- Resource contention not detected - -**Correct Approach:** - -**Post-Rebalancing Validation:** - -**1. Verify VM placement:** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -For each migrated VM, confirm `.status.nodeName` matches expected target node. - -**2. Monitor node resource usage:** - -**MCP Tool**: `nodes_top` (from openshift-virtualization) - -**Before vs After Comparison:** - -| Node | CPU Before | CPU After | Improvement | -|------|------------|-----------|-------------| -| worker-01 | 85% | 65% | -20% ✓ | -| worker-02 | 78% | 64% | -14% ✓ | -| worker-03 | 42% | 58% | +16% | -| worker-04 | 38% | 53% | +15% | - -**3. Validate application performance:** - -Check application-specific metrics (response time, throughput, error rates). - -**4. Monitor for 24-48 hours:** - -Ensure sustained improvement without unexpected side effects. - ---- - -## Summary: Anti-Pattern Checklist - -Before rebalancing VMs, avoid these critical mistakes: - -**Storage:** -- ❌ Using RWO storage for live migration -- ❌ Not setting SVM volume limits -- ❌ Leaving showmount enabled - -**Scheduling:** -- ❌ Too many complex affinity rules -- ❌ Not configuring machine health checks - -**Resources:** -- ❌ CPU overcommit >1.8x -- ❌ Strict resource limits without justification -- ❌ Using on-premises sizing without testing - -**Network:** -- ❌ Linux bridge on default interface with OVN-Kubernetes -- ❌ Ignoring MTU mismatches -- ❌ Parallel migrations without dedicated network - -**Platform:** -- ❌ Using RHEL compute nodes -- ❌ Master nodes on same VMware host -- ❌ Ignoring etcd latency sensitivity - -**Operations:** -- ❌ Not reducing VM workload during migration -- ❌ Skipping pre-migration validation -- ❌ No post-rebalancing monitoring - -**Production:** -- ❌ Deploying to production without testing -- ❌ Starting big instead of small - ---- - -## Related Documentation - -- [Live Migration Best Practices](./live-migration-best-practices.md) - What TO do for successful migrations -- [Performance Tuning](./performance-tuning.md) - Optimization strategies -- [Production Considerations](./production-considerations.md) - Right-sizing, workload planning, HA strategies - ---- - -**Last Updated**: 2026-02-24 -**OpenShift Virtualization Versions**: 4.17, 4.18, 4.19, 4.20 -**Status**: Curated from official Red Hat sources and production experience diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/live-migration-best-practices.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/live-migration-best-practices.md deleted file mode 100644 index 7df97527..00000000 --- a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/live-migration-best-practices.md +++ /dev/null @@ -1,794 +0,0 @@ -# Live Migration Best Practices - -**Purpose**: Configuration parameters, requirements, and best practices for VM live migration in OpenShift Virtualization. - -**When to consult this document**: Before executing live migrations, when configuring cluster-wide migration settings, or when troubleshooting migration performance issues. - ---- - -## Official Sources - -This document is compiled from official Red Hat documentation: - -- [Live Migrating VMs with OpenShift Virtualization](https://developers.redhat.com/articles/2025/07/14/live-migrating-vms-openshift-virtualization) - Red Hat Developer (2025-07-14) -- [How OpenShift Virtualization Supports VM Live Migration](https://developers.redhat.com/articles/2025/06/05/how-openshift-virtualization-supports-vm-live-migration) - Red Hat Developer (2025-06-05) -- [Chapter 12. Live Migration - OpenShift Container Platform 4.18](https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/virtualization/live-migration) - Red Hat Documentation -- [Best Practices for Virtual Machine Deployments on OpenShift Virtualization](https://learn.microsoft.com/en-us/azure/openshift/best-practices-openshift-virtualization) - Microsoft Azure Red Hat OpenShift (2026-02-16) -- [Best Practices to Deploy VMs in Red Hat OpenShift Virtualization](https://docs.netapp.com/us-en/netapp-solutions-virtualization/openshift/os-osv-bpg.html) - NetApp Solutions - ---- - -## Configuration Parameters - -### HyperConverged CR Live Migration Settings - -All live migration settings are configured in the `HyperConverged` custom resource located in the `openshift-cnv` namespace. - -**Default Configuration:** -```yaml -apiVersion: hco.kubevirt.io/v1beta1 -kind: HyperConverged -metadata: - name: kubevirt-hyperconverged - namespace: openshift-cnv -spec: - liveMigrationConfig: - completionTimeoutPerGiB: 800 # Seconds per GiB for migration completion - parallelMigrationsPerCluster: 5 # Max concurrent migrations cluster-wide - parallelOutboundMigrationsPerNode: 2 # Max concurrent migrations per source node - progressTimeout: 150 # Max seconds without progress before cancellation - bandwidthPerMigration: 64Mi # (Optional) Bandwidth limit per migration - network: "" # (Optional) Dedicated secondary network for migration -``` - -**Parameter Explanations:** - -| Parameter | Default | Description | Tuning Guidance | -|-----------|---------|-------------|-----------------| -| `completionTimeoutPerGiB` | 800s | Migration completion duration per gigabyte of VM memory | Increase for high memory write rate (dirty page) workloads | -| `progressTimeout` | 150s | Maximum seconds without migration progress before cancellation | Increase for large VMs (>100GB) or slow networks | -| `parallelMigrationsPerCluster` | 5 | Cluster-wide concurrent migration limit | Increase if network bandwidth allows; decrease if saturation occurs | -| `parallelOutboundMigrationsPerNode` | 2 | Per-node concurrent outbound migration limit | Keep at 2 to prevent single-node overload | -| `bandwidthPerMigration` | 64Mi | (Optional) Bandwidth limit per migration | Set to prevent network saturation; omit for unlimited | -| `network` | "" | (Optional) NetworkAttachmentDefinition for dedicated migration network | Highly recommended for production; see Dedicated Networks section | - -**How to Update Configuration Using MCP Tools:** - -**Step 1: Get current HyperConverged resource** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "hco.kubevirt.io/v1beta1", - "kind": "HyperConverged", - "namespace": "openshift-cnv", - "name": "kubevirt-hyperconverged" -} -``` - -**Step 2: Modify the returned JSON to update liveMigrationConfig** - -Add or update the `.spec.liveMigrationConfig` section: -```json -{ - "spec": { - "liveMigrationConfig": { - "completionTimeoutPerGiB": 1200, - "parallelMigrationsPerCluster": 10, - "progressTimeout": 300, - "bandwidthPerMigration": "32Mi" - } - } -} -``` - -**Step 3: Apply the updated configuration** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters**: -```json -{ - "resource": "" -} -``` - -Pass the complete modified HyperConverged resource as YAML or JSON string. - ---- - -## Prerequisites and Requirements - -### Storage Requirements - -**CRITICAL**: Live migration requires **ReadWriteMany (RWX)** access mode storage. - -**Supported Storage Types for Live Migration:** - -| Storage Type | Access Mode | Live Migration Support | Notes | -|--------------|-------------|------------------------|-------| -| NFS (ontap-nas driver) | RWX | ✅ Supported | Recommended for general use | -| SMB/CIFS (ontap-nas driver) | RWX | ✅ Supported | Windows-compatible | -| iSCSI/FC (ontap-san driver) | RWX (raw block mode only) | ✅ Supported | High performance; requires raw block volumes | -| Local storage / AWS EBS (gp3) | RWO | ❌ NOT Supported | Use cold migration instead | - -**Validation Using MCP Tools:** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -**Expected Output**: Check `.spec.accessModes` in the returned PVC resource. - -For live migration, access modes must include `"ReadWriteMany"`. - -**What Happens with RWO Storage:** - -When attempting live migration with ReadWriteOnce (RWO) storage: -``` -Error: cannot migrate VMI: PVC is not shared, live migration requires -that all PVCs must be shared (using ReadWriteMany access mode) -``` - -**Solution**: Use cold migration workflow for VMs with RWO storage (see REBALANCE_MANUAL.md). - ---- - -### Hardware and Network Requirements - -**Minimum Requirements:** - -- **Nodes**: Red Hat Enterprise Linux CoreOS (RHCOS) compute nodes (RHEL nodes are incompatible) -- **Network**: All nodes must be on the same L2 network or have routable connectivity -- **CPU**: Sufficient CPU headroom on target node for incoming VM workload -- **Memory**: Sufficient free memory on target node (>= VM memory allocation) - -**Recommended for Production:** - -- **Network Cards**: 100Gbps NICs for large VM migrations (>500GB memory) -- **Dedicated Migration Network**: Secondary physical network or VLAN for isolation -- **Storage Backend**: SSD-backed shared storage (NFS-CSI, OpenShift Data Foundation, Azure NetApp Files) -- **MTU Configuration**: Set to 9000 for migration networks to improve efficiency - ---- - -## Dedicated Migration Network (Production Best Practice) - -### Why Use a Dedicated Network? - -**Benefits:** -- Isolates migration traffic from application workloads -- Prevents network contention and performance degradation -- Enables higher bandwidth allocation (e.g., 100Gbps dedicated) -- Improves security and manageability -- Reduces migration time for large VMs - -**When to Use:** -- Production environments with large VMs (>100GB memory) -- Clusters with high application network traffic -- Environments requiring strict network isolation -- High-availability requirements with frequent migrations - -### Configuration Example - -**Step 1: Create NodeNetworkConfigurationPolicy (NNCP)** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters**: -```json -{ - "resource": "apiVersion: nmstate.io/v1\nkind: NodeNetworkConfigurationPolicy\nmetadata:\n name: migration-network-policy\nspec:\n desiredState:\n interfaces:\n - name: br-lm\n description: OVS bridge for live migration\n type: ovs-bridge\n state: up\n bridge:\n allow-extra-patch-ports: true\n port:\n - name: enp4s0\n vlan:\n mode: access\n tag: 3030\n options:\n stp: false" -} -``` - -**Step 2: Create NetworkAttachmentDefinition (NAD)** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters**: -```json -{ - "resource": "apiVersion: k8s.cni.cncf.io/v1\nkind: NetworkAttachmentDefinition\nmetadata:\n name: migration-network\n namespace: openshift-cnv\nspec:\n config: '{\n \"cniVersion\": \"0.3.1\",\n \"name\": \"migration-bridge\",\n \"type\": \"macvlan\",\n \"master\": \"eth1\",\n \"mode\": \"bridge\",\n \"ipam\": {\n \"type\": \"whereabouts\",\n \"range\": \"10.200.5.0/24\",\n \"excludeSubnets\": \"10.200.5.0/30\"\n }\n }'" -} -``` - -**Step 3: Configure HyperConverged CR to Use Network** - -**MCP Tool**: Get current HyperConverged, modify, and update using `resources_create_or_update` - -Add to `.spec.liveMigrationConfig`: -```json -{ - "network": "migration-network" -} -``` - -**Step 4: Verify virt-handler Pods Restarted** - -**MCP Tool**: `pods_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "labelSelector": "kubevirt.io=virt-handler" -} -``` - -**Expected**: All pods show READY status and recent start time (AGE). - -Filter results where `status.containerStatuses[0].ready == true` and `status.containerStatuses[0].restartCount` is recent. - -**Verification After Migration:** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstanceMigration", - "namespace": "", - "name": "" -} -``` - -Check `.status.migrationState.targetNodeAddress` - should be an IP from the dedicated subnet (e.g., 10.200.5.15). - ---- - -## Migration Process and Technologies - -### Pre-copy Migration - -Live migration uses **pre-copy** strategy: - -1. **Initial Copy**: VM continues running on source node while memory is copied to target -2. **Iterative Copy**: Pages modified during copy (dirty pages) are re-copied -3. **Cutover**: Brief pause (<1 second) to copy final dirty pages and switch execution -4. **Cleanup**: Source VM instance is terminated - -**Multi-fd Technology** (for high-load scenarios): - -- Sends data over multiple network streams in parallel -- Maximizes bandwidth utilization during migration -- Handles high dirty page rates (e.g., SAP HANA, databases with high write rates) -- Automatically enabled by KubeVirt when beneficial - -**Migration Phases:** - -``` -Pending → Scheduling → PreparingTarget → Running → Succeeded -``` - -**Monitor with MCP Tools:** - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstanceMigration" -} -``` - -Filter results by `.status.phase` to see current migration status. - ---- - -## Best Practices - -### 1. VM Resource Optimization - -**Enable Dedicated Resources:** - -Configure VMs with dedicated CPU and memory isolation for performance-sensitive workloads: - -```yaml -spec: - template: - spec: - domain: - cpu: - dedicatedCpuPlacement: true - resources: - requests: - memory: 16Gi -``` - -**Benefits:** -- Improves VM performance and latency predictability -- Reduces migration time (less CPU contention) -- Better accuracy for latency predictions - -**When to Use:** -- Database workloads (PostgreSQL, MySQL, SAP HANA) -- Real-time analytics applications -- Low-latency requirements - -### 2. Hugepage Configuration - -For large VMs (>100GB memory), configure hugepages to reduce memory page overhead: - -**Node Configuration Using MCP Tools:** - -**MCP Tool**: `resources_get` then `resources_create_or_update` (from openshift-virtualization) - -**Step 1: Get Node resource** - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node", - "name": "" -} -``` - -**Step 2: Add label to node** - -Modify returned JSON to add `.metadata.labels.cpumanager = "true"`, then update with `resources_create_or_update`. - -**VM Configuration:** -```yaml -spec: - template: - spec: - domain: - memory: - hugepages: - pageSize: 1Gi -``` - -**Benefits:** -- Reduces page-dirtying overhead during migration -- Improves memory access performance -- Faster migration completion for very large VMs (tested with 1TB VMs) - -### 3. Network Optimization - -**Set Network MTU to 9000** (jumbo frames): - -Configure in the NetworkAttachmentDefinition used for migration: - -```yaml -spec: - liveMigrationConfig: - network: migration-network # NetworkAttachmentDefinition with MTU 9000 -``` - -**Benefits:** -- Significantly improves network efficiency -- Reduces packet overhead -- Faster data transfer for large VM migrations - -**Validate MTU Setting Using MCP Tools:** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "openshift-cnv", - "name": "migration-network" -} -``` - -Check `.spec.config` for MTU setting in the JSON configuration. - -### 4. Storage Configuration - -**For Testing/Development:** -- NFS-CSI with SSD backend storage -- Shared storage accessible from all nodes - -**For Production:** -- OpenShift Data Foundation (ODF) with SSD-backed storage -- Azure NetApp Files with appropriate performance tier -- NetApp ONTAP with dedicated SVM for virtualization workloads - -**Storage Validation Before Migration Using MCP Tools:** - -For each VM in rebalance plan: - -**Step 1: Get VM resource** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Step 2: Extract PVC names from `.spec.template.spec.volumes[].persistentVolumeClaim.claimName`** - -**Step 3: For each PVC, verify access mode** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -**Expected**: `.spec.accessModes` must include `"ReadWriteMany"`. - -### 5. Concurrency Management - -**Default Limits:** -- **Cluster-wide**: 5 concurrent migrations -- **Per-node outbound**: 2 concurrent migrations - -**When to Increase:** -- Cluster has sufficient network bandwidth (100Gbps+ NICs) -- Dedicated migration network is configured -- Routine maintenance window with many VMs to migrate - -**When to Decrease:** -- Network saturation detected (monitor with Prometheus) -- Migration failures due to timeouts -- Shared application network (no dedicated migration network) - -**Monitoring Network Saturation Using MCP Tools:** - -**MCP Tool**: `nodes_stats_summary` (from openshift-virtualization) - -**Parameters**: -```json -{ - "name": "" -} -``` - -Review `.network.interfaces[].rxBytes` and `.network.interfaces[].txBytes` for throughput metrics. - -Alternatively, use `nodes_top` for current resource usage: - -**MCP Tool**: `nodes_top` (from openshift-virtualization) - -**Parameters**: -```json -{ - "name": "" -} -``` - -### 6. Pre-Migration Validation Checklist - -Before initiating migration: - -1. ✅ **Storage**: Verify all PVCs use ReadWriteMany (RWX) access mode -2. ✅ **Network**: Confirm all nodes are network-accessible -3. ✅ **Capacity**: Verify target node has sufficient CPU and memory -4. ✅ **Health**: Check `virt-handler` pods are Running (1/1) on all nodes -5. ✅ **Workload**: Consider VM workload intensity (reduce load if possible) -6. ✅ **Limits**: Check current cluster migration count < `parallelMigrationsPerCluster` - -**Validation Using MCP Tools:** - -**1. Check PVC Access Modes:** - -For each VM, use `resources_get` to get VirtualMachine, extract PVC names, then `resources_get` for each PVC and verify `.spec.accessModes` includes `"ReadWriteMany"`. - -**2. Check virt-handler Health:** - -**MCP Tool**: `pods_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "labelSelector": "kubevirt.io=virt-handler" -} -``` - -Filter results where `status.containerStatuses[0].ready == true`. All pods must show ready status. - -**3. Check Current Migration Count:** - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstanceMigration" -} -``` - -Count results where `.status.phase` is NOT "Succeeded" or "Failed". Compare to `parallelMigrationsPerCluster` limit from HyperConverged CR. - -**4. Check Target Node Capacity:** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node", - "name": "" -} -``` - -Review `.status.allocatable` and `.status.capacity` for available CPU and memory. - -Alternatively use `nodes_top`: - -**MCP Tool**: `nodes_top` (from openshift-virtualization) - -**Parameters**: -```json -{ - "name": "" -} -``` - ---- - -## Test Results and Validation - -### SAP HANA 1TB VM Live Migration (Red Hat Developer Article 2025-07-14) - -**Test Environment:** -- **Hardware**: 8-socket Intel Xeon Platinum, 12TB memory, 100Gbps NICs -- **OpenShift**: 4.17.15 -- **VM Size**: 1TB memory (SAP HANA 2.00.081.00.1733303410) -- **Storage**: NFS-CSI with SSD backend -- **Network**: Dedicated 100Gbps secondary network, MTU 9000 - -**Results:** -- ✅ **Idle/Cooled-off**: Live migration completed successfully with **zero data loss or corruption** -- ✅ **High-load**: Migrations progressed as expected even with large volumes of dirty pages -- ✅ **Integrity**: Full VM and data integrity maintained; failed migrations safely canceled -- ⏱️ **Duration**: ~30-60 seconds for typical VMs; longer for 1TB VM under load - -**Key Findings:** -- Multi-fd technology enabled migrations to continue transferring data quickly while dirty pages were being generated -- Dedicated 100Gbps network critical for large VM migrations -- 1GB hugepages reduced page-dirtying overhead - ---- - -## Limitations and Constraints - -### Migration Requirements - -**MUST HAVE for Live Migration:** -- ReadWriteMany (RWX) storage on all VM volumes -- VM currently running (VirtualMachineInstance exists) -- Target node has sufficient capacity (CPU, memory) -- All nodes have RHCOS (not RHEL) - -**CANNOT Live Migrate When:** -- VM uses ReadWriteOnce (RWO) storage → Use cold migration -- VM is stopped (no VirtualMachineInstance) → Use cold migration or start VM first -- Target node is cordoned or NotReady → Choose different target -- Cluster at `parallelMigrationsPerCluster` limit → Wait for completion - -### Known Issues - -**Single Node OpenShift (SNO):** -- VMs created from common templates with `evictionStrategy: LiveMigrate` trigger `VMCannotBeEvicted` alert -- **Workaround**: Use `evictionStrategy: None` for SNO clusters - -**OVN-Kubernetes CNI:** -- Cannot attach Linux bridge or bonding device to host's default interface -- **Workaround**: Use secondary network interface or switch to OpenShift SDN CNI - -**MTU Differences:** -- OVS bridge default MTU: 1400 -- Linux bridge default MTU: 1500 -- **Impact**: May cause fragmentation; configure MTU explicitly - ---- - -## Troubleshooting Common Issues - -### Issue 1: Migration Timeout - -**Symptom:** -``` -Migration exceeded timeout: 150 seconds per GiB -``` - -**Causes:** -- High memory write rate (dirty page rate exceeds transfer rate) -- Insufficient network bandwidth -- Large VM memory size - -**Solutions:** - -**1. Increase timeout (temporary):** - -Use `resources_get` to fetch HyperConverged CR, modify `.spec.liveMigrationConfig`, then update with `resources_create_or_update`: - -```json -{ - "spec": { - "liveMigrationConfig": { - "completionTimeoutPerGiB": 1200, - "progressTimeout": 300 - } - } -} -``` - -**2. Reduce VM workload** during migration: -- Stop write-intensive processes temporarily -- Schedule migration during low-activity window - -**3. Use cold migration** instead (guaranteed completion - see REBALANCE_MANUAL.md) - -**4. Configure auto-converge** (cluster-level KubeVirt setting): -- Throttles vCPU to reduce dirty page rate -- Enables migration convergence for high write-rate VMs - -### Issue 2: Network Saturation - -**Symptom:** -- Multiple concurrent migrations slow or fail -- High network utilization on migration network - -**Solutions:** - -**1. Reduce concurrent migrations:** - -Use `resources_get` to fetch HyperConverged CR, modify `.spec.liveMigrationConfig.parallelMigrationsPerCluster`, then update with `resources_create_or_update`: - -```json -{ - "spec": { - "liveMigrationConfig": { - "parallelMigrationsPerCluster": 3 - } - } -} -``` - -**2. Set bandwidth limit per migration:** - -Modify `.spec.liveMigrationConfig.bandwidthPerMigration`: - -```json -{ - "spec": { - "liveMigrationConfig": { - "bandwidthPerMigration": "32Mi" - } - } -} -``` - -**3. Use dedicated migration network** (see Dedicated Migration Network section) - -### Issue 3: virt-handler Pods Not Ready - -**Symptom:** - -Using `pods_list` with `labelSelector: "kubevirt.io=virt-handler"`, some pods show `status.containerStatuses[0].ready == false`. - -**Causes:** -- Recent HyperConverged configuration change -- Network configuration error -- Node connectivity issue - -**Solutions:** - -**1. Wait for pod restart** (after config change): - -Pods restart automatically after HyperConverged update. Monitor using `pods_list` until all show ready status. - -**2. Check pod logs:** - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "name": "", - "namespace": "openshift-cnv", - "tail": 100 -} -``` - -**3. Verify node network configuration** (if using dedicated network): - -Use `resources_list` to check NodeNetworkConfigurationPolicy: - -**Parameters**: -```json -{ - "apiVersion": "nmstate.io/v1", - "kind": "NodeNetworkConfigurationPolicy" -} -``` - -And NetworkAttachmentDefinition: - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "openshift-cnv" -} -``` - -### Issue 4: Migration Rejected - Cluster Limit Reached - -**Symptom:** -``` -Migration rejected: cluster migration limit reached (5 concurrent) -``` - -**Solutions:** - -**1. Wait for ongoing migrations** to complete: - -Monitor using `resources_list`: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstanceMigration" -} -``` - -Filter for migrations where `.status.phase` is NOT "Succeeded" or "Failed". - -**2. Increase cluster limit** (if network allows): - -Use `resources_get` to fetch HyperConverged CR, modify `.spec.liveMigrationConfig.parallelMigrationsPerCluster`, then update with `resources_create_or_update`: - -```json -{ - "spec": { - "liveMigrationConfig": { - "parallelMigrationsPerCluster": 10 - } - } -} -``` - -**3. Migrate VMs sequentially** instead of batch operation - ---- - -## Related Documentation - -- [Performance Tuning Guide](./performance-tuning.md) - Advanced tuning for migration performance -- [Anti-Patterns](./anti-patterns.md) - Common mistakes to avoid -- [Production Considerations](./production-considerations.md) - Right-sizing, workload planning, HA strategies -- [Troubleshooting: Scheduling Errors](../../../docs/troubleshooting/scheduling-errors.md) - ErrorUnschedulable after cold migration - ---- - -**Last Updated**: 2026-02-24 -**OpenShift Virtualization Versions**: 4.17, 4.18, 4.19, 4.20 -**Status**: Production-ready guidance from official Red Hat sources diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/performance-tuning.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/performance-tuning.md deleted file mode 100644 index f1af24e9..00000000 --- a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/performance-tuning.md +++ /dev/null @@ -1,719 +0,0 @@ -# Performance Tuning for VM Rebalancing - -**Purpose**: Advanced performance tuning parameters, optimization strategies, and monitoring guidance for VM live migration and rebalancing operations. - -**When to consult this document**: When migrations are slow, when planning large-scale rebalancing, or when optimizing cluster performance for frequent migrations. - ---- - -## Official Sources - -This document is compiled from official Red Hat documentation: - -- [Live Migrating VMs with OpenShift Virtualization](https://developers.redhat.com/articles/2025/07/14/live-migrating-vms-openshift-virtualization) - Red Hat Developer (2025-07-14) -- [Best Practices for Virtual Machine Deployments on OpenShift Virtualization](https://learn.microsoft.com/en-us/azure/openshift/best-practices-openshift-virtualization) - Microsoft Azure Red Hat OpenShift (2026-02-16) -- [Announcing Right-Sizing for OpenShift Virtualization](https://developers.redhat.com/articles/2025/04/28/announcing-right-sizing-openshift-virtualization) - Red Hat Developer (2025-04-28) -- [Best Practices to Deploy VMs in Red Hat OpenShift Virtualization](https://docs.netapp.com/us-en/netapp-solutions-virtualization/openshift/os-osv-bpg.html) - NetApp Solutions - ---- - -## Right-Sizing Virtual Machines - -### Why Right-Sizing Matters for Rebalancing - -Properly sized VMs: -- Migrate faster (smaller memory footprint) -- Reduce network bandwidth requirements -- Improve cluster resource utilization -- Enable more efficient load balancing -- Prevent resource contention during migrations - -### Right-Sizing Methodology - -**Step 1: Define Health Metrics** - -Target healthy resource utilization ranges: - -| Resource | Target Range | Warning Threshold | Critical Threshold | -|----------|--------------|-------------------|-------------------| -| CPU Utilization | 60-70% average | >80% | >90% | -| Memory Pressure | <80% | >85% | >95% | -| Disk I/O Latency | <10ms | >50ms | >100ms | -| Network Throughput | <70% capacity | >80% | >90% | - -**Step 2: Monitor VM Resource Usage** - -**Using MCP Tools:** - -**MCP Tool**: `pods_top` (from openshift-virtualization) - -**Parameters**: -```json -{ - "all_namespaces": true, - "label_selector": "kubevirt.io=virt-launcher" -} -``` - -This returns CPU and memory usage for all VM launcher pods. Filter by specific namespace or VM name as needed. - -**For detailed metrics:** - -**MCP Tool**: `nodes_stats_summary` (from openshift-virtualization) - -**Parameters**: -```json -{ - "name": "" -} -``` - -Review `.pods[].containers[]` metrics for specific VM resource consumption including: -- `cpu.usageNanoCores` - Current CPU usage -- `memory.workingSetBytes` - Active memory usage -- `rootfs.usedBytes` - Disk usage - -**Step 3: Analyze Historical Data** - -Collect metrics over time (minimum 7 days for meaningful patterns): - -- Peak usage periods -- Resource saturation events -- Correlation between workload and resource consumption -- Trending (growing vs stable resource needs) - -**Step 4: Adjust VM Specifications** - -Based on observed metrics, resize VMs using `resources_get` and `resources_create_or_update`: - -**Example: Resize VM Memory** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Modify `.spec.template.spec.domain.resources.requests.memory` based on usage analysis, then update with `resources_create_or_update`. - -**Avoid Overprovisioning:** -- Don't rely on on-premises sizing references -- Benchmark your actual workloads -- Consider OpenShift Virtualization architectural overhead (see Architectural Overhead section) - ---- - -## Architectural Overhead - -### OpenShift Virtualization Performance Characteristics - -Running VMs in OpenShift Virtualization introduces architectural overhead compared to bare metal or native pods: - -**Observed Performance (Azure Red Hat OpenShift with Standard_D96ds_v5 nodes, OpenShift 4.20, Virtualization 4.20):** - -| Workload Type | VM Performance | Pod Performance | Overhead | -|---------------|----------------|-----------------|----------| -| **Compute** (events/sec) | 525,022 | 546,997 | ~4% slower | -| **Compute** (latency ms) | 0.70 | 0.65 | ~8% higher latency | -| **Storage** (1 thread TPM) | 4,332 | 6,303 | ~31% slower | -| **Storage** (32 threads TPM) | 64,294 | 103,359 | ~38% slower | -| **Network** (64B, 1 thread Gbps) | 0.4 | 0.9 | ~56% slower | -| **Network** (1024B, 8 threads Gbps) | 24.7 | 28.9 | ~15% slower | - -**Key Takeaways:** -- Compute overhead is minimal (~4-8%) -- Storage and network have higher overhead (15-56% depending on workload) -- Multi-threaded workloads show better relative performance - -**Implications for Rebalancing:** -- VMs require more time to migrate than equivalent containerized workloads -- Plan capacity with overhead in mind (don't fill nodes to 100%) -- Network-intensive VMs benefit most from dedicated migration networks -- Consider workload characteristics when planning concurrent migrations - ---- - -## Tuned Configuration for High-Performance VMs - -### SAP HANA Tuning Example - -For database and high-performance workloads, apply tuned profiles to guest OS: - -**Tuned Profile (RHEL Guest):** - -```ini -[main] -summary=Optimize for SAP HANA and high-performance VMs - -[cpu] -force_latency=cstate.id:3|70 -governor=performance -energy_perf_bias=performance -min_perf_pct=100 - -[vm] -transparent_hugepages=never - -[sysctl] -# Semaphore limits -kernel.sem = 32000 1024000000 500 32000 - -# Disable NUMA balancing for predictable performance -kernel.numa_balancing = 0 - -# Scheduler tuning for low latency -kernel.sched_min_granularity_ns = 3000000 -kernel.sched_wakeup_granularity_ns = 4000000 - -# Memory management -vm.dirty_ratio = 40 -vm.dirty_background_ratio = 10 -vm.swappiness = 10 -``` - -**When to Apply:** -- Database VMs (PostgreSQL, MySQL, Oracle, SAP HANA) -- Real-time analytics workloads -- Low-latency trading platforms -- High-performance computing (HPC) VMs - -**Impact on Migration:** -- Reduces dirty page rate (faster convergence) -- More predictable migration times -- Better performance during and after migration - ---- - -## CPU and Memory Overcommit - -### Understanding Overcommit Ratios - -OpenShift Virtualization allows overcommit of CPU and memory resources, enabling higher VM density per node. - -**Default Overcommit Ratios:** -- CPU: No overcommit (1:1 mapping) -- Memory: No overcommit (1:1 mapping) - -**Recommended Production Limits (Red Hat Guidance):** -- **CPU Overcommit**: Maximum 1.8x physical cores -- **Memory Overcommit**: Maximum 0.9x physical memory - -**Consequences of Exceeding Limits:** -- **CPU**: Throttling causes slowness across all workloads on affected node -- **Memory**: OOM (Out of Memory) kills, VM crashes, data loss - -### Configuring Overcommit - -**Update HyperConverged CR using MCP Tools:** - -**Step 1: Get HyperConverged resource** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "hco.kubevirt.io/v1beta1", - "kind": "HyperConverged", - "namespace": "openshift-cnv", - "name": "kubevirt-hyperconverged" -} -``` - -**Step 2: Modify overcommit configuration** - -Add to `.spec.resourceRequirements`: - -```yaml -spec: - resourceRequirements: - vmiCPUAllocationRatio: 1.5 # Allow 1.5x CPU overcommit - vmiMemoryOvercommitPercent: 20 # Allow 20% memory overcommit -``` - -**Step 3: Update using `resources_create_or_update`** - -**Best Practices:** -- Use conservative overcommit for production (1.2x CPU max, 10% memory max) -- Use higher overcommit for dev/test (1.8x CPU, 20% memory acceptable) -- Monitor node resource usage closely after enabling overcommit -- Adjust based on actual VM behavior patterns - -**Impact on Rebalancing:** -- Higher overcommit = more VMs per node = longer migration times -- Rebalancing may be needed more frequently with overcommit -- Target node capacity calculations must account for overcommit ratios - ---- - -## Network Performance Tuning - -### MTU Configuration - -**Why MTU Matters:** -- Default MTU (1500 bytes) causes fragmentation for large data transfers -- Jumbo frames (MTU 9000) significantly improve network efficiency -- Critical for large VM migrations (>100GB memory) - -**Set MTU in NetworkAttachmentDefinition:** - -When creating dedicated migration network, include MTU setting: - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters** (excerpt): -```json -{ - "resource": "apiVersion: k8s.cni.cncf.io/v1\nkind: NetworkAttachmentDefinition\nmetadata:\n name: migration-network\n namespace: openshift-cnv\nspec:\n config: '{\n \"cniVersion\": \"0.3.1\",\n \"name\": \"migration-bridge\",\n \"type\": \"macvlan\",\n \"master\": \"eth1\",\n \"mode\": \"bridge\",\n \"mtu\": 9000,\n \"ipam\": {...}\n }'" -} -``` - -**Validate MTU:** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "openshift-cnv", - "name": "migration-network" -} -``` - -Check `.spec.config` for `"mtu": 9000`. - -### NAPI and Multiqueue Tuning - -For network-intensive workloads, enable multiqueue virtio-net: - -**VM Configuration:** -```yaml -spec: - template: - spec: - domain: - devices: - interfaces: - - name: default - model: virtio - masquerade: {} - ports: - - port: 80 - networkInterfaceMultiqueue: true # Enable multiqueue -``` - -**Benefits:** -- Parallelizes network processing across multiple vCPUs -- Improves throughput for high-bandwidth workloads -- Reduces latency for network-intensive applications - -**When to Use:** -- VMs with >4 vCPUs -- High network throughput requirements (>10Gbps) -- Web servers, load balancers, network appliances - ---- - -## Storage Performance Optimization - -### Storage Class Selection - -Different storage backends have different performance characteristics: - -| Storage Backend | IOPS | Latency | Throughput | Best For | -|-----------------|------|---------|------------|----------| -| OpenShift Data Foundation (ODF) | High | Low (<5ms) | Very High | General purpose, production | -| Azure NetApp Files (ANF) Premium | Very High | Very Low (<1ms) | Very High | Database, high-performance | -| NFS-CSI (SSD-backed) | Medium | Medium (5-10ms) | High | Dev/test, general use | -| AWS EBS gp3 | Medium | Medium (10-20ms) | Medium | Cost-effective, RWO only | - -**Check Storage Class Performance:** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass", - "name": "" -} -``` - -Review `.parameters` for performance tier, provisioning type, and backend configuration. - -### Storage Limits (NetApp ONTAP) - -When using NetApp storage backends, configure limits to prevent resource exhaustion: - -**SVM Volume Limits:** - -Set maximum volumes per SVM to prevent Trident from consuming all storage capacity. - -**Storage Quotas:** - -Implement storage limits on SVMs to enforce resource boundaries. - -**Trident Backend Parameters:** - -Configure in Trident backend definition: -- `limitVolumeSize`: Maximum individual volume size (e.g., "100Gi") -- `limitVolumePoolSize`: Maximum FlexVol size for economy drivers (e.g., "500Gi") - -**Impact on Rebalancing:** -- Storage limits prevent VMs from growing unbounded -- Predictable storage capacity aids in target node selection -- Quota enforcement ensures fair resource distribution - ---- - -## Migration Bandwidth Management - -### Bandwidth Per Migration - -**Purpose**: Limit bandwidth consumption per migration to prevent network saturation. - -**Default**: Unlimited (no bandwidth limit) - -**When to Set:** -- Shared application network (no dedicated migration network) -- Multiple concurrent migrations planned -- Network capacity constraints - -**Configuration Using MCP Tools:** - -**Step 1: Get HyperConverged resource** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "hco.kubevirt.io/v1beta1", - "kind": "HyperConverged", - "namespace": "openshift-cnv", - "name": "kubevirt-hyperconverged" -} -``` - -**Step 2: Set bandwidth limit** - -Modify `.spec.liveMigrationConfig.bandwidthPerMigration`: - -```yaml -spec: - liveMigrationConfig: - bandwidthPerMigration: 64Mi # 64 MiB/s per migration -``` - -Common values: -- `32Mi` - Conservative (256 Mbps) -- `64Mi` - Default (512 Mbps) -- `128Mi` - High bandwidth (1 Gbps) -- Omit field for unlimited - -**Step 3: Update using `resources_create_or_update`** - -**Monitoring Bandwidth Usage:** - -**MCP Tool**: `nodes_stats_summary` (from openshift-virtualization) - -**Parameters**: -```json -{ - "name": "" -} -``` - -Review `.network.interfaces[].rxBytes` and `.network.interfaces[].txBytes` for current throughput. - -**Tuning Guidance:** -- Start conservative (32-64Mi) and increase if migrations are slow -- Monitor network utilization during migrations -- Unlimited bandwidth is acceptable with dedicated migration network - ---- - -## Concurrency Limits Tuning - -### Parallel Migrations Per Cluster - -**Default**: 5 concurrent migrations cluster-wide - -**When to Increase:** -- Dedicated migration network with high bandwidth (100Gbps) -- Routine maintenance windows requiring many migrations -- Cluster has >20 nodes - -**When to Decrease:** -- Network saturation observed -- Migration failures due to timeouts -- Shared application network - -**Configuration:** - -Modify `.spec.liveMigrationConfig.parallelMigrationsPerCluster` in HyperConverged CR: - -```yaml -spec: - liveMigrationConfig: - parallelMigrationsPerCluster: 10 # Increase from default 5 -``` - -**Conservative**: 3-5 migrations -**Moderate**: 5-10 migrations -**Aggressive**: 10-20 migrations (requires dedicated network) - -### Parallel Outbound Migrations Per Node - -**Default**: 2 concurrent outbound migrations per source node - -**Recommendation**: Keep at 2 to prevent single-node overload. - -**Why 2 is Optimal:** -- Prevents source node CPU/memory saturation -- Limits network bandwidth consumption per node -- Avoids cascading performance degradation -- Tested and validated by Red Hat - -**Only increase to 3-4 if:** -- Node has very high CPU/memory headroom (>50% free) -- Dedicated high-bandwidth migration network -- Extensive testing validates stability - ---- - -## Timeout Configuration - -### Completion Timeout Per GiB - -**Default**: 800 seconds per GiB of VM memory - -**Calculation**: For a 16GB VM, timeout = 16 * 800 = 12,800 seconds (~3.5 hours) - -**When to Increase:** -- High dirty page rate workloads (databases, caching systems) -- VMs with >100GB memory -- Network bandwidth constraints - -**When to Decrease:** -- Fast dedicated migration network (100Gbps) -- Low dirty page rate (mostly idle VMs) -- Want faster failure detection - -**Configuration:** - -Modify `.spec.liveMigrationConfig.completionTimeoutPerGiB` in HyperConverged CR: - -```yaml -spec: - liveMigrationConfig: - completionTimeoutPerGiB: 1200 # Increase for large/busy VMs -``` - -**Tuning by Workload Type:** - -| Workload Type | Recommended Timeout | Rationale | -|---------------|---------------------|-----------| -| Database (write-heavy) | 1200-1600s | High dirty page rate | -| Web server (mostly read) | 600-800s | Low dirty page rate | -| Caching (Redis/Memcached) | 1600-2000s | Very high dirty page rate | -| General purpose | 800s (default) | Balanced | - -### Progress Timeout - -**Default**: 150 seconds without progress before cancellation - -**Purpose**: Detects stuck migrations and fails fast rather than hanging indefinitely. - -**When to Increase:** -- Very large VMs (>500GB memory) -- Slow networks (<1Gbps) -- Initial memory copy takes >2 minutes - -**When to Decrease:** -- Want faster failure detection -- Prefer to retry quickly rather than wait - -**Configuration:** - -Modify `.spec.liveMigrationConfig.progressTimeout` in HyperConverged CR: - -```yaml -spec: - liveMigrationConfig: - progressTimeout: 300 # 5 minutes without progress -``` - -**Recommended Values:** -- Small VMs (<50GB): 150s (default) -- Medium VMs (50-200GB): 200-300s -- Large VMs (>200GB): 300-600s - ---- - -## Monitoring and Observability - -### Key Metrics to Monitor - -**During Rebalancing Operations:** - -1. **Migration Progress** - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstanceMigration" -} -``` - -Monitor `.status.phase` for each migration (Pending → Scheduling → PreparingTarget → Running → Succeeded). - -2. **Node Resource Usage** - -**MCP Tool**: `nodes_top` (from openshift-virtualization) - -**Parameters**: -```json -{ - "name": "" -} -``` - -Track CPU and memory utilization before, during, and after migrations. - -3. **Network Throughput** - -**MCP Tool**: `nodes_stats_summary` (from openshift-virtualization) - -**Parameters**: -```json -{ - "name": "" -} -``` - -Review `.network.interfaces[]` metrics for bandwidth usage. - -4. **VM Health** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.status.conditions[]` for VM health status. - -### Performance Benchmarking - -**Before Rebalancing:** -- Establish baseline performance metrics -- Document current resource utilization -- Identify performance-sensitive VMs - -**During Rebalancing:** -- Monitor migration duration -- Track network bandwidth consumption -- Watch for resource contention - -**After Rebalancing:** -- Validate improved load distribution -- Confirm no performance degradation -- Document improvements achieved - -**Tools for Benchmarking:** -- Apache JMeter (web application load testing) -- stress-ng (CPU/memory stress testing) -- fio (storage I/O benchmarking) -- iperf3 (network throughput testing) - ---- - -## Scaling Strategies - -### Scale Out vs Scale Up - -**Scale Out** (add more nodes): -- **Pros**: Better fault tolerance, more migration targets, horizontal capacity growth -- **Cons**: Higher complexity, more licensing costs, requires cluster expansion - -**Scale Up** (larger node sizes): -- **Pros**: Simpler management, fewer migration hops, better resource consolidation -- **Cons**: Larger blast radius, limited by maximum instance size, single point of failure risk - -**For Demanding Workloads:** - -From Microsoft Azure Red Hat OpenShift guidance: -> "Scale out or up for demanding workloads: Add more nodes or upsize the nodes in your Azure Red Hat OpenShift cluster for high concurrency or resource-intensive applications." - -**Recommendation:** -- Start with scale-up to minimum 8-core Azure VMs (per OpenShift Virtualization requirements) -- Scale-out when individual nodes exceed 70-80% sustained utilization -- Balance between node size and cluster size for optimal resilience - -### Node Pool Strategy - -**Workload-Specific Node Pools:** - -Create dedicated node pools for different VM workload types using labels, taints, and tolerations: - -**Example Node Pool Configuration:** - -**Pool 1: General VMs** -- Node labels: `workload-type=general` -- Node taints: None -- VM tolerations: Not required - -**Pool 2: High-Performance VMs** -- Node labels: `workload-type=high-performance` -- Node taints: `performance=dedicated:NoSchedule` -- VM tolerations: Match taint - -**Pool 3: GPU Workloads** -- Node labels: `workload-type=gpu` -- Node taints: `nvidia.com/gpu=present:NoSchedule` -- VM tolerations: Match taint - -**Apply Labels to Nodes Using MCP Tools:** - -**MCP Tool**: `resources_get` then `resources_create_or_update` (from openshift-virtualization) - -Get node, modify `.metadata.labels`, then update. - -**Benefits:** -- Simplifies maintenance (drain entire pool) -- Limits blast radius (failures contained to pool) -- Improves resource efficiency (right-sized pools) -- Enables topology spread rules (VMs across zones/pools) - ---- - -## Related Documentation - -- [Live Migration Best Practices](./live-migration-best-practices.md) - Configuration parameters and requirements -- [Anti-Patterns](./anti-patterns.md) - Common mistakes to avoid -- [Production Considerations](./production-considerations.md) - Right-sizing, workload planning, HA strategies - ---- - -**Last Updated**: 2026-02-24 -**OpenShift Virtualization Versions**: 4.17, 4.18, 4.19, 4.20 -**Status**: Production-ready guidance from official Red Hat sources diff --git a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/production-considerations.md b/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/production-considerations.md deleted file mode 100644 index 919280a8..00000000 --- a/evaluation/with_skills/rh-virt__vm-rebalance/environment/skills/vm-rebalance/references/production-considerations.md +++ /dev/null @@ -1,868 +0,0 @@ -# Production Considerations for VM Rebalancing - -**Purpose**: Production deployment guidance, workload planning, high availability strategies, and operational best practices for VM rebalancing in OpenShift Virtualization. - -**When to consult this document**: Before deploying to production, when planning capacity, or when designing HA/DR strategies. - ---- - -## Official Sources - -This document is compiled from official Red Hat documentation: - -- [Best Practices for Virtual Machine Deployments on OpenShift Virtualization](https://learn.microsoft.com/en-us/azure/openshift/best-practices-openshift-virtualization) - Microsoft Azure Red Hat OpenShift (2026-02-16) -- [Announcing Right-Sizing for OpenShift Virtualization](https://developers.redhat.com/articles/2025/04/28/announcing-right-sizing-openshift-virtualization) - Red Hat Developer (2025-04-28) -- [Best Practices to Deploy VMs in Red Hat OpenShift Virtualization](https://docs.netapp.com/us-en/netapp-solutions-virtualization/openshift/os-osv-bpg.html) - NetApp Solutions -- [OpenShift Virtualization Best Practices](https://trilio.io/openshift-virtualization/) - Trilio - ---- - -## Workload Identification and Categorization - -### Common Workload Types - -Before provisioning VMs, categorize workloads to determine performance and resource requirements: - -| Workload Type | Characteristics | Resource Profile | Migration Considerations | -|---------------|----------------|------------------|--------------------------| -| **General Purpose** | Web servers, app servers, CMS | Moderate CPU/memory | Easy to migrate, low dirty page rate | -| **Database** | RDBMS, NoSQL | High CPU, memory, consistent IOPS | High dirty page rate; schedule migrations carefully | -| **Real-time Analytics** | Operational dashboards | Low latency, high throughput | Sensitive to migration pause; use dedicated network | -| **AI/ML** | Training, inference | Very high CPU/GPU, memory | Large memory footprint; long migration times | -| **Data Streaming** | Event-driven architectures | High throughput, low latency | Network-intensive; avoid concurrent migrations | -| **Batch Processing** | Periodic jobs | Variable resources | Migrate during job idle periods | -| **HPC** | Scientific simulations | Very high CPU, memory | Extremely long migrations; consider cold migration | -| **Edge/IoT** | Sensor aggregation | Low resources | Easy to migrate, scale horizontally | -| **Media Processing** | Encoding, streaming | High CPU, network | High dirty page rate during processing | -| **Dev/Test** | Development environments | Variable | Higher overcommit acceptable | - -### Workload Assessment Using MCP Tools - -**Step 1: Inventory Current VMs** - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine" -} -``` - -**Step 2: Analyze Resource Usage** - -**MCP Tool**: `pods_top` (from openshift-virtualization) - -**Parameters**: -```json -{ - "all_namespaces": true, - "label_selector": "kubevirt.io=virt-launcher" -} -``` - -**Step 3: Categorize by Usage Pattern** - -Group VMs by observed characteristics: -- CPU-intensive: >70% CPU utilization -- Memory-intensive: >80% memory utilization -- I/O-intensive: High storage throughput -- Network-intensive: High network bandwidth - -**Step 4: Plan Rebalancing Strategy** - -Based on workload type: -- **CPU-intensive**: Balance CPU across nodes -- **Memory-intensive**: Balance memory across nodes -- **I/O-intensive**: Distribute across different storage backends -- **Network-intensive**: Stagger migrations to avoid saturation - ---- - -## Right-Sizing Virtual Machines for Production - -### Health Metrics Definition - -Establish target ranges for healthy resource utilization: - -**CPU Utilization:** -- **Target**: 60-70% average usage -- **Warning**: >80% sustained -- **Critical**: >90% sustained -- **Action**: Scale up VM or rebalance to less loaded node - -**Memory Pressure:** -- **Target**: 70-80% utilization -- **Warning**: >85% with swap activity -- **Critical**: >95% or OOM events -- **Action**: Increase VM memory or reduce workload - -**Disk I/O:** -- **Target**: <10ms latency, <70% queue depth -- **Warning**: >50ms latency -- **Critical**: >100ms latency or queue saturation -- **Action**: Move to faster storage tier or distribute workload - -**Network Throughput:** -- **Target**: <70% interface capacity -- **Warning**: >80% sustained -- **Critical**: >90% or packet loss -- **Action**: Enable multiqueue, use faster NICs, rebalance - -### Monitoring Setup Using MCP Tools - -**VM-Level Metrics:** - -**MCP Tool**: `pods_top` (from openshift-virtualization) - -Provides current CPU and memory usage for VM launcher pods. - -**Node-Level Metrics:** - -**MCP Tool**: `nodes_top` (from openshift-virtualization) - -Shows aggregate node resource consumption. - -**Detailed Statistics:** - -**MCP Tool**: `nodes_stats_summary` (from openshift-virtualization) - -**Parameters**: -```json -{ - "name": "" -} -``` - -Provides comprehensive metrics including: -- Per-pod resource usage -- Container-level metrics -- Filesystem usage -- Network interface statistics -- PSI (Pressure Stall Information) metrics on cgroup v2 systems - -### Sizing Recommendations by Workload - -**Database Workloads:** -- Start with: 4-8 vCPU, 16-32Gi memory -- Storage: Premium SSD or NVMe with RWX support -- Network: Enable multiqueue virtio-net -- Special: Consider dedicated CPU placement (`dedicatedCpuPlacement: true`) - -**Web Servers:** -- Start with: 2-4 vCPU, 4-8Gi memory -- Storage: Standard SSD acceptable -- Network: Standard configuration sufficient -- Special: Scale horizontally rather than vertically - -**AI/ML Workloads:** -- Start with: 8-16 vCPU, 32-64Gi memory -- Storage: High-performance SSD -- Network: High bandwidth (consider dedicated migration network) -- Special: GPU support currently not available (plan accordingly) - -**Dev/Test Environments:** -- Start with: 1-2 vCPU, 2-4Gi memory -- Storage: Standard tier acceptable -- Network: Standard configuration -- Special: Higher overcommit ratios acceptable (1.5-1.8x CPU) - -### Minimum Requirements - -From Microsoft Azure Red Hat OpenShift documentation: -> "Minimum core requirement: OpenShift Virtualization requires a minimum of eight (8) core Azure VMs for OpenShift worker nodes." - -**Implications for Rebalancing:** -- Worker nodes must have ≥8 cores -- Plan VM placement considering this minimum -- Avoid creating nodes smaller than this threshold - ---- - -## High Availability Strategies - -### VM-Level High Availability - -**RunStrategy Configuration:** - -```yaml -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -metadata: - name: critical-app -spec: - runStrategy: Always # Ensures VM restarts after failures -``` - -**RunStrategy Options:** - -| Strategy | Behavior | Use Case | -|----------|----------|----------| -| `Always` | VM runs continuously; restarts on failure | Production VMs requiring HA | -| `RerunOnFailure` | Restarts only if VM crashes | Batch workloads | -| `Manual` | User controls start/stop | Dev/test VMs | -| `Halted` | VM stays stopped | Maintenance, cold storage | - -**Eviction Strategy:** - -OpenShift Virtualization automatically sets `evictionStrategy` to `LiveMigrate` for VMs with RWX storage: - -```yaml -spec: - template: - spec: - evictionStrategy: LiveMigrate # Automatically set for RWX VMs -``` - -**Note for Single Node OpenShift (SNO):** - -From known issues: -> "In a Single Node OpenShift (SNO) cluster, a VMCannotBeEvicted alert occurs on virtual machines created from common templates that have the eviction strategy set to LiveMigrate." - -**Workaround**: Use `evictionStrategy: None` for SNO clusters. - -### Pod Anti-Affinity for VM Replicas - -For critical applications, deploy multiple VM replicas with anti-affinity: - -```yaml -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -metadata: - name: web-server-replica-1 - labels: - app: web-server -spec: - template: - metadata: - labels: - app: web-server - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - web-server - topologyKey: kubernetes.io/hostname # Different nodes -``` - -**Benefits:** -- VMs distributed across different failure domains -- Node failure affects only one replica -- Improves overall availability - -**Verify Distribution Using MCP Tools:** - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "labelSelector": "app=web-server" -} -``` - -Check `.status.nodeName` for each instance to confirm distribution. - -### Machine Health Checks - -**Critical for Automatic Failover:** - -From Red Hat documentation: -> "If a node fails and machine health checks are not deployed on your cluster, virtual machines (VMs) with RunStrategy: Always configured are not automatically relocated to healthy nodes." - -**Deploy Machine Health Checks:** - -Configure at cluster level to detect and remediate node failures. This enables automatic VM recovery without manual intervention. - -**Monitor Node Health:** - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -Filter for nodes where `.status.conditions[]` shows unhealthy states (`Ready=False`, `DiskPressure=True`, `MemoryPressure=True`). - -### Topology Spread for Zone Resilience - -For multi-zone clusters, use topology spread rules: - -```yaml -spec: - template: - spec: - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: topology.kubernetes.io/zone - whenUnsatisfiable: DoNotSchedule - labelSelector: - matchLabels: - app: critical-app -``` - -**Benefits:** -- VMs spread across availability zones -- Zone failure doesn't affect all replicas -- Improved disaster recovery - ---- - -## Capacity Planning - -### Cluster Sizing Approach - -**Scale Out vs Scale Up:** - -From Microsoft Azure Red Hat OpenShift guidance: -> "Scale out or up for demanding workloads: Add more nodes or upsize the nodes in your Azure Red Hat OpenShift cluster for high concurrency or resource-intensive applications." - -**Scale Out (add more nodes):** -- **Pros**: Better fault tolerance, horizontal growth, more migration targets -- **Cons**: Higher complexity, more license costs, requires cluster expansion - -**When to Scale Out:** -- Current nodes consistently >70-80% utilized -- Need more fault isolation -- Planning for growth -- HA requirements mandate distribution - -**Scale Up (larger node sizes):** -- **Pros**: Simpler management, better resource consolidation, fewer migration hops -- **Cons**: Larger blast radius, limited by max instance size, single point of failure risk - -**When to Scale Up:** -- VMs don't fit on existing nodes -- Few large VMs rather than many small VMs -- Simplicity valued over distribution - -**Recommended Approach:** -1. Start with moderate node sizes (8-16 cores) -2. Scale out to 3-5 nodes minimum for HA -3. Scale up only when specific VMs require larger nodes -4. Maintain headroom (30-40% free capacity) for migrations and failures - -### Node Pool Strategy - -**Create workload-specific pools using labels and taints:** - -**Pool Configuration Example:** - -**General VM Pool:** -- Node size: 8-16 cores, 32-64GB RAM -- Labels: `workload-type=general` -- No taints (default scheduling) - -**High-Performance Pool:** -- Node size: 16-32 cores, 64-128GB RAM -- Labels: `workload-type=high-performance`, `cpumanager=true` -- Taints: `performance=dedicated:NoSchedule` - -**GPU Pool (future):** -- Node size: GPU-enabled instances -- Labels: `workload-type=gpu` -- Taints: `nvidia.com/gpu=present:NoSchedule` - -**Configure Labels Using MCP Tools:** - -**MCP Tool**: `resources_get` then `resources_create_or_update` (from openshift-virtualization) - -**Step 1: Get Node** - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node", - "name": "" -} -``` - -**Step 2: Add Labels** - -Modify `.metadata.labels`: -```json -{ - "workload-type": "high-performance", - "cpumanager": "true" -} -``` - -**Step 3: Update Node** - -Use `resources_create_or_update` with modified resource. - -**Benefits of Node Pools:** -- Simplifies maintenance (drain entire pool) -- Limits blast radius (failures contained) -- Improves efficiency (right-sized for workload) -- Enables topology spread rules - -### Capacity Headroom - -**Reserve capacity for:** -- Node failures (n-1 redundancy minimum) -- VM migrations (target nodes need free resources) -- Burst workloads (temporary spikes) -- New VM deployments - -**Recommended Headroom:** -- **Production**: 30-40% free capacity cluster-wide -- **Dev/Test**: 20-30% free capacity -- **Minimum**: 20% free capacity (below this, rebalancing becomes difficult) - -**Monitor Capacity Using MCP Tools:** - -**MCP Tool**: `nodes_top` (from openshift-virtualization) - -Calculate cluster-wide utilization: -``` -Total CPU Used / Total CPU Capacity = Cluster CPU Utilization -Total Memory Used / Total Memory Capacity = Cluster Memory Utilization -``` - -**Action Thresholds:** -- <70%: Healthy headroom -- 70-80%: Plan for expansion -- >80%: Add nodes urgently -- >90%: Emergency capacity issue - ---- - -## Storage Planning for Production - -### Storage Backend Selection - -**OpenShift Data Foundation (ODF):** -- **Best for**: General purpose, production workloads -- **Performance**: High IOPS, low latency (<5ms) -- **RWX Support**: Yes -- **Considerations**: Requires dedicated storage nodes; use taints/tolerations to isolate ODF workload - -**Azure NetApp Files (ANF):** -- **Best for**: High-performance databases, latency-sensitive apps -- **Performance**: Very high IOPS, very low latency (<1ms) -- **RWX Support**: Yes -- **Considerations**: Choose performance tier based on workload requirements - -**NFS-CSI (SSD-backed):** -- **Best for**: Dev/test, general use -- **Performance**: Medium IOPS, medium latency (5-10ms) -- **RWX Support**: Yes -- **Considerations**: Cost-effective, sufficient for non-critical workloads - -**AWS EBS gp3:** -- **Best for**: Cost-effective storage -- **Performance**: Medium IOPS, medium latency (10-20ms) -- **RWX Support**: No (RWO only) -- **Considerations**: Cannot use live migration; cold migration only - -### Storage QoS and Limits - -**NetApp ONTAP QoS:** - -From NetApp documentation: -> "Apply QoS policies to SVMs to limit the number of IOPS consumable by the Trident provisioned volumes." - -**Why QoS Matters:** -- Prevents one VM from starving others -- Protects non-Trident workloads from VM I/O impact -- Enforces fair resource distribution -- Predictable performance for all VMs - -**SVM Isolation:** - -From NetApp documentation: -> "Establish dedicated Storage Virtual Machines (SVMs) to provide isolation and administrative separation between tenants." - -**Benefits:** -- Tenant isolation -- Privilege delegation -- Resource quota enforcement -- Security boundary - -### Storage Validation Before Rebalancing - -**For Live Migration, verify RWX storage:** - -For each VM: - -**Step 1: Get VM** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Step 2: Extract PVC Names** - -From `.spec.template.spec.volumes[].persistentVolumeClaim.claimName`. - -**Step 3: Check PVC Access Mode** - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -**Required**: `.spec.accessModes` must include `"ReadWriteMany"`. - ---- - -## Network Planning - -### Dedicated Migration Network - -**Production Requirement:** - -For production clusters with large VMs or frequent migrations, a dedicated migration network is **highly recommended**. - -**Benefits:** -- Isolates migration traffic from applications -- Enables 100Gbps bandwidth for large VM migrations -- Prevents network contention -- Improves security (separate VLAN) - -**Implementation:** - -See [live-migration-best-practices.md](./live-migration-best-practices.md) for complete configuration steps. - -**Key Components:** -1. Secondary physical NIC or VLAN -2. NodeNetworkConfigurationPolicy (NNCP) -3. NetworkAttachmentDefinition (NAD) -4. HyperConverged CR configuration - -### Network Performance Tuning - -**MTU Configuration:** - -Set to 9000 (jumbo frames) for migration networks: - -```json -{ - "cniVersion": "0.3.1", - "name": "migration-bridge", - "type": "macvlan", - "mtu": 9000, - "ipam": {...} -} -``` - -**Multiqueue virtio-net:** - -Enable for VMs with >4 vCPUs and high network throughput: - -```yaml -spec: - template: - spec: - domain: - devices: - interfaces: - - name: default - model: virtio - networkInterfaceMultiqueue: true -``` - -### Network Isolation - -**Namespace Separation:** - -From best practices: -> "Use Namespaces to provide logical boundary for resources." - -**Pod Security Policies:** - -Disable privileged container capabilities for VM launcher pods to enhance security. - -**Separate Export Policies:** - -For NFS storage, implement separate export policies for infrastructure nodes vs application nodes. - ---- - -## Operational Best Practices - -### Phased Production Rollout - -From best practices: -> "Begin with non-critical or dev/test workloads before moving production systems - this phased approach allows teams to gain hands-on experience while minimizing risk." - -**Recommended Phases:** - -**Phase 1: Development/Test (2-4 weeks)** -- Deploy 5-10 test VMs -- Validate performance vs expectations -- Test live and cold migration workflows -- Benchmark resource overhead -- Train operations team - -**Phase 2: Non-Critical Production (4-8 weeks)** -- Migrate non-critical workloads (internal tools, QA environments) -- Monitor stability and performance -- Refine sizing and configurations -- Build runbooks and procedures -- Establish monitoring and alerting - -**Phase 3: Critical Production (Ongoing)** -- Migrate critical workloads in prioritized order -- Ensure HA and DR fully configured -- 24/7 monitoring and on-call support -- Document rollback procedures -- Conduct regular DR tests - -### Start Small, Scale Gradually - -From best practices: -> "When starting with OpenShift Virtualization, it's essential to start small and scale up as needed to avoid over-provisioning and wasting resources." - -**Growth Strategy:** - -**Month 1-2: Pilot** -- 5-10 VMs -- Single workload type -- Limited users -- Focus on learning - -**Month 3-6: Expansion** -- 20-50 VMs -- Multiple workload types -- Broader user base -- Refine processes - -**Month 6-12: Production Scale** -- 50-200+ VMs -- All workload types -- Organization-wide -- Mature operations - -**Benefits:** -- Avoids over-provisioning -- Iterative learning -- Cost-effective growth -- Risk mitigation - -### Monitoring and Alerting - -**Key Metrics to Monitor:** - -**VM Health:** -- Status (Running, Stopped, Error) -- Resource utilization (CPU, memory, disk, network) -- Guest agent connectivity -- Migration status - -**Node Health:** -- Resource utilization -- virt-handler pod status -- Network connectivity -- Storage backend health - -**Cluster Health:** -- Current migration count vs limits -- HyperConverged CR status -- Storage capacity and performance -- Network saturation - -**Migration Operations:** -- Success/failure rate -- Average migration duration -- Timeout occurrences -- Concurrent migration count - -**Alert Thresholds:** - -| Metric | Warning | Critical | -|--------|---------|----------| -| Node CPU | >80% | >90% | -| Node Memory | >85% | >95% | -| Migration Failures | >10% | >25% | -| virt-handler Pods Not Ready | Any | >1 | -| Cluster Migration Limit | >80% (4/5) | At limit (5/5) | - -### Backup and Disaster Recovery - -**VM Snapshots:** - -Use vm-snapshot skills for point-in-time backups before risky operations: -- Before major migrations -- Before configuration changes -- Before OS upgrades in guest -- Regular backup schedule (daily/weekly) - -**Disaster Recovery Planning:** - -**Multi-Zone Deployment:** -- Distribute VMs across availability zones -- Use topology spread constraints -- Configure zone-resilient storage - -**Backup Strategy:** -- Regular VM snapshots -- Export critical VM definitions -- Document restore procedures -- Test DR scenarios quarterly - -**RTO/RPO Targets:** - -Define recovery objectives: -- **RTO** (Recovery Time Objective): How quickly must VMs be recovered? -- **RPO** (Recovery Point Objective): How much data loss is acceptable? - -**Example Targets:** - -| Workload Tier | RTO | RPO | Strategy | -|---------------|-----|-----|----------| -| Critical | <15 min | <5 min | Multi-zone HA, frequent snapshots | -| Important | <1 hour | <1 hour | Daily snapshots, documented restore | -| Standard | <4 hours | <24 hours | Weekly snapshots, manual restore | - ---- - -## Cost Optimization - -### Resource Efficiency - -**Avoid Overprovisioning:** - -From Microsoft Azure Red Hat OpenShift guidance: -> "Avoid overprovisioning by aligning resources with actual usage patterns." - -**Cost Factors:** -- Azure compute costs (worker node instances) -- OpenShift licensing -- VM operating system licensing -- Storage costs (capacity and performance tier) -- Network egress charges - -**Optimization Strategies:** - -**1. Right-size VMs based on actual usage** - -Monitor with `pods_top` and resize VMs that are consistently under-utilized. - -**2. Use appropriate storage tiers** - -Don't use Premium storage for dev/test VMs; match tier to workload requirements. - -**3. Implement auto-scaling** - -For workloads with variable demand, use horizontal scaling rather than over-provisioning. - -**4. Consolidate with overcommit** - -In dev/test environments, use higher overcommit ratios (1.5-1.8x CPU) to maximize density. - -**5. Schedule non-critical VMs** - -Stop dev/test VMs during off-hours to reduce costs. - -### Load Balancing for Efficiency - -**Rebalancing Improves Efficiency:** -- Prevents hotspots (overloaded nodes) -- Enables better resource utilization -- Reduces need for emergency node additions -- Extends hardware lifespan (even wear) - -**Regular Rebalancing Schedule:** -- **Weekly**: Review node utilization, plan migrations if imbalance detected -- **Monthly**: Comprehensive rebalancing to optimize distribution -- **Quarterly**: Capacity planning and infrastructure rightsizing - ---- - -## Security Considerations - -### Tenant Isolation - -**Namespace Separation:** - -Deploy VMs for different tenants/teams in separate namespaces. - -**Network Policies:** - -Implement NetworkPolicies to restrict inter-VM communication: - -```yaml -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: tenant-isolation - namespace: tenant-a -spec: - podSelector: {} - policyTypes: - - Ingress - - Egress - ingress: - - from: - - namespaceSelector: - matchLabels: - tenant: tenant-a -``` - -**RBAC:** - -Grant users permissions only for their namespace's VMs, not cluster-wide access. - -### VM Security Hardening - -**Guest OS Security:** -- Regular patching and updates -- Disable unnecessary services -- Configure firewall rules -- Enable SELinux/AppArmor - -**Secrets Management:** -- Use Kubernetes Secrets for credentials -- Inject secrets into VMs via cloud-init -- Rotate secrets regularly -- Never store secrets in VM images - -**Access Control:** -- SSH key authentication only (disable password auth) -- Implement bastion/jump hosts -- Use VPN for remote access -- Audit access logs - ---- - -## Related Documentation - -- [Live Migration Best Practices](./live-migration-best-practices.md) - Configuration parameters and requirements -- [Performance Tuning](./performance-tuning.md) - Optimization strategies -- [Anti-Patterns](./anti-patterns.md) - Common mistakes to avoid - ---- - -**Last Updated**: 2026-02-24 -**OpenShift Virtualization Versions**: 4.17, 4.18, 4.19, 4.20 -**Status**: Production-ready guidance from official Red Hat sources diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/.ai-index/semantic-index.json deleted file mode 100644 index 2512c2a3..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/.ai-index/semantic-index.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "index_version": "1.0.0", - "last_updated": "2026-02-17", - "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", - - "documents": [ - { - "file": "INDEX.md", - "title": "VM Troubleshooting Guide - Index", - "category": "navigation", - "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], - "use_cases": ["navigation", "tool-reference", "general-guidance"], - "token_estimate": 450, - "skills": ["all"] - }, - { - "file": "scheduling-errors.md", - "title": "VM Scheduling Errors", - "category": "scheduling", - "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], - "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], - "errors_covered": [ - "ErrorUnschedulable - Node Taints", - "ErrorUnschedulable - Insufficient Resources", - "ErrorUnschedulable - Node Selector Mismatch" - ], - "token_estimate": 820, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "storage-errors.md", - "title": "VM Storage Errors", - "category": "storage", - "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], - "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], - "errors_covered": [ - "ErrorDataVolumeNotReady - DataVolume Still Provisioning", - "ErrorDataVolumeNotReady - Storage Class Not Found", - "ErrorDataVolumeNotReady - Insufficient Storage Quota", - "ErrorPvcNotFound", - "Storage Deletion Failures", - "DataVolume Cloning Failures" - ], - "token_estimate": 1350, - "skills": ["vm-creator", "vm-delete", "vm-clone"] - }, - { - "file": "lifecycle-errors.md", - "title": "VM Lifecycle Errors", - "category": "lifecycle", - "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], - "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], - "errors_covered": [ - "VM Stuck in Terminating State", - "VM Won't Start (Non-Scheduling Issues)", - "VM Won't Stop" - ], - "token_estimate": 1150, - "skills": ["vm-delete", "vm-lifecycle-manager"] - }, - { - "file": "runtime-errors.md", - "title": "VM Runtime Errors", - "category": "runtime", - "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], - "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], - "errors_covered": [ - "CrashLoopBackOff" - ], - "token_estimate": 850, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "network-errors.md", - "title": "VM Network Errors", - "category": "networking", - "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], - "use_cases": ["vm-creation", "network-troubleshooting"], - "errors_covered": [ - "Network Attachment Failures" - ], - "token_estimate": 580, - "skills": ["vm-creator"] - } - ], - - "skill_to_docs_mapping": { - "vm-creator": { - "primary_docs": ["scheduling-errors.md", "storage-errors.md"], - "secondary_docs": ["runtime-errors.md", "network-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3650, - "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" - }, - "vm-delete": { - "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 2950, - "description": "VM deletion failures - stuck terminating, storage cleanup" - }, - "vm-clone": { - "primary_docs": ["storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 1800, - "description": "VM cloning failures - DataVolume cloning, storage provisioning" - }, - "vm-lifecycle-manager": { - "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], - "secondary_docs": ["runtime-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3270, - "description": "VM start/stop failures - lifecycle management, scheduling" - }, - "vm-inventory": { - "primary_docs": [], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 450, - "description": "General guidance for status interpretation" - } - }, - - "error_to_docs_mapping": { - "ErrorUnschedulable": "scheduling-errors.md", - "ErrorDataVolumeNotReady": "storage-errors.md", - "ErrorPvcNotFound": "storage-errors.md", - "Terminating": "lifecycle-errors.md", - "CrashLoopBackOff": "runtime-errors.md", - "NetworkAttachmentFailed": "network-errors.md" - }, - - "cross_references": { - "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], - "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], - "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], - "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], - "network-errors.md": ["scheduling-errors.md"] - }, - - "progressive_disclosure_strategy": { - "description": "Load only the minimum documentation needed for current task", - "pattern": "error_type → primary_doc → cross_references (if needed)", - "token_savings": "29% average reduction vs single-file (based on rh-sre data)", - "navigation_overhead_reduction": "85% vs single-file navigation" - } -} diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/INDEX.md deleted file mode 100644 index a2e76963..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/INDEX.md +++ /dev/null @@ -1,332 +0,0 @@ ---- -title: VM Troubleshooting Guide - Index -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 - - title: OpenShift Virtualization - Virtual Machine Status - url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html - date_accessed: 2026-02-06 - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] -semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] -use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] -last_updated: 2026-02-17 ---- - -# VM Troubleshooting Guide - Index - -## Overview - -This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: - -- **Creation**: Scheduling failures, storage provisioning issues -- **Lifecycle**: Start/stop failures, stuck states -- **Deletion**: Resources stuck in Terminating, storage cleanup failures -- **Cloning**: DataVolume cloning errors, cross-namespace issues -- **Networking**: Secondary network attachment failures -- **Runtime**: CrashLoopBackOff, guest OS boot failures - -This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. - ---- - -## 🗂️ Troubleshooting Categories - -The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: - -### 1. [Scheduling Errors](scheduling-errors.md) -**When to use**: VM fails to schedule on any node - -**Errors covered**: -- ErrorUnschedulable - Node Taints -- ErrorUnschedulable - Insufficient Resources -- ErrorUnschedulable - Node Selector Mismatch - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 2. [Storage Errors](storage-errors.md) -**When to use**: VM has storage provisioning, deletion, or cloning issues - -**Errors covered**: -- ErrorDataVolumeNotReady (all 3 subsections) -- ErrorPvcNotFound -- Storage Deletion Failures -- DataVolume Cloning Failures - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### 3. [Lifecycle Errors](lifecycle-errors.md) -**When to use**: VM has start/stop/termination issues - -**Errors covered**: -- VM Stuck in Terminating State -- VM Won't Start (Non-Scheduling Issues) -- VM Won't Stop - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### 4. [Runtime Errors](runtime-errors.md) -**When to use**: VM repeatedly crashes or fails at runtime - -**Errors covered**: -- CrashLoopBackOff - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 5. [Network Errors](network-errors.md) -**When to use**: VM has secondary network attachment failures - -**Errors covered**: -- Network Attachment Failures (Multus, SR-IOV) - -**Skills that use this**: vm-create - ---- - -## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last - -### Critical Principle: MCP-First Approach - -**All diagnostic procedures in this guide follow the MCP-first pattern**: - -``` -1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) -2. ⚠️ IF FAILS: Ask user permission to use CLI command -3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval -``` - -### Why MCP Tools First? - -- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources -- **Consistency**: Same interface across all operations -- **Better Error Handling**: MCP tools return structured errors -- **Audit Trail**: MCP tool usage is logged and trackable - -### Command Pattern Examples - -Throughout this guide, you'll see diagnostic steps formatted like this: - -**✅ CORRECT Pattern (MCP First)**: - -```markdown -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Extract**: `.metadata.finalizers` from returned JSON - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` -``` - -### Available MCP Tools - -The `openshift-virtualization` MCP server provides these tools: - -**Resource Operations**: -- `resources_get` - Get specific resource (replaces `oc get `) -- `resources_list` - List resources (replaces `oc get `) -- `resources_delete` - Delete resource (replaces `oc delete`) -- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) - -**Pod Operations**: -- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) -- `pods_get` - Get pod details (replaces `oc get pod`) -- `pods_log` - Get pod logs (replaces `oc logs`) -- `pods_exec` - Execute in pod (replaces `oc exec`) -- `pods_delete` - Delete pod (replaces `oc delete pod`) -- `pods_top` - Pod resource usage (replaces `oc top pods`) - -**Events & Monitoring**: -- `events_list` - List events (replaces `oc get events`) -- `nodes_top` - Node resource usage (replaces `oc top nodes`) -- `nodes_log` - Node logs (replaces `oc adm node-logs`) -- `nodes_stats_summary` - Detailed node stats - -**VM Operations** (KubeVirt toolset): -- `vm_create` - Create VMs -- `vm_lifecycle` - Start/stop/restart VMs - -**Namespaces**: -- `namespaces_list` - List namespaces (replaces `oc get namespaces`) -- `projects_list` - List OpenShift projects (replaces `oc get projects`) - -### When CLI Commands Are Required - -Some operations have **NO MCP equivalent** and require CLI: - -- `virtctl` commands (console, VNC access) -- `oc debug node` (node debugging) -- `oc auth can-i` (permission checks) -- `oc adm taint` (node taint management) -- Complex JSON patch operations - -For these, the guide will note: **"CLI Only - No MCP equivalent"** - -### Quick Reference: CLI → MCP Mapping - -| CLI Command | MCP Tool Equivalent | -|-------------|---------------------| -| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | -| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | -| `oc logs -n ` | `pods_log` with name="", namespace="" | -| `oc get events -n ` | `events_list` with namespace="" | -| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | - -**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. - -### How to Read Diagnostic Sections - -Each error section includes: -1. **Symptom** - What you observe -2. **Description** - What's happening -3. **Possible Causes** - Why it's happening -4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback -5. **Solutions** - **MCP tools first**, then CLI fallback -6. **Verification** - **MCP tools first**, then CLI fallback - -**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. - ---- - -## 🔍 Quick Navigation by Skill - -**vm-create**: -- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics -- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning -- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff -- [Network Errors](network-errors.md) - Network attachment failures - -**vm-delete**: -- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state -- [Storage Errors](storage-errors.md) - Storage deletion failures - -**vm-clone**: -- [Storage Errors](storage-errors.md) - DataVolume cloning failures - -**vm-lifecycle-manager**: -- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop -- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling - -**vm-inventory**: -- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed - ---- - -## 📊 Documentation Coverage & Maintenance - -### Current Coverage - -This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: - -- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) -- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures -- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop -- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) -- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems - -**Total errors documented**: 12 error types across 6 categories - ---- - -### Encountering Undocumented Errors - -#### For AI Agents (Claude Code) - -If you encounter an error **not documented** in the categories above: - -1. **Report to user** with all available details (error message, affected resources, namespace) -2. **Provide best-effort diagnostics** using MCP tools: - - `resources_get` to inspect resource status - - `pods_log` to check virt-launcher or compute container logs - - `events_list` to view Kubernetes events -3. **Suggest documentation update**: - ``` - ⚠️ This error is not yet documented in the troubleshooting guide. - - **Error**: - **Affected resource**: / - - I recommend adding this error to the troubleshooting guide: - - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file - - If it's a new error class → Create new category file in docs/troubleshooting/ - - Would you like me to help document this error for future reference? - ``` - -4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs - ---- - -#### For Human Contributors - -To document a new error: - -1. **Determine the category**: - - Scheduling issue → `scheduling-errors.md` - - Storage problem → `storage-errors.md` - - Lifecycle/start/stop → `lifecycle-errors.md` - - Runtime crash → `runtime-errors.md` - - Network issue → `network-errors.md` - - New error class → Create new category file (e.g., `security-errors.md`) - -2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): - - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback - - **Solutions**: MCP Tool approach → Implementation → CLI Fallback - - **Verification**: MCP Tool checks → Expected results → CLI Fallback - -3. **Update semantic index**: - - Edit `.ai-index/semantic-index.json` - - Add error to `error_to_docs_mapping` - - Update relevant skill's `primary_docs` or `secondary_docs` - - Increment token estimates if needed - -4. **Reference in skill files**: - - Update skill's Reference Documentation section - - Ensure skill's Document Consultation steps point to the right category - -5. **Update this INDEX.md**: - - Add error to the appropriate category section (lines 50-120) - - Update "Current Coverage" list above - ---- - -## 📚 Additional Resources - -- [rh-virt README](../../README.md) - MCP server setup and skill documentation -- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources -- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/lifecycle-errors.md deleted file mode 100644 index 6167fcc8..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/lifecycle-errors.md +++ /dev/null @@ -1,869 +0,0 @@ ---- -title: VM Lifecycle Errors -category: kubevirt -sources: - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 -tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] -semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] -use_cases: [vm-deletion, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Lifecycle Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM lifecycle issues including start/stop failures and stuck termination states. - -**When to use this document**: -- VM stuck in `Terminating` state -- VM won't start (runStrategy is Always but VM never reaches Running) -- VM won't stop (runStrategy is Halted but VM never reaches Stopped) - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### VM Stuck in Terminating State - -**Symptom**: VM shows status `Terminating` but deletion never completes - -**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). - -**Possible Causes**: -- Finalizers blocking deletion -- PVC/DataVolume still attached and preventing cleanup -- VirtualMachineInstance (VMI) not terminating properly -- Custom controllers or operators blocking deletion -- Stuck virt-launcher pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check finalizers on the VM**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` - -**2. Check if VMI still exists**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name containing "virt-launcher-". - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check events for deletion issues**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results where `involvedObject.name` == "" and sort by timestamp. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' -``` - -**5. Check VM deletion timestamp**: - -Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. - -If present, VM is in deletion process. If null, VM is not being deleted. - -**Common Finalizer Patterns**: -- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer -- `foregroundDeletion` - Waits for dependent objects to be deleted -- Custom finalizers from operators - -**Solutions** (Use MCP Tools First): - -1. **Wait for dependent resources** (safest approach): - - Use `resources_list` to check if PVCs, DataVolumes are still deleting - - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) - -2. **Force delete VMI** (if VMI is stuck): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable - requires explicit user permission): - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -3. **Force delete virt-launcher pod** (if pod is stuck): - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - # Find the virt-launcher pod name first - oc get pods -n | grep virt-launcher- - - # Then delete it - oc delete pod virt-launcher--xxx -n --force --grace-period=0 - ``` - -4. **Remove finalizers** (⚠️ dangerous - use only as last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` - 2. Remove items from `.metadata.finalizers` array - 3. Update VM using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch not easily done via MCP): - ```bash - # This operation is complex for MCP - may need CLI - oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Check for protection labels** (vm-delete skill specific): - - Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. - - If value is "true", the vm-delete skill refuses deletion (this is expected behavior). - -**Verification**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error if deletion successful. - -**CLI Fallback**: -```bash -oc get vm -n -# Should return: Error from server (NotFound) -``` - ---- - - ---- - -### VM Won't Start (Non-Scheduling Issues) - -**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state - -**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. - -**Possible Causes**: -- Guest OS kernel panic or boot failure -- Cloud-init configuration errors -- Missing or corrupted disk image -- Insufficient memory for guest OS to boot -- QEMU/KVM virtualization errors -- VirtualMachineInstance (VMI) creation failures -- virt-launcher pod crashes - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VMI (VirtualMachineInstance) status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and overall VMI state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**2. Check VMI conditions for errors**: - -Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. View virt-launcher pod logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -First, get pod name from step 3, then: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -Look for QEMU errors, memory allocation failures, device errors. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**5. Check virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. - -Alternatively, use `pods_get` to get full pod details: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**6. Access VM console to see guest OS boot messages**: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -Look for kernel panic, initramfs errors, filesystem errors. - -**7. Check VM events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep -A 20 "Events:" -``` - -**Common Error Patterns**: - -1. **Guest OS Boot Failure**: - - Console shows kernel panic - - Guest hangs at GRUB or boot loader - - Cloud-init errors during first boot - -2. **Insufficient Memory**: - - Guest OS kills processes due to OOM - - VMI logs show memory allocation errors - -3. **QEMU Crashes**: - - virt-launcher logs show QEMU segfaults - - VMI repeatedly restarts - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for kernel panic, initramfs errors, filesystem errors. - -2. **Check virt-launcher pod logs for QEMU errors**: - - Use `pods_log` from diagnostic step 4 to view logs. - - Look for: - - "qemu-system-x86_64: ..." errors - - Memory allocation failures - - Device errors - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx - ``` - -3. **Increase memory if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check VMI memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -4. **Verify disk image integrity**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check PVC is bound: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - If using container disk, verify image exists and is accessible (check VMI spec). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get pvc -n - ``` - -5. **Check cloud-init configuration** (if applicable): - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. - - Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq - ``` - -6. **Restart VMI** (soft reset): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete VMI (VM controller will recreate it): - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for new VMI to start (use `resources_get` to check status). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n -w - ``` - -7. **Check virtualization extensions** (KVM): - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - - # In debug shell: - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should eventually return `Running`). - -Check VMI is running: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should show `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should eventually return: Running - -oc get vmi -n -# Should show: Running -``` - ---- - -### VM Won't Stop - -**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state - -**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. - -**Possible Causes**: -- Guest OS not responding to ACPI shutdown signal -- virt-launcher pod stuck and not terminating -- VirtualMachineInstance (VMI) deletion blocked by finalizers -- Guest shutdown scripts hanging -- Filesystem sync issues in guest OS - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (might show `Stopping`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**2. Check VMI status and deletion timestamp**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check VMI events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vmi -n | grep -A 10 "Events:" -``` - -**5. Check VMI finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.metadata.finalizers}' -``` - -**6. Check if guest is responsive** (if VMI still exists): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Patterns**: -- VMI shows `deletionTimestamp` but never actually deletes -- virt-launcher pod in `Terminating` state -- VM runStrategy is `Halted` but printableStatus shows `Stopping` - -**Solutions** (Use MCP Tools First): - -1. **Wait for graceful shutdown** (default: 30 seconds): - - Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. - -2. **Force stop by deleting VMI**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - This is the standard way to force-stop a VM. - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n - # Should return: Error from server (NotFound) - ``` - -3. **Force delete VMI with grace period** (if VMI won't delete): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -4. **Force delete virt-launcher pod**: - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find the pod using `pods_list_in_namespace` from diagnostic step 3. - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - ⚠️ **Note**: For force deletion with grace period, use CLI fallback. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') - oc delete pod $POD_NAME -n --force --grace-period=0 - ``` - -5. **Remove VMI finalizers** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VMI using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update VMI using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Patch VM runStrategy directly** (ensure consistency): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` (diagnostic step 1) - 2. Set `.spec.runStrategy` to `"Halted"` - 3. Update VM using `resources_create_or_update` with modified JSON - - **CLI Fallback** (merge patch easier via CLI): - Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Stopped` or `Halted`). - -Verify VMI is gone: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Verify virt-launcher pod is gone: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for pods matching `virt-launcher-`. Should return no results. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Stopped or Halted - -oc get vmi -n -# Should return: Error from server (NotFound) - -oc get pods -n | grep virt-launcher- -# Should return: No resources found -``` - -**Prevention**: -- Ensure guest OS has ACPI support enabled -- Use proper shutdown commands in guest OS -- Avoid forceful stops unless necessary (can corrupt guest filesystem) - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/network-errors.md deleted file mode 100644 index 82fa313c..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/network-errors.md +++ /dev/null @@ -1,429 +0,0 @@ ---- -title: VM Network Errors -category: kubevirt -sources: - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] -semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] -use_cases: [vm-creation, network-troubleshooting] -related_docs: [INDEX.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Network Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. - -**When to use this document**: -- VM created successfully but secondary networks not attached -- NetworkAttachmentDefinition not found errors -- Multus CNI failures -- SR-IOV device attachment issues - -**Skills that use this**: vm-create - ---- - -### Network Attachment Failures - -**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working - -**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). - -**Possible Causes**: -- NetworkAttachmentDefinition doesn't exist in the namespace -- Multus CNI not installed or not configured on cluster -- Namespace mismatch (NAD in different namespace than VM) -- Interface name conflicts in VM spec -- Bridge/network configuration errors in NAD -- SR-IOV device not available (if using SR-IOV) - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List NetworkAttachmentDefinitions in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "" -} -``` - -Review `.items[].metadata.name` for available NADs. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -n -``` - -**2. List NetworkAttachmentDefinitions in all namespaces**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition" -} -``` - -Omit `namespace` parameter to list across all namespaces. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -A -``` - -**3. Check specific NetworkAttachmentDefinition**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "", - "name": "" -} -``` - -Review `.spec.config` for CNI configuration. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definition -n -o yaml -``` - -**4. Check VM network configuration**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.networks` to see network references. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq -``` - -**5. Check VM domain interfaces**: - -Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq -``` - -**6. Check virt-launcher pod network annotations** (shows actual attachments): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**7. Check Multus is installed**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "openshift-multus" -} -``` - -Should show Multus CNI pods running. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n openshift-multus -``` - -**8. Check for errors in virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n -``` - -**Common Error Messages**: -- `"network-attachment-definition not found"` - NAD doesn't exist in namespace -- `"multus CNI not configured"` - Multus not installed or misconfigured -- `"interface name conflict"` - Duplicate interface names in VM spec -- `"failed to add network"` - CNI plugin error (check NAD config) -- `"no available devices"` - SR-IOV device not available (if using SR-IOV) - -**Solutions** (Use MCP Tools First): - -1. **Verify NetworkAttachmentDefinition exists in correct namespace**: - - Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. - - If NAD is in different namespace, copy it to VM namespace: - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get NAD from source namespace using `resources_get` - 2. Modify `.metadata.namespace` to target namespace - 3. Create NAD in target namespace using `resources_create_or_update` - - **CLI Fallback** (stream processing easier via CLI): - Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" - ```bash - oc get network-attachment-definition -n -o yaml | \ - sed "s/namespace: /namespace: /" | \ - oc apply -f - - ``` - -2. **Create missing NetworkAttachmentDefinition**: - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - Example: Linux bridge network - - **Parameters**: - ```json - { - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "metadata": { - "name": "vlan100", - "namespace": "" - }, - "spec": { - "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" - ```bash - cat < - spec: - config: '{ - "cniVersion": "0.3.1", - "type": "bridge", - "bridge": "br1", - "vlan": 100, - "ipam": { - "type": "host-local", - "subnet": "192.168.100.0/24" - } - }' - EOF - ``` - -3. **Check Multus CNI installation**: - - Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. - - To check cluster network operator: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "config.openshift.io/v1", - "kind": "ClusterOperator" - } - ``` - - Filter for `network` operator. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n openshift-multus - oc get clusteroperators network - ``` - -4. **Fix interface name conflicts** (if VM has duplicate names): - - Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. - - Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. - - **CLI Fallback** (interactive edit easier via CLI): - Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' - oc edit vm -n - ``` - -5. **Validate NAD configuration syntax**: - - Use `resources_get` from diagnostic step 3, extract `.spec.config`. - - Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get network-attachment-definition -n -o jsonpath='{.spec.config}' - ``` - -6. **Check SR-IOV device availability** (if using SR-IOV networks): - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - List SR-IOV network node policies: - - **Parameters**: - ```json - { - "apiVersion": "sriovnetwork.openshift.io/v1", - "kind": "SriovNetworkNodePolicy", - "namespace": "openshift-sriov-network-operator" - } - ``` - - Check SR-IOV device plugin pods: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "openshift-sriov-network-operator" - } - ``` - - Filter for pods with "device-plugin" in name. - - Check available SR-IOV devices on node: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "Node", - "name": "" - } - ``` - - Review `.status.allocatable` for SR-IOV resources. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get sriovnetworknodepolicy -n openshift-sriov-network-operator - oc get pods -n openshift-sriov-network-operator | grep device-plugin - oc describe node | grep -A 10 "Allocatable:" - ``` - -7. **Recreate VM with corrected network configuration** (if needed): - - If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -After remediation, check virt-launcher pod network status: - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -Should show all attached networks with status. Example output: -```json -[ - { - "name": "openshift-sdn", - "interface": "eth0", - "ips": ["10.128.2.10"], - "default": true - }, - { - "name": "vlan100", - "interface": "net1", - "ips": ["192.168.100.5"] - } -] -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**Check from inside VM** (via console): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# In guest OS: -ip addr show -# Should show all network interfaces (eth0, net1, etc.) -``` - -**Common Network Types**: -- **Linux Bridge**: Layer 2 bridge for VLAN networks -- **SR-IOV**: High-performance direct device assignment -- **macvlan**: MAC-based VLAN for container networks -- **OVN-Kubernetes**: OpenShift native overlay network - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/runtime-errors.md deleted file mode 100644 index 804d3cbc..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/runtime-errors.md +++ /dev/null @@ -1,616 +0,0 @@ ---- -title: VM Runtime Errors -category: kubevirt -sources: - - title: Kubernetes Pod Lifecycle - url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ - date_accessed: 2026-02-17 -tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] -semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] -use_cases: [vm-creation, vm-lifecycle, diagnostics] -related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Runtime Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. - -**When to use this document**: -- VM or virt-launcher pod shows `CrashLoopBackOff` status -- virt-launcher pod repeatedly restarting -- Guest OS kernel panics on boot - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### CrashLoopBackOff - -**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting - -**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. - -**Possible Causes**: -- Guest OS kernel panic on boot -- Insufficient resources (memory/CPU) for guest OS -- Corrupted disk image or filesystem -- QEMU/libvirt crashes due to configuration errors -- Missing or incompatible device drivers in guest -- Resource limits too low for virt-launcher pod -- Virtualization features (KVM) not available on node - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check virt-launcher pod restart count**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# Look at RESTARTS column -``` - -**2. View recent crash logs** (previous container instance): - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "previous": true, - "tail": 100 -} -``` - -Look for QEMU errors, kernel panics, or segfaults. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n virt-launcher--xxx --previous -``` - -**3. Check current virt-launcher logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. - -**CLI Fallback** (if MCP unavailable or all containers needed): -```bash -oc logs -n virt-launcher--xxx --all-containers -``` - -**4. Check VMI conditions for crash details**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Extract `.status.conditions` for crash details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**5. Check pod events for crash reasons**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" -``` - -**6. Check pod resource limits**: - -**MCP Tool**: `pods_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -Extract `.spec.containers[0].resources` for resource limits. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq -``` - -**7. Check node kubelet logs for OOM kills**: - -⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -oc adm node-logs -u kubelet | grep -i oom -``` - -**8. Access guest console** (if VM briefly starts): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Crash Patterns**: - -1. **Guest Kernel Panic**: - - Console logs show kernel panic messages - - Guest crashes immediately after boot - - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" - -2. **OOM (Out of Memory)**: - - Pod killed with reason: `OOMKilled` - - Guest runs out of memory during boot or operation - - virt-launcher logs show memory allocation failures - -3. **QEMU Crash**: - - virt-launcher logs show QEMU segmentation fault - - Symptoms: "qemu-system-x86_64: terminated by signal" - - Configuration incompatibility or QEMU bug - -4. **Disk Image Corruption**: - - Guest cannot boot from disk - - Filesystem errors in guest console - - DataVolume import failed - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for kernel panic or boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for: - - Kernel panic messages - - Initramfs errors - - Filesystem mounting failures - - Missing device errors - -2. **Review virt-launcher crash logs**: - - Use `pods_log` with `previous: true` from diagnostic step 2. - - Look for: - - QEMU command line errors - - Device initialization failures - - Memory allocation errors - - Signal termination (SIGSEGV, SIGABRT) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx --previous - ``` - -3. **Check for OOM (Out of Memory) kills**: - - **MCP Tool**: `pods_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - Extract `.status.containerStatuses[0].lastState.terminated.reason`. - - If returns `"OOMKilled"`: - - Option 1: Increase virt-launcher memory limits - - Option 2: Decrease guest memory allocation - - Option 3: Use smaller instance type - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' - ``` - -4. **Increase resources if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check current memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -5. **Verify disk image integrity**: - - **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters for list**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" - } - ``` - - **Parameters for specific DV**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check `.status.phase` (should be `Succeeded`). - - If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get datavolume -n -o jsonpath='{.status.phase}' - ``` - -6. **Check virtualization (KVM) availability**: - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -7. **Simplify VM configuration** (eliminate variables): - - Try creating minimal VM using vm-create skill with: - - Small instance type - - No secondary networks - - Simple container disk (e.g., Fedora) - - No cloud-init - - If minimal VM works, add features back one by one. - -8. **Recreate VM with different workload** (test disk image): - - If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) - -After remediation, check pod restart count stops increasing: - -**Parameters for pods**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). - -Check VM reaches Running state: - -**Parameters for VM**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# RESTARTS should stabilize - -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Running -``` - -Verify guest is responsive: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# Should show login prompt or OS console -``` - -**Advanced Debugging**: - -**MCP Tool**: `pods_exec` (from openshift-virtualization) - -Check libvirt domain XML: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["virsh", "dumpxml", "1"] -} -``` - -Check QEMU process: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["ps", "aux"] -} -``` - -Filter output for "qemu" process. - -**CLI Fallback** (if MCP unavailable): -```bash -oc exec -n virt-launcher--xxx -- virsh dumpxml 1 -oc exec -n virt-launcher--xxx -- ps aux | grep qemu -``` - -**Prevention**: -- Start with minimal VM configuration and add complexity gradually -- Use recommended instance types for your workload -- Test disk images before deploying to production -- Ensure nodes have adequate resources and KVM support -- Monitor resource usage and set appropriate limits - ---- - -## Workaround Patterns for MCP Tool Limitations - -### General Pattern: Diagnose → Propose → Confirm → Execute - -When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): - -**1. Diagnose** the root cause using Kubernetes commands -```bash -oc describe vm -n -oc get events -n --field-selector involvedObject.name= -``` - -**2. Propose** a manual workaround to the user with clear explanation -- Explain why the issue occurred -- Show what will be changed -- List alternative options - -**3. Confirm** with user before executing (Human-in-the-Loop) -- Wait for explicit "yes" or "apply workaround" -- Never auto-execute modifications - -**4. Execute** the workaround using oc/kubectl -```bash -oc patch vm -n --type=merge -p '...' -``` - -**5. Verify** the fix was successful -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**6. Document** the limitation and suggest filing enhancement request -- Note this is temporary until MCP tool is enhanced -- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues - -### Example: Adding Tolerations Workaround - -**Diagnostic Output**: -```markdown -## ⚠️ VM Scheduling Issue Detected - -**Root Cause**: Node taints prevent VM scheduling - -**Details**: -- Found 3 nodes with taint: `virtualization=true:NoSchedule` -- VM spec does not include matching tolerations -- This prevents VM scheduling on virtualization-dedicated nodes -``` - -**Proposed Workaround**: -```bash -oc patch vm web-server -n vms --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' -``` - -**User Confirmation Required**: -``` -How would you like to proceed? -- "apply workaround" - I'll patch the VM with tolerations -- "manual" - I'll provide instructions for you to apply manually -- "cancel" - Delete the VM and abort creation -``` - -**After Execution**: -```markdown -## ✓ Workaround Applied Successfully - -**Action**: Added tolerations for taint `virtualization=true:NoSchedule` -**New Status**: Stopped (VM can now be scheduled) - -**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. -Future VMs in this cluster will need the same fix until the tool is enhanced. -``` - ---- - -## VM Status Reference - -### Status Values - -| Status | Meaning | Action Required | -|--------|---------|-----------------| -| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | -| `Running` | VM is running | Normal | -| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | -| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | -| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | -| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | -| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | -| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | -| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | -| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | - -### Checking VM Status - -```bash -# Get printable status -oc get vm -n -o jsonpath='{.status.printableStatus}' - -# Get detailed status and conditions -oc get vm -n -o jsonpath='{.status}' | jq - -# Watch status changes in real-time -oc get vm -n -w -``` - ---- - -## Best Practices for Agents - -When implementing diagnostic workflows: - -1. **Always verify VM status** after creation (wait 5-10 seconds first) -2. **Consult this document** when encountering error status values -3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) -4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) -5. **Respect human-in-the-loop** for all VM modifications -6. **Document temporary workarounds** and their limitations clearly -7. **Suggest filing issues** for missing MCP tool features - -### Document Consultation Pattern - -```markdown -**Document Consultation** (REQUIRED): -1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes -2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." -``` - ---- - -## Known MCP Tool Limitations - -### vm_create tool - -**Currently Supported**: -- ✓ Namespace, name (required) -- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) -- ✓ Size hints (small, medium, large) -- ✓ Storage size -- ✓ Autostart flag -- ✓ Networks (Multus NetworkAttachmentDefinitions) -- ✓ Performance family (u1, o1, c1, m1) -- ✓ Instance type, preference - -**Not Currently Supported** (requires workarounds): -- ✗ Tolerations (for node taints) -- ✗ Node selectors -- ✗ Affinity/anti-affinity rules -- ✗ Resource requests/limits (beyond instance type) -- ✗ Custom labels/annotations -- ✗ SSH keys injection -- ✗ Cloud-init user data - -**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. - -**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues - ---- - -## Additional Resources - -- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) -- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) -- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) -- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) - ---- - -[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/scheduling-errors.md deleted file mode 100644 index e47c8c4f..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/scheduling-errors.md +++ /dev/null @@ -1,417 +0,0 @@ ---- -title: VM Scheduling Errors -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 -tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] -semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] -use_cases: [vm-creation, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Scheduling Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**When to use this document**: -- VM shows status `ErrorUnschedulable` after creation or start attempt -- VM events mention scheduling failures, taints, resources, or node selectors - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -## ErrorUnschedulable - -**Symptom**: VM shows status `ErrorUnschedulable` after creation - -**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**Possible Causes**: - -### 1. Node Taints (Most Common) - -Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM events for scheduling failures**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `involvedObject.name` == "" and look for messages like: -- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= -``` - -**2. Check node taints in the cluster**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' -``` - -**Common Taint Patterns**: -- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule -- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes -- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads - -**Solution - Add Tolerations to VM**: - -The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. - -**Workaround (post-creation using MCP Tools)**: - -**Step 1**: Get current VM spec - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Step 2**: Modify the returned JSON to add tolerations - -Add to `.spec.template.spec.tolerations`: -```json -{ - "tolerations": [ - { - "key": "virtualization", - "operator": "Equal", - "value": "true", - "effect": "NoSchedule" - } - ] -} -``` - -**Step 3**: Update VM with modified spec - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters**: -```json -{ - "resource": "" -} -``` - -Pass the complete modified VM resource as YAML or JSON string. - -**Step 4**: Verify tolerations were added - -Use `resources_get` again and check `.spec.template.spec.tolerations` in response. - -**Step 5**: Check if VM status improved - -Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. - -**CLI Fallback** (if MCP patch is too complex): -```bash -# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' - -# Verify tolerations -oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq - -# Check status -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**Example - Multiple Tolerations**: -```bash -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" - - key: "dedicated" - operator: "Equal" - value: "virt-workloads" - effect: "NoSchedule" -' -``` - -**Toleration Operators**: -- `Equal` - Key and value must match exactly -- `Exists` - Only key must exist (ignores value) - -**Toleration Effects**: -- `NoSchedule` - Don't schedule new pods (existing pods continue) -- `PreferNoSchedule` - Avoid scheduling if possible -- `NoExecute` - Don't schedule AND evict existing pods - -**Alternative Solutions**: -1. **Remove node taints** (if you have cluster-admin access): - ```bash - oc adm taint nodes virtualization=true:NoSchedule- - ``` - -2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits - -3. **File enhancement request** - Request tolerations support in openshift-mcp-server: - https://github.com/openshift/openshift-mcp-server/issues - ---- - -### 2. Insufficient Resources - -Not enough CPU, memory, or storage available on any node. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM resource requests**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' -``` - -**2. Check node resource availability**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. - -Alternatively, use `nodes_top` MCP tool for current resource usage. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe nodes | grep -A 5 "Allocated resources" -``` - -**3. Look for VM events mentioning "Insufficient"**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep "Insufficient" -``` - -**Example Event**: -``` -0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. -``` - -**Solutions** (Use MCP Tools First): - -1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) -2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill -3. **Delete unused VMs** - Use vm-delete skill to free up resources -4. **Check resource quotas**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters for quota**: - ```json - { - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" - } - ``` - - **Parameters for limit range**: - ```json - { - "apiVersion": "v1", - "kind": "LimitRange", - "namespace": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - oc describe limitrange -n - ``` - ---- - -### 3. Node Selector Mismatch - -VM requires specific node labels that don't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM node selector requirements**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.nodeSelector` to see required node labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' -``` - -**2. List available node labels**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.metadata.labels` for available labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes --show-labels -``` - -**3. Check if any nodes match the selector**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node", - "labelSelector": "=" -} -``` - -Should return at least one node with matching labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -l = -``` - -**Solutions** (Use MCP Tools First): - -**Option 1: Remove node selector from VM** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get current VM using `resources_get` (diagnostic step 1) -2. Remove `.spec.template.spec.nodeSelector` field -3. Update VM using `resources_create_or_update` with modified JSON - -**CLI Fallback** (JSON patch easier via CLI): -Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" -```bash -oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' -``` - -**Option 2: Add label to nodes** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get node using `resources_get` -2. Add label to `.metadata.labels` -3. Update node using `resources_create_or_update` - -⚠️ **Note**: Node labeling typically requires cluster admin privileges. - -**CLI Fallback** (simpler via CLI): -Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" -```bash -oc label node = -``` - ---- - -[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/storage-errors.md deleted file mode 100644 index 16ba584d..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/storage-errors.md +++ /dev/null @@ -1,1011 +0,0 @@ ---- -title: VM Storage Errors -category: kubevirt -sources: - - title: KubeVirt CDI - DataVolumes - url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ - date_accessed: 2026-02-17 - - title: Kubernetes Persistent Volumes - url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ - date_accessed: 2026-02-17 -tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] -semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] -use_cases: [vm-creation, vm-deletion, vm-cloning] -related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] -last_updated: 2026-02-17 ---- - -# VM Storage Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. - -**When to use this document**: -- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` -- Storage deletion fails after VM deletion -- DataVolume cloning operations fail -- PVC provisioning issues - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### ErrorDataVolumeNotReady - -**Symptom**: VM shows status `ErrorDataVolumeNotReady` - -**Description**: The DataVolume (persistent storage) backing the VM is not ready. - -**Possible Causes**: - -#### 1. DataVolume Still Provisioning - -Storage provisioning takes time, especially for large disks or when importing images. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Get detailed DataVolume information**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` and `.status.conditions` for provisioning details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o json -``` - -**3. Check PVC (PersistentVolumeClaim) bound status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Check `.status.phase` for each PVC (should be `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. - -#### 2. Storage Class Not Found - -The requested storage class doesn't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List available storage classes**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" -} -``` - -Review the list of available storage classes (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -``` - -**2. Check DataVolume's requested storage class**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.spec.pvc.storageClassName` in the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' -``` - -**Solution**: -1. Use a valid storage class from the cluster -2. Recreate VM with correct storage class parameter - -#### 3. Insufficient Storage Quota - -Namespace has insufficient storage quota to provision the PVC. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check resource quotas**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**2. Check storage usage**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase -``` - -**Solution**: -1. Request quota increase from cluster admin -2. Delete unused PVCs to free quota -3. Reduce VM storage size - ---- - -### ErrorPvcNotFound - -**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List PVCs in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review the list of available PVCs (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check VM's PVC references**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' -``` - -**Solution**: -- Wait for DataVolume to create the PVC -- Manually create missing PVC -- Fix VM spec to reference correct PVC name - ---- - - ---- - -### Storage Deletion Failures - -**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace - -**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. - -**Possible Causes**: -- PVC still bound to active PersistentVolume with `Retain` policy -- DataVolume still being referenced by another resource -- CDI (Containerized Data Importer) controller issues -- Storage class retention policy preventing deletion -- Finalizers on PVC/DataVolume blocking cleanup -- PVC still mounted by a pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check PVC status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check specific PVC phase**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Released` or `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o jsonpath='{.status.phase}' -``` - -**3. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**4. Check what's using the PVC**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' -``` - -**5. Check PVC finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. - -**6. Check DataVolume finalizers**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.metadata.finalizers}' -``` - -**7. Check PV reclaim policy**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume" -} -``` - -Filter results for PV where `.spec.claimRef.name` matches ``. - -To get specific PV policy: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume", - "name": "" -} -``` - -Check `.spec.persistentVolumeReclaimPolicy`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pv | grep -oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' -``` - -**Common Finalizer Patterns**: -- `kubernetes.io/pvc-protection` - Protects PVC while in use -- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer - -**Solutions** (Use MCP Tools First): - -1. **Delete DataVolume first, then PVC**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete DataVolume first (often blocks PVC deletion): - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Wait a few seconds, then delete PVC: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete datavolume -n - oc delete pvc -n - ``` - -2. **Check for pods still using PVC**: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "" - } - ``` - - Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. - - Then delete the pods using `pods_delete`: - - **Parameters**: - ```json - { - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' - oc delete pod -n - ``` - -3. **Force delete PVC** (if safe to do so): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete pvc -n --grace-period=0 --force - ``` - -4. **Remove finalizers from PVC** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PVC using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update PVC using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Remove finalizers from DataVolume** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current DataVolume using `resources_get` (diagnostic step 6) - 2. Remove items from `.metadata.finalizers` array - 3. Update DataVolume using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Change PV reclaim policy** (if PV has Retain policy): - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PV policy using `resources_get` (diagnostic step 7) - 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` - 3. Update PV using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" - ```bash - oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' - oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' - ``` - -**Storage Quota Check** (Use MCP Tools First): - -After deletion, verify storage quota is freed: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters for quota check**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.used` to verify storage quota is freed. - -**Parameters for PVC verification**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -oc get pvc -n -``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Confirm PVC is deleted: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Confirm DataVolume is deleted: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -# Should return: Error from server (NotFound) - -oc get datavolume -n -# Should return: Error from server (NotFound) -``` - ---- - -### DataVolume Cloning Failures - -**Symptom**: VM clone created successfully but DataVolume clone operation fails - -**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. - -**Possible Causes**: -- CSI driver doesn't support volume cloning -- Source PVC storage class incompatible with cloning -- Cross-namespace cloning not permitted by storage backend -- Insufficient storage quota in target namespace -- Source PVC not in `Bound` state -- Storage class doesn't have volume cloning enabled -- CDI (Containerized Data Importer) controller issues - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume clone status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and `.metadata.name`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Check DataVolume events for errors**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe datavolume -n -``` - -**3. Check DataVolume phase**: - -Use `resources_get` from step 1, extract `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -``` - -**4. Check if storage class supports cloning**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass", - "name": "" -} -``` - -Review the full YAML output for cloning-related configurations. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -o yaml | grep -A 5 -i clone -``` - -**5. Check CSI driver capabilities**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "CSIDriver" -} -``` - -Review `.items[].metadata.name` for available CSI drivers. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get csidriver -``` - -**6. Check source PVC status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Bound` for cloning to work). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**7. Check target namespace storage quota**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**8. Check CDI controller logs**: - -**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) - -First, list pods in openshift-cnv namespace: - -**Parameters for pods_list_in_namespace**: -```json -{ - "namespace": "openshift-cnv", - "labelSelector": "app.kubernetes.io/component=cdi-deployment" -} -``` - -Then get logs using `pods_log`: - -**Parameters**: -```json -{ - "namespace": "openshift-cnv", - "name": "", - "tail": 100 -} -``` - -**CLI Fallback** (if MCP unavailable or easier via CLI): -```bash -oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') -``` - -**Common Error Messages**: -- `"volume cloning is not supported"` - CSI driver lacks clone capability -- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage -- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace -- `"insufficient quota"` - Target namespace lacks storage quota -- `"source PVC not bound"` - Source PVC must be in Bound state for cloning -- `"StorageClass does not support cloning"` - Storage class configuration issue - -**Solutions** (Use MCP Tools First): - -1. **Check storage class clone support**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" - } - ``` - - For each storage class in `.items[]`, check: - - `.metadata.name` (storage class name) - - `.provisioner` (CSI driver) - - Storage classes using CSI drivers typically support cloning. Look for provisioners like: - - `csi.ovirt.org` (oVirt CSI) - - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) - - `ebs.csi.aws.com` (AWS EBS CSI) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner - ``` - -2. **Verify source PVC is bound**: - - Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pvc -n -o jsonpath='{.status.phase}' - ``` - -3. **Check target namespace quota**: - - Use `resources_list` from diagnostic step 7 to check quota. - - If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - ``` - -4. **Use snapshot-based cloning** (alternative method): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Step 1**: Create VolumeSnapshot of source PVC - - **Parameters**: - ```json - { - "apiVersion": "snapshot.storage.k8s.io/v1", - "kind": "VolumeSnapshot", - "metadata": { - "name": "-snapshot", - "namespace": "" - }, - "spec": { - "source": { - "persistentVolumeClaimName": "" - } - } - } - ``` - - **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) - - **Step 3**: Create new DataVolume from snapshot - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "metadata": { - "name": "-rootdisk", - "namespace": "" - }, - "spec": { - "source": { - "snapshot": { - "name": "-snapshot", - "namespace": "" - } - }, - "storage": { - "resources": { - "requests": { - "storage": "50Gi" - } - }, - "storageClassName": "" - } - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" - ```bash - cat <-snapshot - namespace: - spec: - source: - persistentVolumeClaimName: - EOF - - oc get volumesnapshot -snapshot -n - - cat <-rootdisk - namespace: - spec: - source: - snapshot: - name: -snapshot - namespace: - storage: - resources: - requests: - storage: 50Gi - storageClassName: - EOF - ``` - -5. **Use "new empty storage" option** (vm-clone skill): - - If cloning isn't supported, create VM with empty storage - - Manually copy data if needed - -6. **Cross-namespace cloning workaround**: - - Some storage backends require snapshot for cross-namespace cloning - - Create snapshot in source namespace, restore in target namespace (see solution 4 above) - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Check DataVolume reached Succeeded phase: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should return `Succeeded`). - -Check PVC was created and bound: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "-rootdisk" -} -``` - -Check `.status.phase` (should return `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -# Should return: Succeeded - -oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' -# Should return: Bound -``` - -**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/skills/vm-snapshot-create/SKILL.md b/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/skills/vm-snapshot-create/SKILL.md deleted file mode 100644 index e651c6ef..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-create/environment/skills/vm-snapshot-create/SKILL.md +++ /dev/null @@ -1,423 +0,0 @@ ---- -name: vm-snapshot-create -description: | - Create virtual machine snapshots for backup and recovery. - - Use when: - - "Create a snapshot of VM [name]" - - "Backup VM [name] before upgrade" - - "Take a snapshot of [vm]" - - Validates storage class snapshot support, CSI driver capabilities, and guest agent status before snapshot creation. - - NOT for VM cloning (use vm-clone to create independent copies). - -model: inherit -color: green ---- - -# /vm-snapshot-create Skill - -Create virtual machine snapshots in OpenShift Virtualization. Snapshots capture the state and data of a VM at a specific point in time, enabling backup, recovery, and testing workflows. - -**Implementation Note**: This skill uses generic Kubernetes resource tools (`resources_create_or_update`) to manage VirtualMachineSnapshot resources. Dedicated snapshot tools do not currently exist in the openshift-virtualization MCP server. - -## Prerequisites - -**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools**: -- `resources_create_or_update` (from openshift-virtualization) - Create VirtualMachineSnapshot -- `resources_get` (from openshift-virtualization) - Verify VM exists and get status -- `resources_list` (from openshift-virtualization) - List StorageClass, VolumeSnapshotClass - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster (>= 4.19) -- OpenShift Virtualization operator installed -- ServiceAccount with RBAC permissions to create VirtualMachineSnapshot resources -- Storage backend with snapshot support (CSI driver with snapshot capabilities) - -## When to Use This Skill - -**Trigger this skill when:** -- User wants to create a backup of a VM before changes -- User wants to create a recovery point -- User explicitly requests snapshot creation - -**User phrases that trigger this skill:** -- "Create a snapshot of VM database-01" -- "Backup VM web-server before upgrade" -- "Take a snapshot of production-app" - -**Do NOT use this skill when:** -- User wants to clone a VM → Use `vm-clone` skill (creates independent copy) -- User wants to list snapshots → Use `vm-snapshot-list` skill -- User wants to restore from snapshot → Use `vm-snapshot-restore` skill - -## Workflow - -### Step 1: Gather Snapshot Information - -**Required Information from User:** -1. **VM Name** - Name of the VM to snapshot -2. **Namespace** - Namespace where VM exists -3. **Snapshot Name** (Optional) - Name for the snapshot (auto-generated if not provided) - -If namespace not provided, ask for it explicitly. - -### Step 2: Verify VM Exists and Get Status - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Expected Output**: VirtualMachine resource with status - -**Error Handling**: -- If VM not found → Report error, suggest using vm-inventory skill -- If permission denied → Report RBAC error - -**Extract VM Details:** -- Current status (Running, Stopped) -- Storage configuration (DataVolumes, PVCs) -- **IMPORTANT**: Save `status.volumeSnapshotStatuses` for storage analysis - -### Step 3: Verify Storage Snapshot Capabilities - -**CRITICAL: This comprehensive storage analysis MUST execute BEFORE asking user about VM running state.** - -This step analyzes storage backend capabilities to determine snapshot behavior and requirements. The analysis includes 9 substeps. - -[Continue with all 9 substeps from the original file: 1c.1 through 1c.9, checking volume snapshot status, hot-plugged volumes, storage class, VolumeSnapshotClass, CSI driver capabilities, guest agent status, Windows VSS, and storing analysis results] - -### Step 4: Check VM Running State (Enhanced with Storage Analysis) - -**From the VM resource in Step 2**, check `status.printableStatus`. - -**Use storage analysis results from Step 3** to provide accurate guidance. - -[Include the three scenarios: VM must be stopped, VM can run (online supported), VM is stopped - with all the guest agent and Windows VSS warnings] - -### Step 5: Stop Running VM (if user chose "stop-and-snapshot") - -**ONLY execute if user chose "stop-and-snapshot" in Step 4.** - -Use `vm_lifecycle` MCP tool or vm-lifecycle-manager skill to stop the VM. - -### Step 6: Estimate Storage Consumption - -**From the VM resource**, estimate snapshot storage: -- Initial snapshot may be same size as VM disk -- Subsequent snapshots smaller (only deltas) - -### Step 7: Present Snapshot Configuration for Confirmation - -**Include storage analysis results from Step 3 in the configuration presentation.** - -[Include the full confirmation template with storage backend analysis, guest agent status, volumes to snapshot, etc.] - -**Wait for user confirmation.** - -**Handle response:** -- If "yes" → Proceed to Step 8 (execute snapshot) -- If "no", "cancel", or anything else → Cancel operation - -### Step 8: Create the Snapshot - -**ONLY PROCEED AFTER user confirmation in Step 7.** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Construct VirtualMachineSnapshot YAML:** - -```yaml -apiVersion: snapshot.kubevirt.io/v1beta1 -kind: VirtualMachineSnapshot -metadata: - name: - namespace: -spec: - source: - apiGroup: kubevirt.io - kind: VirtualMachine - name: -``` - -**If snapshot name not provided by user**, generate one: -- Format: `-snapshot-` -- Example: `database-01-snapshot-20260218-143022` - -**Parameters**: -```json -{ - "resource": "apiVersion: snapshot.kubevirt.io/v1beta1\nkind: VirtualMachineSnapshot\nmetadata:\n name: \n namespace: \nspec:\n source:\n apiGroup: kubevirt.io\n kind: VirtualMachine\n name: " -} -``` - -**Report progress:** -```markdown -📸 Creating VM snapshot... -✓ Snapshot `` created for VM `` -``` - -### Step 9: Monitor Snapshot Status - -**After creation, monitor snapshot readiness using `resources_get`.** - -Check `status.phase`: -- `InProgress` → Still creating -- `Succeeded` → Snapshot ready -- `Failed` → Snapshot failed - -**Wait up to 5 minutes for snapshot to complete.** - -### Step 10: Report Snapshot Creation Results - -**Extract snapshot indications** from `status.indications`: -- `GuestAgent` - Guest agent coordinated the snapshot -- `Online` - Snapshot taken while VM was running - -**On success:** - -```markdown -## ✓ VM Snapshot Created Successfully - -**VM**: `` (namespace: ``) -**Snapshot**: `` - -### Snapshot Details -- **Name**: `` -- **Status**: Ready -- **Created**: -- **VM Status at Snapshot**: - -### Snapshot Coordination (from status.indications) - -- ✅ **Guest Agent Coordination**: Active -- ✅ **Filesystem Freeze/Thaw**: Performed during snapshot -- ✅ **Actual Consistency**: Application-consistent - - - -- ⚠️ **Guest Agent Coordination**: Not active -- ⚠️ **Actual Consistency**: Crash-consistent (best-effort) - - -### Next Steps - -**To list all snapshots:** -"List snapshots for VM " - -**To restore from this snapshot:** -"Restore VM from snapshot " - -**To delete this snapshot:** -"Delete snapshot " -``` - -## Dependencies - -### Required MCP Servers -- `openshift-virtualization` - OpenShift MCP server with kubevirt toolset - -### Required MCP Tools -- `resources_create_or_update` (from openshift-virtualization) - Create VirtualMachineSnapshot -- `resources_get` (from openshift-virtualization) - Verify VM and snapshot status -- `resources_list` (from openshift-virtualization) - List StorageClass, VolumeSnapshotClass - -### Related Skills -- `vm-snapshot-list` - List snapshots after creation -- `vm-snapshot-restore` - Restore VMs from snapshots -- `vm-snapshot-delete` - Delete old snapshots -- `vm-lifecycle-manager` - Stop VMs before snapshot -- `vm-inventory` - List VMs before creating snapshots - -### Reference Documentation - -**Official Red Hat Documentation:** -- [OpenShift Virtualization Snapshots - OpenShift 4.20](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt-managing-vm-snapshots) -- [Live Snapshots in OpenShift Virtualization](https://www.redhat.com/en/blog/live-snapshots-in-openshift-virtualization) - -**Upstream Documentation:** -- [KubeVirt VM Snapshots](https://kubevirt.io/user-guide/operations/snapshot_restore_api/) -- [CSI Volume Snapshots](https://kubernetes.io/docs/concepts/storage/volume-snapshots/) - -## Critical: Human-in-the-Loop Requirements - -**IMPORTANT:** This skill performs operations that affect VM data and storage. You MUST: - -1. **Before Creating Snapshots** - - Analyze storage backend capabilities - - Verify VM exists and get current state - - Check for hot-plugged volumes (blocks snapshots) - - Show storage consumption estimate - - Present snapshot configuration - - Ask: "Proceed with snapshot creation? (yes/no)" - - Wait for explicit "yes" - -2. **Never Auto-Execute** - - **NEVER create without user confirmation** - - **NEVER skip storage analysis** - - **NEVER skip hot-plugged volume check** - -**Why This Matters:** -- **Storage Consumption**: Snapshots consume storage proportional to VM disk size -- **Hot-Plugged Volumes**: Cannot snapshot VMs with hot-plugged volumes -- **Consistency**: Online vs offline snapshots have different consistency guarantees -- **Guest Agent**: Required for application-consistent snapshots - -## Common Issues - -### Issue 1: Snapshot Creation Fails - VolumeSnapshotClass Not Found - -**Error**: "VolumeSnapshotClass not found" or "CSI driver doesn't support snapshots" - -**Cause**: The storage backend doesn't have a VolumeSnapshotClass configured for the CSI driver, or the CSI driver doesn't support snapshots at all. - -**Solution:** -1. **Check if VolumeSnapshotClass exists**: Use `resources_list` with apiVersion="snapshot.storage.k8s.io/v1", kind="VolumeSnapshotClass" -2. **Verify CSI driver supports snapshots**: Check StorageClass provisioner field -3. **Contact cluster admin**: Request VolumeSnapshotClass configuration for your storage backend -4. **Alternative**: Use `vm-clone` skill for VM backup instead of snapshots - -### Issue 2: Snapshot Creation Blocked - Hot-Plugged Volumes Detected - -**Error**: "Cannot create snapshot - VM has hot-plugged volumes" - -**Cause**: The VM has volumes that were attached after VM creation without restarting the VM. Hot-plugged volumes block snapshot creation in OpenShift Virtualization. - -**Solution:** -1. **Stop the VM**: Use vm-lifecycle-manager skill to stop the VM -2. **Remove hot-plugged volumes**: Detach volumes that aren't needed -3. **Persist volumes to VM spec**: Add hot-plugged volumes to `spec.template.spec.volumes` to make them permanent -4. **Restart the VM**: Start the VM to apply the changes -5. **Retry snapshot**: Once hot-plugged volumes are resolved, create the snapshot - -**Related**: See [OpenShift Virtualization documentation](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt-hot-plugging-virtual-disks) for hot-plugging details - -### Issue 3: Snapshot Created but Consistency Warning - -**Error**: Snapshot created successfully but shows "crash-consistent" without guest agent - -**Cause**: The VM doesn't have QEMU guest agent installed or running, so the snapshot couldn't coordinate filesystem freeze/thaw during creation. - -**Solution:** -1. **For Linux VMs**: Install qemu-guest-agent package - ```bash - # RHEL/CentOS/Fedora - sudo dnf install qemu-guest-agent - sudo systemctl enable --now qemu-guest-agent - ``` -2. **For Windows VMs**: Install VirtIO drivers which include the guest agent -3. **Verify agent status**: Check VM status for `AgentConnected: True` condition -4. **Future snapshots**: Once guest agent is running, subsequent snapshots will be application-consistent -5. **Current snapshot**: The crash-consistent snapshot is still usable, but may have minor inconsistencies - -## Security Considerations - -- **RBAC Enforcement**: Requires permissions for VirtualMachineSnapshot resources -- **Storage Quotas**: Respects namespace storage quotas -- **Hot-Plugged Volume Detection**: Prevents snapshots when hot-plugged volumes present -- **KUBECONFIG Security**: Credentials never exposed in output -- **Namespace Isolation**: Snapshots scoped to namespace boundaries -- **Audit Trail**: All snapshot operations logged in Kubernetes API audit logs - -## Example Usage - -### Example 1: Create Snapshot Before Upgrade - -``` -User: "Create a snapshot of database-01 before I upgrade it" - -Agent: "Which namespace is database-01 in?" - -User: "production" - -Agent: [Invokes vm-snapshot-create skill] - [Step 2: Verifies VM exists] - [Step 3: Analyzes storage backend - AWS EBS, online snapshots supported] - [Step 4: Checks VM status - Running, guest agent active] - [Step 6: Estimates storage - 50Gi] - [Step 7: Presents configuration] - -## VM Snapshot - Review Configuration - -**VM to Snapshot:** -- **Name**: `database-01` -- **Namespace**: `production` -- **Status**: Running -- **Storage**: 50Gi - -**Storage Backend Analysis:** -- **Storage Class**: `gp3-csi` -- **CSI Driver**: `ebs.csi.aws.com` -- **Online Snapshots**: ✅ Supported -- **Consistency Level**: `crash-consistent` -- **VolumeSnapshotClass**: `ebs-snapclass` ✓ - -**Guest Agent Status:** -- ✅ **QEMU Guest Agent**: Connected and running -- ✅ **Filesystem Freeze/Thaw**: Will be performed during snapshot -- ✅ **Expected Consistency**: Application-consistent (with freeze) - -**Snapshot Details:** -- **Snapshot Name**: `database-01-snapshot-20260218-143022` (auto-generated) -- **Type**: Online snapshot -- **Estimated Storage**: ~50Gi (initial snapshot) - -**Proceed with snapshot creation? (yes/no)** - -User: "yes" - -Agent: [Step 8: Creates snapshot] - -📸 Creating VM snapshot... -✓ Snapshot `database-01-snapshot-20260218-143022` created for VM `database-01` - - [Step 9: Monitors status] - -⏳ Snapshot creation in progress... - Status: InProgress - -⏳ Snapshot creation in progress... - Status: Succeeded - - [Step 10: Reports results] - -## ✓ VM Snapshot Created Successfully - -**VM**: `database-01` (namespace: `production`) -**Snapshot**: `database-01-snapshot-20260218-143022` - -### Snapshot Details -- **Name**: `database-01-snapshot-20260218-143022` -- **Status**: Ready -- **Created**: 2026-02-18 14:30:22 UTC -- **VM Status at Snapshot**: Running - -### Snapshot Coordination (from status.indications) -- ✅ **Guest Agent Coordination**: Active -- ✅ **Filesystem Freeze/Thaw**: Performed during snapshot -- ✅ **Actual Consistency**: Application-consistent - -### Next Steps - -**To list all snapshots:** -"List snapshots for VM database-01" - -**To restore from this snapshot:** -"Restore VM database-01 from snapshot database-01-snapshot-20260218-143022" - -You can now safely upgrade the database. If the upgrade fails, restore using the command above. -``` diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/.ai-index/semantic-index.json deleted file mode 100644 index 2512c2a3..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/.ai-index/semantic-index.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "index_version": "1.0.0", - "last_updated": "2026-02-17", - "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", - - "documents": [ - { - "file": "INDEX.md", - "title": "VM Troubleshooting Guide - Index", - "category": "navigation", - "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], - "use_cases": ["navigation", "tool-reference", "general-guidance"], - "token_estimate": 450, - "skills": ["all"] - }, - { - "file": "scheduling-errors.md", - "title": "VM Scheduling Errors", - "category": "scheduling", - "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], - "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], - "errors_covered": [ - "ErrorUnschedulable - Node Taints", - "ErrorUnschedulable - Insufficient Resources", - "ErrorUnschedulable - Node Selector Mismatch" - ], - "token_estimate": 820, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "storage-errors.md", - "title": "VM Storage Errors", - "category": "storage", - "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], - "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], - "errors_covered": [ - "ErrorDataVolumeNotReady - DataVolume Still Provisioning", - "ErrorDataVolumeNotReady - Storage Class Not Found", - "ErrorDataVolumeNotReady - Insufficient Storage Quota", - "ErrorPvcNotFound", - "Storage Deletion Failures", - "DataVolume Cloning Failures" - ], - "token_estimate": 1350, - "skills": ["vm-creator", "vm-delete", "vm-clone"] - }, - { - "file": "lifecycle-errors.md", - "title": "VM Lifecycle Errors", - "category": "lifecycle", - "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], - "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], - "errors_covered": [ - "VM Stuck in Terminating State", - "VM Won't Start (Non-Scheduling Issues)", - "VM Won't Stop" - ], - "token_estimate": 1150, - "skills": ["vm-delete", "vm-lifecycle-manager"] - }, - { - "file": "runtime-errors.md", - "title": "VM Runtime Errors", - "category": "runtime", - "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], - "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], - "errors_covered": [ - "CrashLoopBackOff" - ], - "token_estimate": 850, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "network-errors.md", - "title": "VM Network Errors", - "category": "networking", - "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], - "use_cases": ["vm-creation", "network-troubleshooting"], - "errors_covered": [ - "Network Attachment Failures" - ], - "token_estimate": 580, - "skills": ["vm-creator"] - } - ], - - "skill_to_docs_mapping": { - "vm-creator": { - "primary_docs": ["scheduling-errors.md", "storage-errors.md"], - "secondary_docs": ["runtime-errors.md", "network-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3650, - "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" - }, - "vm-delete": { - "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 2950, - "description": "VM deletion failures - stuck terminating, storage cleanup" - }, - "vm-clone": { - "primary_docs": ["storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 1800, - "description": "VM cloning failures - DataVolume cloning, storage provisioning" - }, - "vm-lifecycle-manager": { - "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], - "secondary_docs": ["runtime-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3270, - "description": "VM start/stop failures - lifecycle management, scheduling" - }, - "vm-inventory": { - "primary_docs": [], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 450, - "description": "General guidance for status interpretation" - } - }, - - "error_to_docs_mapping": { - "ErrorUnschedulable": "scheduling-errors.md", - "ErrorDataVolumeNotReady": "storage-errors.md", - "ErrorPvcNotFound": "storage-errors.md", - "Terminating": "lifecycle-errors.md", - "CrashLoopBackOff": "runtime-errors.md", - "NetworkAttachmentFailed": "network-errors.md" - }, - - "cross_references": { - "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], - "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], - "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], - "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], - "network-errors.md": ["scheduling-errors.md"] - }, - - "progressive_disclosure_strategy": { - "description": "Load only the minimum documentation needed for current task", - "pattern": "error_type → primary_doc → cross_references (if needed)", - "token_savings": "29% average reduction vs single-file (based on rh-sre data)", - "navigation_overhead_reduction": "85% vs single-file navigation" - } -} diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/INDEX.md deleted file mode 100644 index a2e76963..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/INDEX.md +++ /dev/null @@ -1,332 +0,0 @@ ---- -title: VM Troubleshooting Guide - Index -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 - - title: OpenShift Virtualization - Virtual Machine Status - url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html - date_accessed: 2026-02-06 - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] -semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] -use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] -last_updated: 2026-02-17 ---- - -# VM Troubleshooting Guide - Index - -## Overview - -This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: - -- **Creation**: Scheduling failures, storage provisioning issues -- **Lifecycle**: Start/stop failures, stuck states -- **Deletion**: Resources stuck in Terminating, storage cleanup failures -- **Cloning**: DataVolume cloning errors, cross-namespace issues -- **Networking**: Secondary network attachment failures -- **Runtime**: CrashLoopBackOff, guest OS boot failures - -This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. - ---- - -## 🗂️ Troubleshooting Categories - -The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: - -### 1. [Scheduling Errors](scheduling-errors.md) -**When to use**: VM fails to schedule on any node - -**Errors covered**: -- ErrorUnschedulable - Node Taints -- ErrorUnschedulable - Insufficient Resources -- ErrorUnschedulable - Node Selector Mismatch - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 2. [Storage Errors](storage-errors.md) -**When to use**: VM has storage provisioning, deletion, or cloning issues - -**Errors covered**: -- ErrorDataVolumeNotReady (all 3 subsections) -- ErrorPvcNotFound -- Storage Deletion Failures -- DataVolume Cloning Failures - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### 3. [Lifecycle Errors](lifecycle-errors.md) -**When to use**: VM has start/stop/termination issues - -**Errors covered**: -- VM Stuck in Terminating State -- VM Won't Start (Non-Scheduling Issues) -- VM Won't Stop - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### 4. [Runtime Errors](runtime-errors.md) -**When to use**: VM repeatedly crashes or fails at runtime - -**Errors covered**: -- CrashLoopBackOff - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 5. [Network Errors](network-errors.md) -**When to use**: VM has secondary network attachment failures - -**Errors covered**: -- Network Attachment Failures (Multus, SR-IOV) - -**Skills that use this**: vm-create - ---- - -## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last - -### Critical Principle: MCP-First Approach - -**All diagnostic procedures in this guide follow the MCP-first pattern**: - -``` -1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) -2. ⚠️ IF FAILS: Ask user permission to use CLI command -3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval -``` - -### Why MCP Tools First? - -- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources -- **Consistency**: Same interface across all operations -- **Better Error Handling**: MCP tools return structured errors -- **Audit Trail**: MCP tool usage is logged and trackable - -### Command Pattern Examples - -Throughout this guide, you'll see diagnostic steps formatted like this: - -**✅ CORRECT Pattern (MCP First)**: - -```markdown -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Extract**: `.metadata.finalizers` from returned JSON - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` -``` - -### Available MCP Tools - -The `openshift-virtualization` MCP server provides these tools: - -**Resource Operations**: -- `resources_get` - Get specific resource (replaces `oc get `) -- `resources_list` - List resources (replaces `oc get `) -- `resources_delete` - Delete resource (replaces `oc delete`) -- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) - -**Pod Operations**: -- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) -- `pods_get` - Get pod details (replaces `oc get pod`) -- `pods_log` - Get pod logs (replaces `oc logs`) -- `pods_exec` - Execute in pod (replaces `oc exec`) -- `pods_delete` - Delete pod (replaces `oc delete pod`) -- `pods_top` - Pod resource usage (replaces `oc top pods`) - -**Events & Monitoring**: -- `events_list` - List events (replaces `oc get events`) -- `nodes_top` - Node resource usage (replaces `oc top nodes`) -- `nodes_log` - Node logs (replaces `oc adm node-logs`) -- `nodes_stats_summary` - Detailed node stats - -**VM Operations** (KubeVirt toolset): -- `vm_create` - Create VMs -- `vm_lifecycle` - Start/stop/restart VMs - -**Namespaces**: -- `namespaces_list` - List namespaces (replaces `oc get namespaces`) -- `projects_list` - List OpenShift projects (replaces `oc get projects`) - -### When CLI Commands Are Required - -Some operations have **NO MCP equivalent** and require CLI: - -- `virtctl` commands (console, VNC access) -- `oc debug node` (node debugging) -- `oc auth can-i` (permission checks) -- `oc adm taint` (node taint management) -- Complex JSON patch operations - -For these, the guide will note: **"CLI Only - No MCP equivalent"** - -### Quick Reference: CLI → MCP Mapping - -| CLI Command | MCP Tool Equivalent | -|-------------|---------------------| -| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | -| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | -| `oc logs -n ` | `pods_log` with name="", namespace="" | -| `oc get events -n ` | `events_list` with namespace="" | -| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | - -**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. - -### How to Read Diagnostic Sections - -Each error section includes: -1. **Symptom** - What you observe -2. **Description** - What's happening -3. **Possible Causes** - Why it's happening -4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback -5. **Solutions** - **MCP tools first**, then CLI fallback -6. **Verification** - **MCP tools first**, then CLI fallback - -**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. - ---- - -## 🔍 Quick Navigation by Skill - -**vm-create**: -- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics -- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning -- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff -- [Network Errors](network-errors.md) - Network attachment failures - -**vm-delete**: -- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state -- [Storage Errors](storage-errors.md) - Storage deletion failures - -**vm-clone**: -- [Storage Errors](storage-errors.md) - DataVolume cloning failures - -**vm-lifecycle-manager**: -- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop -- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling - -**vm-inventory**: -- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed - ---- - -## 📊 Documentation Coverage & Maintenance - -### Current Coverage - -This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: - -- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) -- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures -- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop -- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) -- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems - -**Total errors documented**: 12 error types across 6 categories - ---- - -### Encountering Undocumented Errors - -#### For AI Agents (Claude Code) - -If you encounter an error **not documented** in the categories above: - -1. **Report to user** with all available details (error message, affected resources, namespace) -2. **Provide best-effort diagnostics** using MCP tools: - - `resources_get` to inspect resource status - - `pods_log` to check virt-launcher or compute container logs - - `events_list` to view Kubernetes events -3. **Suggest documentation update**: - ``` - ⚠️ This error is not yet documented in the troubleshooting guide. - - **Error**: - **Affected resource**: / - - I recommend adding this error to the troubleshooting guide: - - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file - - If it's a new error class → Create new category file in docs/troubleshooting/ - - Would you like me to help document this error for future reference? - ``` - -4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs - ---- - -#### For Human Contributors - -To document a new error: - -1. **Determine the category**: - - Scheduling issue → `scheduling-errors.md` - - Storage problem → `storage-errors.md` - - Lifecycle/start/stop → `lifecycle-errors.md` - - Runtime crash → `runtime-errors.md` - - Network issue → `network-errors.md` - - New error class → Create new category file (e.g., `security-errors.md`) - -2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): - - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback - - **Solutions**: MCP Tool approach → Implementation → CLI Fallback - - **Verification**: MCP Tool checks → Expected results → CLI Fallback - -3. **Update semantic index**: - - Edit `.ai-index/semantic-index.json` - - Add error to `error_to_docs_mapping` - - Update relevant skill's `primary_docs` or `secondary_docs` - - Increment token estimates if needed - -4. **Reference in skill files**: - - Update skill's Reference Documentation section - - Ensure skill's Document Consultation steps point to the right category - -5. **Update this INDEX.md**: - - Add error to the appropriate category section (lines 50-120) - - Update "Current Coverage" list above - ---- - -## 📚 Additional Resources - -- [rh-virt README](../../README.md) - MCP server setup and skill documentation -- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources -- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/lifecycle-errors.md deleted file mode 100644 index 6167fcc8..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/lifecycle-errors.md +++ /dev/null @@ -1,869 +0,0 @@ ---- -title: VM Lifecycle Errors -category: kubevirt -sources: - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 -tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] -semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] -use_cases: [vm-deletion, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Lifecycle Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM lifecycle issues including start/stop failures and stuck termination states. - -**When to use this document**: -- VM stuck in `Terminating` state -- VM won't start (runStrategy is Always but VM never reaches Running) -- VM won't stop (runStrategy is Halted but VM never reaches Stopped) - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### VM Stuck in Terminating State - -**Symptom**: VM shows status `Terminating` but deletion never completes - -**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). - -**Possible Causes**: -- Finalizers blocking deletion -- PVC/DataVolume still attached and preventing cleanup -- VirtualMachineInstance (VMI) not terminating properly -- Custom controllers or operators blocking deletion -- Stuck virt-launcher pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check finalizers on the VM**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` - -**2. Check if VMI still exists**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name containing "virt-launcher-". - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check events for deletion issues**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results where `involvedObject.name` == "" and sort by timestamp. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' -``` - -**5. Check VM deletion timestamp**: - -Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. - -If present, VM is in deletion process. If null, VM is not being deleted. - -**Common Finalizer Patterns**: -- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer -- `foregroundDeletion` - Waits for dependent objects to be deleted -- Custom finalizers from operators - -**Solutions** (Use MCP Tools First): - -1. **Wait for dependent resources** (safest approach): - - Use `resources_list` to check if PVCs, DataVolumes are still deleting - - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) - -2. **Force delete VMI** (if VMI is stuck): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable - requires explicit user permission): - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -3. **Force delete virt-launcher pod** (if pod is stuck): - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - # Find the virt-launcher pod name first - oc get pods -n | grep virt-launcher- - - # Then delete it - oc delete pod virt-launcher--xxx -n --force --grace-period=0 - ``` - -4. **Remove finalizers** (⚠️ dangerous - use only as last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` - 2. Remove items from `.metadata.finalizers` array - 3. Update VM using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch not easily done via MCP): - ```bash - # This operation is complex for MCP - may need CLI - oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Check for protection labels** (vm-delete skill specific): - - Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. - - If value is "true", the vm-delete skill refuses deletion (this is expected behavior). - -**Verification**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error if deletion successful. - -**CLI Fallback**: -```bash -oc get vm -n -# Should return: Error from server (NotFound) -``` - ---- - - ---- - -### VM Won't Start (Non-Scheduling Issues) - -**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state - -**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. - -**Possible Causes**: -- Guest OS kernel panic or boot failure -- Cloud-init configuration errors -- Missing or corrupted disk image -- Insufficient memory for guest OS to boot -- QEMU/KVM virtualization errors -- VirtualMachineInstance (VMI) creation failures -- virt-launcher pod crashes - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VMI (VirtualMachineInstance) status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and overall VMI state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**2. Check VMI conditions for errors**: - -Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. View virt-launcher pod logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -First, get pod name from step 3, then: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -Look for QEMU errors, memory allocation failures, device errors. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**5. Check virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. - -Alternatively, use `pods_get` to get full pod details: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**6. Access VM console to see guest OS boot messages**: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -Look for kernel panic, initramfs errors, filesystem errors. - -**7. Check VM events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep -A 20 "Events:" -``` - -**Common Error Patterns**: - -1. **Guest OS Boot Failure**: - - Console shows kernel panic - - Guest hangs at GRUB or boot loader - - Cloud-init errors during first boot - -2. **Insufficient Memory**: - - Guest OS kills processes due to OOM - - VMI logs show memory allocation errors - -3. **QEMU Crashes**: - - virt-launcher logs show QEMU segfaults - - VMI repeatedly restarts - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for kernel panic, initramfs errors, filesystem errors. - -2. **Check virt-launcher pod logs for QEMU errors**: - - Use `pods_log` from diagnostic step 4 to view logs. - - Look for: - - "qemu-system-x86_64: ..." errors - - Memory allocation failures - - Device errors - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx - ``` - -3. **Increase memory if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check VMI memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -4. **Verify disk image integrity**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check PVC is bound: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - If using container disk, verify image exists and is accessible (check VMI spec). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get pvc -n - ``` - -5. **Check cloud-init configuration** (if applicable): - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. - - Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq - ``` - -6. **Restart VMI** (soft reset): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete VMI (VM controller will recreate it): - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for new VMI to start (use `resources_get` to check status). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n -w - ``` - -7. **Check virtualization extensions** (KVM): - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - - # In debug shell: - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should eventually return `Running`). - -Check VMI is running: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should show `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should eventually return: Running - -oc get vmi -n -# Should show: Running -``` - ---- - -### VM Won't Stop - -**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state - -**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. - -**Possible Causes**: -- Guest OS not responding to ACPI shutdown signal -- virt-launcher pod stuck and not terminating -- VirtualMachineInstance (VMI) deletion blocked by finalizers -- Guest shutdown scripts hanging -- Filesystem sync issues in guest OS - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (might show `Stopping`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**2. Check VMI status and deletion timestamp**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check VMI events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vmi -n | grep -A 10 "Events:" -``` - -**5. Check VMI finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.metadata.finalizers}' -``` - -**6. Check if guest is responsive** (if VMI still exists): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Patterns**: -- VMI shows `deletionTimestamp` but never actually deletes -- virt-launcher pod in `Terminating` state -- VM runStrategy is `Halted` but printableStatus shows `Stopping` - -**Solutions** (Use MCP Tools First): - -1. **Wait for graceful shutdown** (default: 30 seconds): - - Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. - -2. **Force stop by deleting VMI**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - This is the standard way to force-stop a VM. - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n - # Should return: Error from server (NotFound) - ``` - -3. **Force delete VMI with grace period** (if VMI won't delete): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -4. **Force delete virt-launcher pod**: - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find the pod using `pods_list_in_namespace` from diagnostic step 3. - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - ⚠️ **Note**: For force deletion with grace period, use CLI fallback. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') - oc delete pod $POD_NAME -n --force --grace-period=0 - ``` - -5. **Remove VMI finalizers** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VMI using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update VMI using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Patch VM runStrategy directly** (ensure consistency): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` (diagnostic step 1) - 2. Set `.spec.runStrategy` to `"Halted"` - 3. Update VM using `resources_create_or_update` with modified JSON - - **CLI Fallback** (merge patch easier via CLI): - Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Stopped` or `Halted`). - -Verify VMI is gone: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Verify virt-launcher pod is gone: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for pods matching `virt-launcher-`. Should return no results. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Stopped or Halted - -oc get vmi -n -# Should return: Error from server (NotFound) - -oc get pods -n | grep virt-launcher- -# Should return: No resources found -``` - -**Prevention**: -- Ensure guest OS has ACPI support enabled -- Use proper shutdown commands in guest OS -- Avoid forceful stops unless necessary (can corrupt guest filesystem) - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/network-errors.md deleted file mode 100644 index 82fa313c..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/network-errors.md +++ /dev/null @@ -1,429 +0,0 @@ ---- -title: VM Network Errors -category: kubevirt -sources: - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] -semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] -use_cases: [vm-creation, network-troubleshooting] -related_docs: [INDEX.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Network Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. - -**When to use this document**: -- VM created successfully but secondary networks not attached -- NetworkAttachmentDefinition not found errors -- Multus CNI failures -- SR-IOV device attachment issues - -**Skills that use this**: vm-create - ---- - -### Network Attachment Failures - -**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working - -**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). - -**Possible Causes**: -- NetworkAttachmentDefinition doesn't exist in the namespace -- Multus CNI not installed or not configured on cluster -- Namespace mismatch (NAD in different namespace than VM) -- Interface name conflicts in VM spec -- Bridge/network configuration errors in NAD -- SR-IOV device not available (if using SR-IOV) - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List NetworkAttachmentDefinitions in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "" -} -``` - -Review `.items[].metadata.name` for available NADs. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -n -``` - -**2. List NetworkAttachmentDefinitions in all namespaces**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition" -} -``` - -Omit `namespace` parameter to list across all namespaces. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -A -``` - -**3. Check specific NetworkAttachmentDefinition**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "", - "name": "" -} -``` - -Review `.spec.config` for CNI configuration. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definition -n -o yaml -``` - -**4. Check VM network configuration**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.networks` to see network references. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq -``` - -**5. Check VM domain interfaces**: - -Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq -``` - -**6. Check virt-launcher pod network annotations** (shows actual attachments): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**7. Check Multus is installed**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "openshift-multus" -} -``` - -Should show Multus CNI pods running. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n openshift-multus -``` - -**8. Check for errors in virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n -``` - -**Common Error Messages**: -- `"network-attachment-definition not found"` - NAD doesn't exist in namespace -- `"multus CNI not configured"` - Multus not installed or misconfigured -- `"interface name conflict"` - Duplicate interface names in VM spec -- `"failed to add network"` - CNI plugin error (check NAD config) -- `"no available devices"` - SR-IOV device not available (if using SR-IOV) - -**Solutions** (Use MCP Tools First): - -1. **Verify NetworkAttachmentDefinition exists in correct namespace**: - - Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. - - If NAD is in different namespace, copy it to VM namespace: - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get NAD from source namespace using `resources_get` - 2. Modify `.metadata.namespace` to target namespace - 3. Create NAD in target namespace using `resources_create_or_update` - - **CLI Fallback** (stream processing easier via CLI): - Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" - ```bash - oc get network-attachment-definition -n -o yaml | \ - sed "s/namespace: /namespace: /" | \ - oc apply -f - - ``` - -2. **Create missing NetworkAttachmentDefinition**: - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - Example: Linux bridge network - - **Parameters**: - ```json - { - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "metadata": { - "name": "vlan100", - "namespace": "" - }, - "spec": { - "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" - ```bash - cat < - spec: - config: '{ - "cniVersion": "0.3.1", - "type": "bridge", - "bridge": "br1", - "vlan": 100, - "ipam": { - "type": "host-local", - "subnet": "192.168.100.0/24" - } - }' - EOF - ``` - -3. **Check Multus CNI installation**: - - Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. - - To check cluster network operator: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "config.openshift.io/v1", - "kind": "ClusterOperator" - } - ``` - - Filter for `network` operator. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n openshift-multus - oc get clusteroperators network - ``` - -4. **Fix interface name conflicts** (if VM has duplicate names): - - Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. - - Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. - - **CLI Fallback** (interactive edit easier via CLI): - Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' - oc edit vm -n - ``` - -5. **Validate NAD configuration syntax**: - - Use `resources_get` from diagnostic step 3, extract `.spec.config`. - - Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get network-attachment-definition -n -o jsonpath='{.spec.config}' - ``` - -6. **Check SR-IOV device availability** (if using SR-IOV networks): - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - List SR-IOV network node policies: - - **Parameters**: - ```json - { - "apiVersion": "sriovnetwork.openshift.io/v1", - "kind": "SriovNetworkNodePolicy", - "namespace": "openshift-sriov-network-operator" - } - ``` - - Check SR-IOV device plugin pods: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "openshift-sriov-network-operator" - } - ``` - - Filter for pods with "device-plugin" in name. - - Check available SR-IOV devices on node: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "Node", - "name": "" - } - ``` - - Review `.status.allocatable` for SR-IOV resources. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get sriovnetworknodepolicy -n openshift-sriov-network-operator - oc get pods -n openshift-sriov-network-operator | grep device-plugin - oc describe node | grep -A 10 "Allocatable:" - ``` - -7. **Recreate VM with corrected network configuration** (if needed): - - If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -After remediation, check virt-launcher pod network status: - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -Should show all attached networks with status. Example output: -```json -[ - { - "name": "openshift-sdn", - "interface": "eth0", - "ips": ["10.128.2.10"], - "default": true - }, - { - "name": "vlan100", - "interface": "net1", - "ips": ["192.168.100.5"] - } -] -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**Check from inside VM** (via console): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# In guest OS: -ip addr show -# Should show all network interfaces (eth0, net1, etc.) -``` - -**Common Network Types**: -- **Linux Bridge**: Layer 2 bridge for VLAN networks -- **SR-IOV**: High-performance direct device assignment -- **macvlan**: MAC-based VLAN for container networks -- **OVN-Kubernetes**: OpenShift native overlay network - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/runtime-errors.md deleted file mode 100644 index 804d3cbc..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/runtime-errors.md +++ /dev/null @@ -1,616 +0,0 @@ ---- -title: VM Runtime Errors -category: kubevirt -sources: - - title: Kubernetes Pod Lifecycle - url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ - date_accessed: 2026-02-17 -tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] -semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] -use_cases: [vm-creation, vm-lifecycle, diagnostics] -related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Runtime Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. - -**When to use this document**: -- VM or virt-launcher pod shows `CrashLoopBackOff` status -- virt-launcher pod repeatedly restarting -- Guest OS kernel panics on boot - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### CrashLoopBackOff - -**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting - -**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. - -**Possible Causes**: -- Guest OS kernel panic on boot -- Insufficient resources (memory/CPU) for guest OS -- Corrupted disk image or filesystem -- QEMU/libvirt crashes due to configuration errors -- Missing or incompatible device drivers in guest -- Resource limits too low for virt-launcher pod -- Virtualization features (KVM) not available on node - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check virt-launcher pod restart count**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# Look at RESTARTS column -``` - -**2. View recent crash logs** (previous container instance): - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "previous": true, - "tail": 100 -} -``` - -Look for QEMU errors, kernel panics, or segfaults. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n virt-launcher--xxx --previous -``` - -**3. Check current virt-launcher logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. - -**CLI Fallback** (if MCP unavailable or all containers needed): -```bash -oc logs -n virt-launcher--xxx --all-containers -``` - -**4. Check VMI conditions for crash details**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Extract `.status.conditions` for crash details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**5. Check pod events for crash reasons**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" -``` - -**6. Check pod resource limits**: - -**MCP Tool**: `pods_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -Extract `.spec.containers[0].resources` for resource limits. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq -``` - -**7. Check node kubelet logs for OOM kills**: - -⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -oc adm node-logs -u kubelet | grep -i oom -``` - -**8. Access guest console** (if VM briefly starts): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Crash Patterns**: - -1. **Guest Kernel Panic**: - - Console logs show kernel panic messages - - Guest crashes immediately after boot - - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" - -2. **OOM (Out of Memory)**: - - Pod killed with reason: `OOMKilled` - - Guest runs out of memory during boot or operation - - virt-launcher logs show memory allocation failures - -3. **QEMU Crash**: - - virt-launcher logs show QEMU segmentation fault - - Symptoms: "qemu-system-x86_64: terminated by signal" - - Configuration incompatibility or QEMU bug - -4. **Disk Image Corruption**: - - Guest cannot boot from disk - - Filesystem errors in guest console - - DataVolume import failed - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for kernel panic or boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for: - - Kernel panic messages - - Initramfs errors - - Filesystem mounting failures - - Missing device errors - -2. **Review virt-launcher crash logs**: - - Use `pods_log` with `previous: true` from diagnostic step 2. - - Look for: - - QEMU command line errors - - Device initialization failures - - Memory allocation errors - - Signal termination (SIGSEGV, SIGABRT) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx --previous - ``` - -3. **Check for OOM (Out of Memory) kills**: - - **MCP Tool**: `pods_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - Extract `.status.containerStatuses[0].lastState.terminated.reason`. - - If returns `"OOMKilled"`: - - Option 1: Increase virt-launcher memory limits - - Option 2: Decrease guest memory allocation - - Option 3: Use smaller instance type - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' - ``` - -4. **Increase resources if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check current memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -5. **Verify disk image integrity**: - - **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters for list**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" - } - ``` - - **Parameters for specific DV**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check `.status.phase` (should be `Succeeded`). - - If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get datavolume -n -o jsonpath='{.status.phase}' - ``` - -6. **Check virtualization (KVM) availability**: - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -7. **Simplify VM configuration** (eliminate variables): - - Try creating minimal VM using vm-create skill with: - - Small instance type - - No secondary networks - - Simple container disk (e.g., Fedora) - - No cloud-init - - If minimal VM works, add features back one by one. - -8. **Recreate VM with different workload** (test disk image): - - If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) - -After remediation, check pod restart count stops increasing: - -**Parameters for pods**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). - -Check VM reaches Running state: - -**Parameters for VM**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# RESTARTS should stabilize - -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Running -``` - -Verify guest is responsive: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# Should show login prompt or OS console -``` - -**Advanced Debugging**: - -**MCP Tool**: `pods_exec` (from openshift-virtualization) - -Check libvirt domain XML: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["virsh", "dumpxml", "1"] -} -``` - -Check QEMU process: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["ps", "aux"] -} -``` - -Filter output for "qemu" process. - -**CLI Fallback** (if MCP unavailable): -```bash -oc exec -n virt-launcher--xxx -- virsh dumpxml 1 -oc exec -n virt-launcher--xxx -- ps aux | grep qemu -``` - -**Prevention**: -- Start with minimal VM configuration and add complexity gradually -- Use recommended instance types for your workload -- Test disk images before deploying to production -- Ensure nodes have adequate resources and KVM support -- Monitor resource usage and set appropriate limits - ---- - -## Workaround Patterns for MCP Tool Limitations - -### General Pattern: Diagnose → Propose → Confirm → Execute - -When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): - -**1. Diagnose** the root cause using Kubernetes commands -```bash -oc describe vm -n -oc get events -n --field-selector involvedObject.name= -``` - -**2. Propose** a manual workaround to the user with clear explanation -- Explain why the issue occurred -- Show what will be changed -- List alternative options - -**3. Confirm** with user before executing (Human-in-the-Loop) -- Wait for explicit "yes" or "apply workaround" -- Never auto-execute modifications - -**4. Execute** the workaround using oc/kubectl -```bash -oc patch vm -n --type=merge -p '...' -``` - -**5. Verify** the fix was successful -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**6. Document** the limitation and suggest filing enhancement request -- Note this is temporary until MCP tool is enhanced -- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues - -### Example: Adding Tolerations Workaround - -**Diagnostic Output**: -```markdown -## ⚠️ VM Scheduling Issue Detected - -**Root Cause**: Node taints prevent VM scheduling - -**Details**: -- Found 3 nodes with taint: `virtualization=true:NoSchedule` -- VM spec does not include matching tolerations -- This prevents VM scheduling on virtualization-dedicated nodes -``` - -**Proposed Workaround**: -```bash -oc patch vm web-server -n vms --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' -``` - -**User Confirmation Required**: -``` -How would you like to proceed? -- "apply workaround" - I'll patch the VM with tolerations -- "manual" - I'll provide instructions for you to apply manually -- "cancel" - Delete the VM and abort creation -``` - -**After Execution**: -```markdown -## ✓ Workaround Applied Successfully - -**Action**: Added tolerations for taint `virtualization=true:NoSchedule` -**New Status**: Stopped (VM can now be scheduled) - -**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. -Future VMs in this cluster will need the same fix until the tool is enhanced. -``` - ---- - -## VM Status Reference - -### Status Values - -| Status | Meaning | Action Required | -|--------|---------|-----------------| -| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | -| `Running` | VM is running | Normal | -| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | -| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | -| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | -| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | -| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | -| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | -| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | -| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | - -### Checking VM Status - -```bash -# Get printable status -oc get vm -n -o jsonpath='{.status.printableStatus}' - -# Get detailed status and conditions -oc get vm -n -o jsonpath='{.status}' | jq - -# Watch status changes in real-time -oc get vm -n -w -``` - ---- - -## Best Practices for Agents - -When implementing diagnostic workflows: - -1. **Always verify VM status** after creation (wait 5-10 seconds first) -2. **Consult this document** when encountering error status values -3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) -4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) -5. **Respect human-in-the-loop** for all VM modifications -6. **Document temporary workarounds** and their limitations clearly -7. **Suggest filing issues** for missing MCP tool features - -### Document Consultation Pattern - -```markdown -**Document Consultation** (REQUIRED): -1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes -2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." -``` - ---- - -## Known MCP Tool Limitations - -### vm_create tool - -**Currently Supported**: -- ✓ Namespace, name (required) -- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) -- ✓ Size hints (small, medium, large) -- ✓ Storage size -- ✓ Autostart flag -- ✓ Networks (Multus NetworkAttachmentDefinitions) -- ✓ Performance family (u1, o1, c1, m1) -- ✓ Instance type, preference - -**Not Currently Supported** (requires workarounds): -- ✗ Tolerations (for node taints) -- ✗ Node selectors -- ✗ Affinity/anti-affinity rules -- ✗ Resource requests/limits (beyond instance type) -- ✗ Custom labels/annotations -- ✗ SSH keys injection -- ✗ Cloud-init user data - -**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. - -**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues - ---- - -## Additional Resources - -- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) -- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) -- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) -- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) - ---- - -[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/scheduling-errors.md deleted file mode 100644 index e47c8c4f..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/scheduling-errors.md +++ /dev/null @@ -1,417 +0,0 @@ ---- -title: VM Scheduling Errors -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 -tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] -semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] -use_cases: [vm-creation, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Scheduling Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**When to use this document**: -- VM shows status `ErrorUnschedulable` after creation or start attempt -- VM events mention scheduling failures, taints, resources, or node selectors - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -## ErrorUnschedulable - -**Symptom**: VM shows status `ErrorUnschedulable` after creation - -**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**Possible Causes**: - -### 1. Node Taints (Most Common) - -Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM events for scheduling failures**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `involvedObject.name` == "" and look for messages like: -- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= -``` - -**2. Check node taints in the cluster**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' -``` - -**Common Taint Patterns**: -- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule -- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes -- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads - -**Solution - Add Tolerations to VM**: - -The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. - -**Workaround (post-creation using MCP Tools)**: - -**Step 1**: Get current VM spec - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Step 2**: Modify the returned JSON to add tolerations - -Add to `.spec.template.spec.tolerations`: -```json -{ - "tolerations": [ - { - "key": "virtualization", - "operator": "Equal", - "value": "true", - "effect": "NoSchedule" - } - ] -} -``` - -**Step 3**: Update VM with modified spec - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters**: -```json -{ - "resource": "" -} -``` - -Pass the complete modified VM resource as YAML or JSON string. - -**Step 4**: Verify tolerations were added - -Use `resources_get` again and check `.spec.template.spec.tolerations` in response. - -**Step 5**: Check if VM status improved - -Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. - -**CLI Fallback** (if MCP patch is too complex): -```bash -# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' - -# Verify tolerations -oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq - -# Check status -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**Example - Multiple Tolerations**: -```bash -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" - - key: "dedicated" - operator: "Equal" - value: "virt-workloads" - effect: "NoSchedule" -' -``` - -**Toleration Operators**: -- `Equal` - Key and value must match exactly -- `Exists` - Only key must exist (ignores value) - -**Toleration Effects**: -- `NoSchedule` - Don't schedule new pods (existing pods continue) -- `PreferNoSchedule` - Avoid scheduling if possible -- `NoExecute` - Don't schedule AND evict existing pods - -**Alternative Solutions**: -1. **Remove node taints** (if you have cluster-admin access): - ```bash - oc adm taint nodes virtualization=true:NoSchedule- - ``` - -2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits - -3. **File enhancement request** - Request tolerations support in openshift-mcp-server: - https://github.com/openshift/openshift-mcp-server/issues - ---- - -### 2. Insufficient Resources - -Not enough CPU, memory, or storage available on any node. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM resource requests**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' -``` - -**2. Check node resource availability**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. - -Alternatively, use `nodes_top` MCP tool for current resource usage. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe nodes | grep -A 5 "Allocated resources" -``` - -**3. Look for VM events mentioning "Insufficient"**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep "Insufficient" -``` - -**Example Event**: -``` -0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. -``` - -**Solutions** (Use MCP Tools First): - -1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) -2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill -3. **Delete unused VMs** - Use vm-delete skill to free up resources -4. **Check resource quotas**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters for quota**: - ```json - { - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" - } - ``` - - **Parameters for limit range**: - ```json - { - "apiVersion": "v1", - "kind": "LimitRange", - "namespace": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - oc describe limitrange -n - ``` - ---- - -### 3. Node Selector Mismatch - -VM requires specific node labels that don't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM node selector requirements**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.nodeSelector` to see required node labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' -``` - -**2. List available node labels**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.metadata.labels` for available labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes --show-labels -``` - -**3. Check if any nodes match the selector**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node", - "labelSelector": "=" -} -``` - -Should return at least one node with matching labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -l = -``` - -**Solutions** (Use MCP Tools First): - -**Option 1: Remove node selector from VM** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get current VM using `resources_get` (diagnostic step 1) -2. Remove `.spec.template.spec.nodeSelector` field -3. Update VM using `resources_create_or_update` with modified JSON - -**CLI Fallback** (JSON patch easier via CLI): -Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" -```bash -oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' -``` - -**Option 2: Add label to nodes** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get node using `resources_get` -2. Add label to `.metadata.labels` -3. Update node using `resources_create_or_update` - -⚠️ **Note**: Node labeling typically requires cluster admin privileges. - -**CLI Fallback** (simpler via CLI): -Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" -```bash -oc label node = -``` - ---- - -[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/storage-errors.md deleted file mode 100644 index 16ba584d..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/docs/troubleshooting/storage-errors.md +++ /dev/null @@ -1,1011 +0,0 @@ ---- -title: VM Storage Errors -category: kubevirt -sources: - - title: KubeVirt CDI - DataVolumes - url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ - date_accessed: 2026-02-17 - - title: Kubernetes Persistent Volumes - url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ - date_accessed: 2026-02-17 -tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] -semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] -use_cases: [vm-creation, vm-deletion, vm-cloning] -related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] -last_updated: 2026-02-17 ---- - -# VM Storage Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. - -**When to use this document**: -- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` -- Storage deletion fails after VM deletion -- DataVolume cloning operations fail -- PVC provisioning issues - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### ErrorDataVolumeNotReady - -**Symptom**: VM shows status `ErrorDataVolumeNotReady` - -**Description**: The DataVolume (persistent storage) backing the VM is not ready. - -**Possible Causes**: - -#### 1. DataVolume Still Provisioning - -Storage provisioning takes time, especially for large disks or when importing images. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Get detailed DataVolume information**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` and `.status.conditions` for provisioning details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o json -``` - -**3. Check PVC (PersistentVolumeClaim) bound status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Check `.status.phase` for each PVC (should be `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. - -#### 2. Storage Class Not Found - -The requested storage class doesn't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List available storage classes**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" -} -``` - -Review the list of available storage classes (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -``` - -**2. Check DataVolume's requested storage class**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.spec.pvc.storageClassName` in the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' -``` - -**Solution**: -1. Use a valid storage class from the cluster -2. Recreate VM with correct storage class parameter - -#### 3. Insufficient Storage Quota - -Namespace has insufficient storage quota to provision the PVC. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check resource quotas**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**2. Check storage usage**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase -``` - -**Solution**: -1. Request quota increase from cluster admin -2. Delete unused PVCs to free quota -3. Reduce VM storage size - ---- - -### ErrorPvcNotFound - -**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List PVCs in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review the list of available PVCs (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check VM's PVC references**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' -``` - -**Solution**: -- Wait for DataVolume to create the PVC -- Manually create missing PVC -- Fix VM spec to reference correct PVC name - ---- - - ---- - -### Storage Deletion Failures - -**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace - -**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. - -**Possible Causes**: -- PVC still bound to active PersistentVolume with `Retain` policy -- DataVolume still being referenced by another resource -- CDI (Containerized Data Importer) controller issues -- Storage class retention policy preventing deletion -- Finalizers on PVC/DataVolume blocking cleanup -- PVC still mounted by a pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check PVC status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check specific PVC phase**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Released` or `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o jsonpath='{.status.phase}' -``` - -**3. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**4. Check what's using the PVC**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' -``` - -**5. Check PVC finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. - -**6. Check DataVolume finalizers**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.metadata.finalizers}' -``` - -**7. Check PV reclaim policy**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume" -} -``` - -Filter results for PV where `.spec.claimRef.name` matches ``. - -To get specific PV policy: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume", - "name": "" -} -``` - -Check `.spec.persistentVolumeReclaimPolicy`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pv | grep -oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' -``` - -**Common Finalizer Patterns**: -- `kubernetes.io/pvc-protection` - Protects PVC while in use -- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer - -**Solutions** (Use MCP Tools First): - -1. **Delete DataVolume first, then PVC**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete DataVolume first (often blocks PVC deletion): - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Wait a few seconds, then delete PVC: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete datavolume -n - oc delete pvc -n - ``` - -2. **Check for pods still using PVC**: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "" - } - ``` - - Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. - - Then delete the pods using `pods_delete`: - - **Parameters**: - ```json - { - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' - oc delete pod -n - ``` - -3. **Force delete PVC** (if safe to do so): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete pvc -n --grace-period=0 --force - ``` - -4. **Remove finalizers from PVC** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PVC using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update PVC using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Remove finalizers from DataVolume** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current DataVolume using `resources_get` (diagnostic step 6) - 2. Remove items from `.metadata.finalizers` array - 3. Update DataVolume using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Change PV reclaim policy** (if PV has Retain policy): - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PV policy using `resources_get` (diagnostic step 7) - 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` - 3. Update PV using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" - ```bash - oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' - oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' - ``` - -**Storage Quota Check** (Use MCP Tools First): - -After deletion, verify storage quota is freed: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters for quota check**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.used` to verify storage quota is freed. - -**Parameters for PVC verification**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -oc get pvc -n -``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Confirm PVC is deleted: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Confirm DataVolume is deleted: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -# Should return: Error from server (NotFound) - -oc get datavolume -n -# Should return: Error from server (NotFound) -``` - ---- - -### DataVolume Cloning Failures - -**Symptom**: VM clone created successfully but DataVolume clone operation fails - -**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. - -**Possible Causes**: -- CSI driver doesn't support volume cloning -- Source PVC storage class incompatible with cloning -- Cross-namespace cloning not permitted by storage backend -- Insufficient storage quota in target namespace -- Source PVC not in `Bound` state -- Storage class doesn't have volume cloning enabled -- CDI (Containerized Data Importer) controller issues - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume clone status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and `.metadata.name`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Check DataVolume events for errors**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe datavolume -n -``` - -**3. Check DataVolume phase**: - -Use `resources_get` from step 1, extract `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -``` - -**4. Check if storage class supports cloning**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass", - "name": "" -} -``` - -Review the full YAML output for cloning-related configurations. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -o yaml | grep -A 5 -i clone -``` - -**5. Check CSI driver capabilities**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "CSIDriver" -} -``` - -Review `.items[].metadata.name` for available CSI drivers. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get csidriver -``` - -**6. Check source PVC status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Bound` for cloning to work). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**7. Check target namespace storage quota**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**8. Check CDI controller logs**: - -**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) - -First, list pods in openshift-cnv namespace: - -**Parameters for pods_list_in_namespace**: -```json -{ - "namespace": "openshift-cnv", - "labelSelector": "app.kubernetes.io/component=cdi-deployment" -} -``` - -Then get logs using `pods_log`: - -**Parameters**: -```json -{ - "namespace": "openshift-cnv", - "name": "", - "tail": 100 -} -``` - -**CLI Fallback** (if MCP unavailable or easier via CLI): -```bash -oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') -``` - -**Common Error Messages**: -- `"volume cloning is not supported"` - CSI driver lacks clone capability -- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage -- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace -- `"insufficient quota"` - Target namespace lacks storage quota -- `"source PVC not bound"` - Source PVC must be in Bound state for cloning -- `"StorageClass does not support cloning"` - Storage class configuration issue - -**Solutions** (Use MCP Tools First): - -1. **Check storage class clone support**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" - } - ``` - - For each storage class in `.items[]`, check: - - `.metadata.name` (storage class name) - - `.provisioner` (CSI driver) - - Storage classes using CSI drivers typically support cloning. Look for provisioners like: - - `csi.ovirt.org` (oVirt CSI) - - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) - - `ebs.csi.aws.com` (AWS EBS CSI) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner - ``` - -2. **Verify source PVC is bound**: - - Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pvc -n -o jsonpath='{.status.phase}' - ``` - -3. **Check target namespace quota**: - - Use `resources_list` from diagnostic step 7 to check quota. - - If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - ``` - -4. **Use snapshot-based cloning** (alternative method): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Step 1**: Create VolumeSnapshot of source PVC - - **Parameters**: - ```json - { - "apiVersion": "snapshot.storage.k8s.io/v1", - "kind": "VolumeSnapshot", - "metadata": { - "name": "-snapshot", - "namespace": "" - }, - "spec": { - "source": { - "persistentVolumeClaimName": "" - } - } - } - ``` - - **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) - - **Step 3**: Create new DataVolume from snapshot - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "metadata": { - "name": "-rootdisk", - "namespace": "" - }, - "spec": { - "source": { - "snapshot": { - "name": "-snapshot", - "namespace": "" - } - }, - "storage": { - "resources": { - "requests": { - "storage": "50Gi" - } - }, - "storageClassName": "" - } - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" - ```bash - cat <-snapshot - namespace: - spec: - source: - persistentVolumeClaimName: - EOF - - oc get volumesnapshot -snapshot -n - - cat <-rootdisk - namespace: - spec: - source: - snapshot: - name: -snapshot - namespace: - storage: - resources: - requests: - storage: 50Gi - storageClassName: - EOF - ``` - -5. **Use "new empty storage" option** (vm-clone skill): - - If cloning isn't supported, create VM with empty storage - - Manually copy data if needed - -6. **Cross-namespace cloning workaround**: - - Some storage backends require snapshot for cross-namespace cloning - - Create snapshot in source namespace, restore in target namespace (see solution 4 above) - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Check DataVolume reached Succeeded phase: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should return `Succeeded`). - -Check PVC was created and bound: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "-rootdisk" -} -``` - -Check `.status.phase` (should return `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -# Should return: Succeeded - -oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' -# Should return: Bound -``` - -**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/skills/vm-snapshot-delete/SKILL.md b/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/skills/vm-snapshot-delete/SKILL.md deleted file mode 100644 index 14f2fb49..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-delete/environment/skills/vm-snapshot-delete/SKILL.md +++ /dev/null @@ -1,447 +0,0 @@ ---- -name: vm-snapshot-delete -description: | - Permanently delete virtual machine snapshots to free storage space. - - Use when: - - "Delete snapshot [snapshot-name]" - - "Remove old snapshots for VM [name]" - - "Free up snapshot storage" - - Requires user confirmation before deletion. - - NOT for restoring VMs (use vm-snapshot-restore instead). - -model: inherit -color: yellow ---- - -# /vm-snapshot-delete Skill - -Permanently delete virtual machine snapshots in OpenShift Virtualization. Deleting snapshots frees storage but removes recovery points. - -## Prerequisites - -**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools**: -- `resources_get` (from openshift-virtualization) - Verify snapshot exists -- `resources_list` (from openshift-virtualization) - List related snapshots -- `resources_delete` (from openshift-virtualization) - Delete snapshot - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster (>= 4.19) -- OpenShift Virtualization operator installed -- ServiceAccount with RBAC permissions to delete VirtualMachineSnapshot resources - -### Prerequisite Verification - -**Before executing, verify MCP server availability:** - -1. **Check MCP Server Configuration** - - Verify `openshift-virtualization` exists in `.mcp.json` - - If missing → Report to user with setup instructions - -2. **Check Environment Variables** - - Verify `KUBECONFIG` is set (check presence only, never expose value) - - If missing → Report to user - -## When to Use This Skill - -**Trigger this skill when:** -- User wants to free storage by removing old snapshots -- User wants to delete a specific snapshot -- User wants to implement snapshot retention policies - -**User phrases that trigger this skill:** -- "Delete snapshot pre-upgrade-backup" -- "Remove old snapshots for VM database-01" -- "Delete all snapshots older than 7 days" -- "Free up snapshot storage" - -**Do NOT use this skill when:** -- User wants to create snapshots → Use `vm-snapshot-create` skill -- User wants to restore from snapshot → Use `vm-snapshot-restore` skill -- User wants to list snapshots → Use `vm-snapshot-list` skill - -## Workflow - -### Step 1: Gather Delete Information - -**Required Information from User:** -1. **Snapshot Name** - Name of snapshot to delete -2. **Namespace** - Namespace where snapshot exists - -If namespace not provided, ask for it. - -### Step 2: Verify Snapshot Exists - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "snapshot.kubevirt.io/v1beta1", - "kind": "VirtualMachineSnapshot", - "namespace": "", - "name": "" -} -``` - -**Expected Output**: VirtualMachineSnapshot resource - -**Error Handling**: -- If snapshot not found → Report error - -**If snapshot not found:** -```markdown -❌ Snapshot Not Found - -**Snapshot**: `` not found in namespace ``. - -**To list available snapshots:** -"List snapshots in namespace " - -Delete operation cancelled. -``` - -**STOP workflow**. - -**If snapshot found**, extract snapshot details: -- `spec.source.name` - VM name -- `metadata.creationTimestamp` - Creation timestamp -- `status.phase` - Snapshot status -- Calculate age from creationTimestamp - -### Step 3: List Other Snapshots for Same VM - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "snapshot.kubevirt.io/v1beta1", - "kind": "VirtualMachineSnapshot", - "namespace": "", - "labelSelector": "vm.kubevirt.io/name=" -} -``` - -**Fallback**: If label selector doesn't work, list all snapshots and filter by `spec.source.name`. - -**Count snapshots** for the VM to determine if this is the last snapshot. - -### Step 4: Present Snapshot Details and Confirm Deletion - -```markdown -## ⚠️ Snapshot Deletion - Review - -**Snapshot to Delete**: `` - -### Snapshot Details -- **Snapshot Name**: `` -- **VM**: `` -- **Namespace**: `` -- **Created**: -- **Age**: -- **Status**: - -### Impact of Deletion -- ✗ Snapshot will be permanently deleted -- ✗ This recovery point will be lost -- ✗ Cannot restore VM to this snapshot state after deletion -- ✓ Storage will be freed - -### Recovery Impact -**Before deletion, consider:** -- Is this snapshot still needed for recovery? -- Are there other recovery points available? -- Could you need to restore to this state in the future? - -**Available snapshots for VM ``:** - - - -⚠️ **WARNING**: This is the ONLY snapshot for VM ``. After deletion, no snapshot recovery points will exist. - - ---- - -**Proceed with snapshot deletion? This action cannot be undone. (yes/no)** -``` - -**Wait for user confirmation.** - -**Handle response:** -- If "yes" → Proceed to Step 5 (execute deletion) -- If "no", "cancel", or anything else → Cancel operation - -**On cancellation:** -```markdown -Snapshot deletion cancelled by user. Snapshot `` preserved. -``` - -**STOP workflow**. - -### Step 5: Delete the Snapshot - -**ONLY PROCEED AFTER user confirmation in Step 4.** - -**MCP Tool**: `resources_delete` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "snapshot.kubevirt.io/v1beta1", - "kind": "VirtualMachineSnapshot", - "namespace": "", - "name": "" -} -``` - -**Example tool invocation:** -```json -resources_delete({ - "apiVersion": "snapshot.kubevirt.io/v1beta1", - "kind": "VirtualMachineSnapshot", - "namespace": "production", - "name": "old-snapshot" -}) -``` - -**Expected Output**: VirtualMachineSnapshot deleted successfully - -**Error Handling**: -- If snapshot not found → Report error (may have been deleted externally) -- If permission denied → Report RBAC error -- If snapshot in use → Report error (snapshot may be in restore process) - -**Report progress:** -```markdown -🗑️ Deleting snapshot... -✓ Snapshot `` deleted -``` - -### Step 6: Report Deletion Results - -**On success:** - -```markdown -## ✓ Snapshot Deleted Successfully - -**Snapshot**: `` (VM: ``, namespace: ``) - -### Deletion Summary -- ✓ Snapshot permanently deleted -- ✓ Storage freed -- ✓ Recovery point removed - -### Impact -- ✗ Cannot restore VM to state -- ✗ Snapshot `` no longer available - - -### Remaining Snapshots for VM `` - - - -These snapshots are still available for recovery. - - - -⚠️ **No snapshots remain** for VM ``. Consider creating new snapshots for future recovery points. - - ---- - -### Next Steps - -**To create a new snapshot:** -"Create snapshot of VM " - -**To list remaining snapshots:** -"List snapshots for VM " -``` - -**On failure:** - -```markdown -## ❌ Snapshot Deletion Failed - -**Error**: - -**Snapshot**: `` (VM: ``, namespace: ``) - -**Common Causes:** -- **Snapshot not found** - May have been deleted externally -- **Insufficient RBAC permissions** - ServiceAccount lacks delete permissions -- **Snapshot in use** - Snapshot may be in active restore process -- **Storage backend error** - CSI driver or storage backend issue - -**Troubleshooting Steps:** - -1. **Verify snapshot still exists:** - "List snapshots for VM " - -2. **Check if snapshot is being used for restore:** - Use `resources_list` to check for active VirtualMachineRestore resources - -3. **Check permissions:** - Use CLI: `oc auth can-i delete virtualmachinesnapshots -n ` - -4. **Wait and retry** if snapshot is in use by restore operation - -Would you like help troubleshooting this error? -``` - -## Common Issues - -### Issue 1: Snapshot Not Found - -**Error**: "Snapshot `` not found in namespace ``" - -**Cause**: Snapshot doesn't exist, was deleted, or wrong namespace/name. - -**Solution:** -1. List snapshots to verify name: "List snapshots in namespace " -2. Check spelling (names are case-sensitive) -3. Try listing in other namespaces if unsure - -### Issue 2: Snapshot In Use During Restore - -**Error**: "Snapshot is in use by restore operation" - -**Cause**: An active VirtualMachineRestore is using this snapshot. - -**Solution:** -1. Check for active restores: Use `resources_list` with apiVersion="snapshot.kubevirt.io/v1beta1", kind="VirtualMachineRestore" -2. Wait for restore to complete, or delete the VirtualMachineRestore resource -3. Retry snapshot deletion - -### Issue 3: Permission Denied - -**Error**: "Forbidden: User lacks permissions to delete virtualmachinesnapshots" - -**Cause**: Missing RBAC permissions for snapshot deletion. - -**Solution:** -1. Check permissions: `oc auth can-i delete virtualmachinesnapshots -n ` -2. Contact cluster admin to grant delete permissions for virtualmachinesnapshots -3. Required permissions: delete verb on snapshot.kubevirt.io/virtualmachinesnapshots - -## Dependencies - -### Required MCP Servers -- `openshift-virtualization` - OpenShift MCP server with kubevirt toolset - -### Required MCP Tools -- `resources_get` (from openshift-virtualization) - Get snapshot details - - Parameters: apiVersion, kind, namespace, name - - Source: https://github.com/openshift/openshift-mcp-server - -- `resources_list` (from openshift-virtualization) - List related snapshots - - Parameters: apiVersion, kind, namespace, labelSelector - - Source: https://github.com/openshift/openshift-mcp-server - -- `resources_delete` (from openshift-virtualization) - Delete Kubernetes resources - - Parameters: apiVersion, kind, namespace, name - - Source: https://github.com/openshift/openshift-mcp-server - -### Related Skills -- `vm-snapshot-list` - List snapshots before deletion -- `vm-snapshot-create` - Create new snapshots -- `vm-snapshot-restore` - Restore VMs from snapshots - -### Reference Documentation -- [OpenShift Virtualization Snapshots](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt-managing-vm-snapshots) -- [KubeVirt VM Snapshots](https://kubevirt.io/user-guide/operations/snapshot_restore_api/) - -## Critical: Human-in-the-Loop Requirements - -**IMPORTANT:** This skill performs destructive operations. You MUST: - -1. **Before Deleting Snapshots** - - Show snapshot details (VM, age, size) - - Confirm snapshot won't be needed for recovery - - List other available snapshots for the VM - - Ask: "Proceed with snapshot deletion? (yes/no)" - - Wait for explicit "yes" - -2. **Never Auto-Execute** - - **NEVER delete without user confirmation** - - **ALWAYS show what will be lost before deletion** - -**Why This Matters:** -- **Recovery Point Loss**: Deleting snapshots removes recovery options permanently -- **No Undo**: Snapshot deletion cannot be reversed -- **Last Snapshot Warning**: Users must know if deleting the only snapshot for a VM - -## Security Considerations - -- **RBAC Enforcement**: Requires delete permissions for VirtualMachineSnapshot resources -- **User Confirmation**: Always requires explicit "yes" before deletion -- **Last Snapshot Warning**: Warns users when deleting the only snapshot for a VM -- **Namespace Isolation**: Snapshots scoped to namespace boundaries -- **Audit Trail**: Deletions logged in Kubernetes API audit logs - -## Example Usage - -``` -User: "Delete snapshot database-01-daily-backup" - -Agent: "Which namespace is the snapshot in?" - -User: "production" - -Agent: [Invokes vm-snapshot-delete skill] - [Step 2: Verifies snapshot exists] - [Step 3: Lists other snapshots] - [Step 4: Presents deletion confirmation] - -## ⚠️ Snapshot Deletion - Review - -Snapshot to Delete: `database-01-daily-backup` - -Snapshot Details: -- Name: `database-01-daily-backup` -- VM: `database-01` -- Namespace: `production` -- Created: 2024-01-14 02:00:00 UTC -- Age: 1 day - -Impact of Deletion: -- ✗ Snapshot will be permanently deleted -- ✗ This recovery point will be lost -- ✓ Storage will be freed - -Available snapshots for VM `database-01`: -- `database-01-pre-upgrade` (created 2024-01-15 10:30) - -Proceed with snapshot deletion? This action cannot be undone. (yes/no) - -User: "yes" - -Agent: [Step 5: Deletes snapshot] - -🗑️ Deleting snapshot... -✓ Snapshot `database-01-daily-backup` deleted - - [Step 6: Reports results] - -## ✓ Snapshot Deleted Successfully - -Snapshot: `database-01-daily-backup` - -Deletion Summary: -- ✓ Snapshot permanently deleted -- ✓ Storage freed -- ✓ Recovery point removed - -Remaining Snapshots for VM `database-01`: -- `database-01-pre-upgrade` (created 2024-01-15 10:30) - -This snapshot is still available for recovery. -``` diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/.ai-index/semantic-index.json deleted file mode 100644 index 2512c2a3..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/.ai-index/semantic-index.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "index_version": "1.0.0", - "last_updated": "2026-02-17", - "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", - - "documents": [ - { - "file": "INDEX.md", - "title": "VM Troubleshooting Guide - Index", - "category": "navigation", - "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], - "use_cases": ["navigation", "tool-reference", "general-guidance"], - "token_estimate": 450, - "skills": ["all"] - }, - { - "file": "scheduling-errors.md", - "title": "VM Scheduling Errors", - "category": "scheduling", - "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], - "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], - "errors_covered": [ - "ErrorUnschedulable - Node Taints", - "ErrorUnschedulable - Insufficient Resources", - "ErrorUnschedulable - Node Selector Mismatch" - ], - "token_estimate": 820, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "storage-errors.md", - "title": "VM Storage Errors", - "category": "storage", - "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], - "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], - "errors_covered": [ - "ErrorDataVolumeNotReady - DataVolume Still Provisioning", - "ErrorDataVolumeNotReady - Storage Class Not Found", - "ErrorDataVolumeNotReady - Insufficient Storage Quota", - "ErrorPvcNotFound", - "Storage Deletion Failures", - "DataVolume Cloning Failures" - ], - "token_estimate": 1350, - "skills": ["vm-creator", "vm-delete", "vm-clone"] - }, - { - "file": "lifecycle-errors.md", - "title": "VM Lifecycle Errors", - "category": "lifecycle", - "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], - "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], - "errors_covered": [ - "VM Stuck in Terminating State", - "VM Won't Start (Non-Scheduling Issues)", - "VM Won't Stop" - ], - "token_estimate": 1150, - "skills": ["vm-delete", "vm-lifecycle-manager"] - }, - { - "file": "runtime-errors.md", - "title": "VM Runtime Errors", - "category": "runtime", - "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], - "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], - "errors_covered": [ - "CrashLoopBackOff" - ], - "token_estimate": 850, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "network-errors.md", - "title": "VM Network Errors", - "category": "networking", - "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], - "use_cases": ["vm-creation", "network-troubleshooting"], - "errors_covered": [ - "Network Attachment Failures" - ], - "token_estimate": 580, - "skills": ["vm-creator"] - } - ], - - "skill_to_docs_mapping": { - "vm-creator": { - "primary_docs": ["scheduling-errors.md", "storage-errors.md"], - "secondary_docs": ["runtime-errors.md", "network-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3650, - "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" - }, - "vm-delete": { - "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 2950, - "description": "VM deletion failures - stuck terminating, storage cleanup" - }, - "vm-clone": { - "primary_docs": ["storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 1800, - "description": "VM cloning failures - DataVolume cloning, storage provisioning" - }, - "vm-lifecycle-manager": { - "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], - "secondary_docs": ["runtime-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3270, - "description": "VM start/stop failures - lifecycle management, scheduling" - }, - "vm-inventory": { - "primary_docs": [], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 450, - "description": "General guidance for status interpretation" - } - }, - - "error_to_docs_mapping": { - "ErrorUnschedulable": "scheduling-errors.md", - "ErrorDataVolumeNotReady": "storage-errors.md", - "ErrorPvcNotFound": "storage-errors.md", - "Terminating": "lifecycle-errors.md", - "CrashLoopBackOff": "runtime-errors.md", - "NetworkAttachmentFailed": "network-errors.md" - }, - - "cross_references": { - "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], - "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], - "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], - "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], - "network-errors.md": ["scheduling-errors.md"] - }, - - "progressive_disclosure_strategy": { - "description": "Load only the minimum documentation needed for current task", - "pattern": "error_type → primary_doc → cross_references (if needed)", - "token_savings": "29% average reduction vs single-file (based on rh-sre data)", - "navigation_overhead_reduction": "85% vs single-file navigation" - } -} diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/INDEX.md deleted file mode 100644 index a2e76963..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/INDEX.md +++ /dev/null @@ -1,332 +0,0 @@ ---- -title: VM Troubleshooting Guide - Index -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 - - title: OpenShift Virtualization - Virtual Machine Status - url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html - date_accessed: 2026-02-06 - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] -semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] -use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] -last_updated: 2026-02-17 ---- - -# VM Troubleshooting Guide - Index - -## Overview - -This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: - -- **Creation**: Scheduling failures, storage provisioning issues -- **Lifecycle**: Start/stop failures, stuck states -- **Deletion**: Resources stuck in Terminating, storage cleanup failures -- **Cloning**: DataVolume cloning errors, cross-namespace issues -- **Networking**: Secondary network attachment failures -- **Runtime**: CrashLoopBackOff, guest OS boot failures - -This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. - ---- - -## 🗂️ Troubleshooting Categories - -The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: - -### 1. [Scheduling Errors](scheduling-errors.md) -**When to use**: VM fails to schedule on any node - -**Errors covered**: -- ErrorUnschedulable - Node Taints -- ErrorUnschedulable - Insufficient Resources -- ErrorUnschedulable - Node Selector Mismatch - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 2. [Storage Errors](storage-errors.md) -**When to use**: VM has storage provisioning, deletion, or cloning issues - -**Errors covered**: -- ErrorDataVolumeNotReady (all 3 subsections) -- ErrorPvcNotFound -- Storage Deletion Failures -- DataVolume Cloning Failures - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### 3. [Lifecycle Errors](lifecycle-errors.md) -**When to use**: VM has start/stop/termination issues - -**Errors covered**: -- VM Stuck in Terminating State -- VM Won't Start (Non-Scheduling Issues) -- VM Won't Stop - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### 4. [Runtime Errors](runtime-errors.md) -**When to use**: VM repeatedly crashes or fails at runtime - -**Errors covered**: -- CrashLoopBackOff - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 5. [Network Errors](network-errors.md) -**When to use**: VM has secondary network attachment failures - -**Errors covered**: -- Network Attachment Failures (Multus, SR-IOV) - -**Skills that use this**: vm-create - ---- - -## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last - -### Critical Principle: MCP-First Approach - -**All diagnostic procedures in this guide follow the MCP-first pattern**: - -``` -1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) -2. ⚠️ IF FAILS: Ask user permission to use CLI command -3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval -``` - -### Why MCP Tools First? - -- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources -- **Consistency**: Same interface across all operations -- **Better Error Handling**: MCP tools return structured errors -- **Audit Trail**: MCP tool usage is logged and trackable - -### Command Pattern Examples - -Throughout this guide, you'll see diagnostic steps formatted like this: - -**✅ CORRECT Pattern (MCP First)**: - -```markdown -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Extract**: `.metadata.finalizers` from returned JSON - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` -``` - -### Available MCP Tools - -The `openshift-virtualization` MCP server provides these tools: - -**Resource Operations**: -- `resources_get` - Get specific resource (replaces `oc get `) -- `resources_list` - List resources (replaces `oc get `) -- `resources_delete` - Delete resource (replaces `oc delete`) -- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) - -**Pod Operations**: -- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) -- `pods_get` - Get pod details (replaces `oc get pod`) -- `pods_log` - Get pod logs (replaces `oc logs`) -- `pods_exec` - Execute in pod (replaces `oc exec`) -- `pods_delete` - Delete pod (replaces `oc delete pod`) -- `pods_top` - Pod resource usage (replaces `oc top pods`) - -**Events & Monitoring**: -- `events_list` - List events (replaces `oc get events`) -- `nodes_top` - Node resource usage (replaces `oc top nodes`) -- `nodes_log` - Node logs (replaces `oc adm node-logs`) -- `nodes_stats_summary` - Detailed node stats - -**VM Operations** (KubeVirt toolset): -- `vm_create` - Create VMs -- `vm_lifecycle` - Start/stop/restart VMs - -**Namespaces**: -- `namespaces_list` - List namespaces (replaces `oc get namespaces`) -- `projects_list` - List OpenShift projects (replaces `oc get projects`) - -### When CLI Commands Are Required - -Some operations have **NO MCP equivalent** and require CLI: - -- `virtctl` commands (console, VNC access) -- `oc debug node` (node debugging) -- `oc auth can-i` (permission checks) -- `oc adm taint` (node taint management) -- Complex JSON patch operations - -For these, the guide will note: **"CLI Only - No MCP equivalent"** - -### Quick Reference: CLI → MCP Mapping - -| CLI Command | MCP Tool Equivalent | -|-------------|---------------------| -| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | -| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | -| `oc logs -n ` | `pods_log` with name="", namespace="" | -| `oc get events -n ` | `events_list` with namespace="" | -| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | - -**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. - -### How to Read Diagnostic Sections - -Each error section includes: -1. **Symptom** - What you observe -2. **Description** - What's happening -3. **Possible Causes** - Why it's happening -4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback -5. **Solutions** - **MCP tools first**, then CLI fallback -6. **Verification** - **MCP tools first**, then CLI fallback - -**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. - ---- - -## 🔍 Quick Navigation by Skill - -**vm-create**: -- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics -- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning -- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff -- [Network Errors](network-errors.md) - Network attachment failures - -**vm-delete**: -- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state -- [Storage Errors](storage-errors.md) - Storage deletion failures - -**vm-clone**: -- [Storage Errors](storage-errors.md) - DataVolume cloning failures - -**vm-lifecycle-manager**: -- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop -- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling - -**vm-inventory**: -- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed - ---- - -## 📊 Documentation Coverage & Maintenance - -### Current Coverage - -This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: - -- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) -- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures -- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop -- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) -- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems - -**Total errors documented**: 12 error types across 6 categories - ---- - -### Encountering Undocumented Errors - -#### For AI Agents (Claude Code) - -If you encounter an error **not documented** in the categories above: - -1. **Report to user** with all available details (error message, affected resources, namespace) -2. **Provide best-effort diagnostics** using MCP tools: - - `resources_get` to inspect resource status - - `pods_log` to check virt-launcher or compute container logs - - `events_list` to view Kubernetes events -3. **Suggest documentation update**: - ``` - ⚠️ This error is not yet documented in the troubleshooting guide. - - **Error**: - **Affected resource**: / - - I recommend adding this error to the troubleshooting guide: - - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file - - If it's a new error class → Create new category file in docs/troubleshooting/ - - Would you like me to help document this error for future reference? - ``` - -4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs - ---- - -#### For Human Contributors - -To document a new error: - -1. **Determine the category**: - - Scheduling issue → `scheduling-errors.md` - - Storage problem → `storage-errors.md` - - Lifecycle/start/stop → `lifecycle-errors.md` - - Runtime crash → `runtime-errors.md` - - Network issue → `network-errors.md` - - New error class → Create new category file (e.g., `security-errors.md`) - -2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): - - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback - - **Solutions**: MCP Tool approach → Implementation → CLI Fallback - - **Verification**: MCP Tool checks → Expected results → CLI Fallback - -3. **Update semantic index**: - - Edit `.ai-index/semantic-index.json` - - Add error to `error_to_docs_mapping` - - Update relevant skill's `primary_docs` or `secondary_docs` - - Increment token estimates if needed - -4. **Reference in skill files**: - - Update skill's Reference Documentation section - - Ensure skill's Document Consultation steps point to the right category - -5. **Update this INDEX.md**: - - Add error to the appropriate category section (lines 50-120) - - Update "Current Coverage" list above - ---- - -## 📚 Additional Resources - -- [rh-virt README](../../README.md) - MCP server setup and skill documentation -- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources -- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/lifecycle-errors.md deleted file mode 100644 index 6167fcc8..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/lifecycle-errors.md +++ /dev/null @@ -1,869 +0,0 @@ ---- -title: VM Lifecycle Errors -category: kubevirt -sources: - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 -tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] -semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] -use_cases: [vm-deletion, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Lifecycle Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM lifecycle issues including start/stop failures and stuck termination states. - -**When to use this document**: -- VM stuck in `Terminating` state -- VM won't start (runStrategy is Always but VM never reaches Running) -- VM won't stop (runStrategy is Halted but VM never reaches Stopped) - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### VM Stuck in Terminating State - -**Symptom**: VM shows status `Terminating` but deletion never completes - -**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). - -**Possible Causes**: -- Finalizers blocking deletion -- PVC/DataVolume still attached and preventing cleanup -- VirtualMachineInstance (VMI) not terminating properly -- Custom controllers or operators blocking deletion -- Stuck virt-launcher pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check finalizers on the VM**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` - -**2. Check if VMI still exists**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name containing "virt-launcher-". - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check events for deletion issues**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results where `involvedObject.name` == "" and sort by timestamp. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' -``` - -**5. Check VM deletion timestamp**: - -Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. - -If present, VM is in deletion process. If null, VM is not being deleted. - -**Common Finalizer Patterns**: -- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer -- `foregroundDeletion` - Waits for dependent objects to be deleted -- Custom finalizers from operators - -**Solutions** (Use MCP Tools First): - -1. **Wait for dependent resources** (safest approach): - - Use `resources_list` to check if PVCs, DataVolumes are still deleting - - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) - -2. **Force delete VMI** (if VMI is stuck): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable - requires explicit user permission): - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -3. **Force delete virt-launcher pod** (if pod is stuck): - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - # Find the virt-launcher pod name first - oc get pods -n | grep virt-launcher- - - # Then delete it - oc delete pod virt-launcher--xxx -n --force --grace-period=0 - ``` - -4. **Remove finalizers** (⚠️ dangerous - use only as last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` - 2. Remove items from `.metadata.finalizers` array - 3. Update VM using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch not easily done via MCP): - ```bash - # This operation is complex for MCP - may need CLI - oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Check for protection labels** (vm-delete skill specific): - - Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. - - If value is "true", the vm-delete skill refuses deletion (this is expected behavior). - -**Verification**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error if deletion successful. - -**CLI Fallback**: -```bash -oc get vm -n -# Should return: Error from server (NotFound) -``` - ---- - - ---- - -### VM Won't Start (Non-Scheduling Issues) - -**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state - -**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. - -**Possible Causes**: -- Guest OS kernel panic or boot failure -- Cloud-init configuration errors -- Missing or corrupted disk image -- Insufficient memory for guest OS to boot -- QEMU/KVM virtualization errors -- VirtualMachineInstance (VMI) creation failures -- virt-launcher pod crashes - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VMI (VirtualMachineInstance) status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and overall VMI state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**2. Check VMI conditions for errors**: - -Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. View virt-launcher pod logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -First, get pod name from step 3, then: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -Look for QEMU errors, memory allocation failures, device errors. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**5. Check virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. - -Alternatively, use `pods_get` to get full pod details: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**6. Access VM console to see guest OS boot messages**: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -Look for kernel panic, initramfs errors, filesystem errors. - -**7. Check VM events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep -A 20 "Events:" -``` - -**Common Error Patterns**: - -1. **Guest OS Boot Failure**: - - Console shows kernel panic - - Guest hangs at GRUB or boot loader - - Cloud-init errors during first boot - -2. **Insufficient Memory**: - - Guest OS kills processes due to OOM - - VMI logs show memory allocation errors - -3. **QEMU Crashes**: - - virt-launcher logs show QEMU segfaults - - VMI repeatedly restarts - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for kernel panic, initramfs errors, filesystem errors. - -2. **Check virt-launcher pod logs for QEMU errors**: - - Use `pods_log` from diagnostic step 4 to view logs. - - Look for: - - "qemu-system-x86_64: ..." errors - - Memory allocation failures - - Device errors - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx - ``` - -3. **Increase memory if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check VMI memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -4. **Verify disk image integrity**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check PVC is bound: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - If using container disk, verify image exists and is accessible (check VMI spec). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get pvc -n - ``` - -5. **Check cloud-init configuration** (if applicable): - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. - - Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq - ``` - -6. **Restart VMI** (soft reset): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete VMI (VM controller will recreate it): - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for new VMI to start (use `resources_get` to check status). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n -w - ``` - -7. **Check virtualization extensions** (KVM): - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - - # In debug shell: - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should eventually return `Running`). - -Check VMI is running: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should show `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should eventually return: Running - -oc get vmi -n -# Should show: Running -``` - ---- - -### VM Won't Stop - -**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state - -**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. - -**Possible Causes**: -- Guest OS not responding to ACPI shutdown signal -- virt-launcher pod stuck and not terminating -- VirtualMachineInstance (VMI) deletion blocked by finalizers -- Guest shutdown scripts hanging -- Filesystem sync issues in guest OS - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (might show `Stopping`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**2. Check VMI status and deletion timestamp**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check VMI events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vmi -n | grep -A 10 "Events:" -``` - -**5. Check VMI finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.metadata.finalizers}' -``` - -**6. Check if guest is responsive** (if VMI still exists): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Patterns**: -- VMI shows `deletionTimestamp` but never actually deletes -- virt-launcher pod in `Terminating` state -- VM runStrategy is `Halted` but printableStatus shows `Stopping` - -**Solutions** (Use MCP Tools First): - -1. **Wait for graceful shutdown** (default: 30 seconds): - - Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. - -2. **Force stop by deleting VMI**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - This is the standard way to force-stop a VM. - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n - # Should return: Error from server (NotFound) - ``` - -3. **Force delete VMI with grace period** (if VMI won't delete): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -4. **Force delete virt-launcher pod**: - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find the pod using `pods_list_in_namespace` from diagnostic step 3. - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - ⚠️ **Note**: For force deletion with grace period, use CLI fallback. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') - oc delete pod $POD_NAME -n --force --grace-period=0 - ``` - -5. **Remove VMI finalizers** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VMI using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update VMI using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Patch VM runStrategy directly** (ensure consistency): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` (diagnostic step 1) - 2. Set `.spec.runStrategy` to `"Halted"` - 3. Update VM using `resources_create_or_update` with modified JSON - - **CLI Fallback** (merge patch easier via CLI): - Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Stopped` or `Halted`). - -Verify VMI is gone: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Verify virt-launcher pod is gone: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for pods matching `virt-launcher-`. Should return no results. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Stopped or Halted - -oc get vmi -n -# Should return: Error from server (NotFound) - -oc get pods -n | grep virt-launcher- -# Should return: No resources found -``` - -**Prevention**: -- Ensure guest OS has ACPI support enabled -- Use proper shutdown commands in guest OS -- Avoid forceful stops unless necessary (can corrupt guest filesystem) - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/network-errors.md deleted file mode 100644 index 82fa313c..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/network-errors.md +++ /dev/null @@ -1,429 +0,0 @@ ---- -title: VM Network Errors -category: kubevirt -sources: - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] -semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] -use_cases: [vm-creation, network-troubleshooting] -related_docs: [INDEX.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Network Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. - -**When to use this document**: -- VM created successfully but secondary networks not attached -- NetworkAttachmentDefinition not found errors -- Multus CNI failures -- SR-IOV device attachment issues - -**Skills that use this**: vm-create - ---- - -### Network Attachment Failures - -**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working - -**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). - -**Possible Causes**: -- NetworkAttachmentDefinition doesn't exist in the namespace -- Multus CNI not installed or not configured on cluster -- Namespace mismatch (NAD in different namespace than VM) -- Interface name conflicts in VM spec -- Bridge/network configuration errors in NAD -- SR-IOV device not available (if using SR-IOV) - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List NetworkAttachmentDefinitions in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "" -} -``` - -Review `.items[].metadata.name` for available NADs. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -n -``` - -**2. List NetworkAttachmentDefinitions in all namespaces**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition" -} -``` - -Omit `namespace` parameter to list across all namespaces. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -A -``` - -**3. Check specific NetworkAttachmentDefinition**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "", - "name": "" -} -``` - -Review `.spec.config` for CNI configuration. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definition -n -o yaml -``` - -**4. Check VM network configuration**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.networks` to see network references. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq -``` - -**5. Check VM domain interfaces**: - -Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq -``` - -**6. Check virt-launcher pod network annotations** (shows actual attachments): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**7. Check Multus is installed**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "openshift-multus" -} -``` - -Should show Multus CNI pods running. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n openshift-multus -``` - -**8. Check for errors in virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n -``` - -**Common Error Messages**: -- `"network-attachment-definition not found"` - NAD doesn't exist in namespace -- `"multus CNI not configured"` - Multus not installed or misconfigured -- `"interface name conflict"` - Duplicate interface names in VM spec -- `"failed to add network"` - CNI plugin error (check NAD config) -- `"no available devices"` - SR-IOV device not available (if using SR-IOV) - -**Solutions** (Use MCP Tools First): - -1. **Verify NetworkAttachmentDefinition exists in correct namespace**: - - Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. - - If NAD is in different namespace, copy it to VM namespace: - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get NAD from source namespace using `resources_get` - 2. Modify `.metadata.namespace` to target namespace - 3. Create NAD in target namespace using `resources_create_or_update` - - **CLI Fallback** (stream processing easier via CLI): - Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" - ```bash - oc get network-attachment-definition -n -o yaml | \ - sed "s/namespace: /namespace: /" | \ - oc apply -f - - ``` - -2. **Create missing NetworkAttachmentDefinition**: - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - Example: Linux bridge network - - **Parameters**: - ```json - { - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "metadata": { - "name": "vlan100", - "namespace": "" - }, - "spec": { - "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" - ```bash - cat < - spec: - config: '{ - "cniVersion": "0.3.1", - "type": "bridge", - "bridge": "br1", - "vlan": 100, - "ipam": { - "type": "host-local", - "subnet": "192.168.100.0/24" - } - }' - EOF - ``` - -3. **Check Multus CNI installation**: - - Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. - - To check cluster network operator: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "config.openshift.io/v1", - "kind": "ClusterOperator" - } - ``` - - Filter for `network` operator. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n openshift-multus - oc get clusteroperators network - ``` - -4. **Fix interface name conflicts** (if VM has duplicate names): - - Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. - - Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. - - **CLI Fallback** (interactive edit easier via CLI): - Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' - oc edit vm -n - ``` - -5. **Validate NAD configuration syntax**: - - Use `resources_get` from diagnostic step 3, extract `.spec.config`. - - Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get network-attachment-definition -n -o jsonpath='{.spec.config}' - ``` - -6. **Check SR-IOV device availability** (if using SR-IOV networks): - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - List SR-IOV network node policies: - - **Parameters**: - ```json - { - "apiVersion": "sriovnetwork.openshift.io/v1", - "kind": "SriovNetworkNodePolicy", - "namespace": "openshift-sriov-network-operator" - } - ``` - - Check SR-IOV device plugin pods: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "openshift-sriov-network-operator" - } - ``` - - Filter for pods with "device-plugin" in name. - - Check available SR-IOV devices on node: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "Node", - "name": "" - } - ``` - - Review `.status.allocatable` for SR-IOV resources. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get sriovnetworknodepolicy -n openshift-sriov-network-operator - oc get pods -n openshift-sriov-network-operator | grep device-plugin - oc describe node | grep -A 10 "Allocatable:" - ``` - -7. **Recreate VM with corrected network configuration** (if needed): - - If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -After remediation, check virt-launcher pod network status: - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -Should show all attached networks with status. Example output: -```json -[ - { - "name": "openshift-sdn", - "interface": "eth0", - "ips": ["10.128.2.10"], - "default": true - }, - { - "name": "vlan100", - "interface": "net1", - "ips": ["192.168.100.5"] - } -] -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**Check from inside VM** (via console): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# In guest OS: -ip addr show -# Should show all network interfaces (eth0, net1, etc.) -``` - -**Common Network Types**: -- **Linux Bridge**: Layer 2 bridge for VLAN networks -- **SR-IOV**: High-performance direct device assignment -- **macvlan**: MAC-based VLAN for container networks -- **OVN-Kubernetes**: OpenShift native overlay network - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/runtime-errors.md deleted file mode 100644 index 804d3cbc..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/runtime-errors.md +++ /dev/null @@ -1,616 +0,0 @@ ---- -title: VM Runtime Errors -category: kubevirt -sources: - - title: Kubernetes Pod Lifecycle - url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ - date_accessed: 2026-02-17 -tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] -semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] -use_cases: [vm-creation, vm-lifecycle, diagnostics] -related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Runtime Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. - -**When to use this document**: -- VM or virt-launcher pod shows `CrashLoopBackOff` status -- virt-launcher pod repeatedly restarting -- Guest OS kernel panics on boot - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### CrashLoopBackOff - -**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting - -**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. - -**Possible Causes**: -- Guest OS kernel panic on boot -- Insufficient resources (memory/CPU) for guest OS -- Corrupted disk image or filesystem -- QEMU/libvirt crashes due to configuration errors -- Missing or incompatible device drivers in guest -- Resource limits too low for virt-launcher pod -- Virtualization features (KVM) not available on node - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check virt-launcher pod restart count**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# Look at RESTARTS column -``` - -**2. View recent crash logs** (previous container instance): - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "previous": true, - "tail": 100 -} -``` - -Look for QEMU errors, kernel panics, or segfaults. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n virt-launcher--xxx --previous -``` - -**3. Check current virt-launcher logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. - -**CLI Fallback** (if MCP unavailable or all containers needed): -```bash -oc logs -n virt-launcher--xxx --all-containers -``` - -**4. Check VMI conditions for crash details**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Extract `.status.conditions` for crash details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**5. Check pod events for crash reasons**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" -``` - -**6. Check pod resource limits**: - -**MCP Tool**: `pods_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -Extract `.spec.containers[0].resources` for resource limits. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq -``` - -**7. Check node kubelet logs for OOM kills**: - -⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -oc adm node-logs -u kubelet | grep -i oom -``` - -**8. Access guest console** (if VM briefly starts): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Crash Patterns**: - -1. **Guest Kernel Panic**: - - Console logs show kernel panic messages - - Guest crashes immediately after boot - - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" - -2. **OOM (Out of Memory)**: - - Pod killed with reason: `OOMKilled` - - Guest runs out of memory during boot or operation - - virt-launcher logs show memory allocation failures - -3. **QEMU Crash**: - - virt-launcher logs show QEMU segmentation fault - - Symptoms: "qemu-system-x86_64: terminated by signal" - - Configuration incompatibility or QEMU bug - -4. **Disk Image Corruption**: - - Guest cannot boot from disk - - Filesystem errors in guest console - - DataVolume import failed - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for kernel panic or boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for: - - Kernel panic messages - - Initramfs errors - - Filesystem mounting failures - - Missing device errors - -2. **Review virt-launcher crash logs**: - - Use `pods_log` with `previous: true` from diagnostic step 2. - - Look for: - - QEMU command line errors - - Device initialization failures - - Memory allocation errors - - Signal termination (SIGSEGV, SIGABRT) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx --previous - ``` - -3. **Check for OOM (Out of Memory) kills**: - - **MCP Tool**: `pods_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - Extract `.status.containerStatuses[0].lastState.terminated.reason`. - - If returns `"OOMKilled"`: - - Option 1: Increase virt-launcher memory limits - - Option 2: Decrease guest memory allocation - - Option 3: Use smaller instance type - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' - ``` - -4. **Increase resources if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check current memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -5. **Verify disk image integrity**: - - **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters for list**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" - } - ``` - - **Parameters for specific DV**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check `.status.phase` (should be `Succeeded`). - - If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get datavolume -n -o jsonpath='{.status.phase}' - ``` - -6. **Check virtualization (KVM) availability**: - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -7. **Simplify VM configuration** (eliminate variables): - - Try creating minimal VM using vm-create skill with: - - Small instance type - - No secondary networks - - Simple container disk (e.g., Fedora) - - No cloud-init - - If minimal VM works, add features back one by one. - -8. **Recreate VM with different workload** (test disk image): - - If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) - -After remediation, check pod restart count stops increasing: - -**Parameters for pods**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). - -Check VM reaches Running state: - -**Parameters for VM**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# RESTARTS should stabilize - -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Running -``` - -Verify guest is responsive: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# Should show login prompt or OS console -``` - -**Advanced Debugging**: - -**MCP Tool**: `pods_exec` (from openshift-virtualization) - -Check libvirt domain XML: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["virsh", "dumpxml", "1"] -} -``` - -Check QEMU process: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["ps", "aux"] -} -``` - -Filter output for "qemu" process. - -**CLI Fallback** (if MCP unavailable): -```bash -oc exec -n virt-launcher--xxx -- virsh dumpxml 1 -oc exec -n virt-launcher--xxx -- ps aux | grep qemu -``` - -**Prevention**: -- Start with minimal VM configuration and add complexity gradually -- Use recommended instance types for your workload -- Test disk images before deploying to production -- Ensure nodes have adequate resources and KVM support -- Monitor resource usage and set appropriate limits - ---- - -## Workaround Patterns for MCP Tool Limitations - -### General Pattern: Diagnose → Propose → Confirm → Execute - -When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): - -**1. Diagnose** the root cause using Kubernetes commands -```bash -oc describe vm -n -oc get events -n --field-selector involvedObject.name= -``` - -**2. Propose** a manual workaround to the user with clear explanation -- Explain why the issue occurred -- Show what will be changed -- List alternative options - -**3. Confirm** with user before executing (Human-in-the-Loop) -- Wait for explicit "yes" or "apply workaround" -- Never auto-execute modifications - -**4. Execute** the workaround using oc/kubectl -```bash -oc patch vm -n --type=merge -p '...' -``` - -**5. Verify** the fix was successful -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**6. Document** the limitation and suggest filing enhancement request -- Note this is temporary until MCP tool is enhanced -- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues - -### Example: Adding Tolerations Workaround - -**Diagnostic Output**: -```markdown -## ⚠️ VM Scheduling Issue Detected - -**Root Cause**: Node taints prevent VM scheduling - -**Details**: -- Found 3 nodes with taint: `virtualization=true:NoSchedule` -- VM spec does not include matching tolerations -- This prevents VM scheduling on virtualization-dedicated nodes -``` - -**Proposed Workaround**: -```bash -oc patch vm web-server -n vms --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' -``` - -**User Confirmation Required**: -``` -How would you like to proceed? -- "apply workaround" - I'll patch the VM with tolerations -- "manual" - I'll provide instructions for you to apply manually -- "cancel" - Delete the VM and abort creation -``` - -**After Execution**: -```markdown -## ✓ Workaround Applied Successfully - -**Action**: Added tolerations for taint `virtualization=true:NoSchedule` -**New Status**: Stopped (VM can now be scheduled) - -**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. -Future VMs in this cluster will need the same fix until the tool is enhanced. -``` - ---- - -## VM Status Reference - -### Status Values - -| Status | Meaning | Action Required | -|--------|---------|-----------------| -| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | -| `Running` | VM is running | Normal | -| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | -| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | -| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | -| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | -| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | -| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | -| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | -| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | - -### Checking VM Status - -```bash -# Get printable status -oc get vm -n -o jsonpath='{.status.printableStatus}' - -# Get detailed status and conditions -oc get vm -n -o jsonpath='{.status}' | jq - -# Watch status changes in real-time -oc get vm -n -w -``` - ---- - -## Best Practices for Agents - -When implementing diagnostic workflows: - -1. **Always verify VM status** after creation (wait 5-10 seconds first) -2. **Consult this document** when encountering error status values -3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) -4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) -5. **Respect human-in-the-loop** for all VM modifications -6. **Document temporary workarounds** and their limitations clearly -7. **Suggest filing issues** for missing MCP tool features - -### Document Consultation Pattern - -```markdown -**Document Consultation** (REQUIRED): -1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes -2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." -``` - ---- - -## Known MCP Tool Limitations - -### vm_create tool - -**Currently Supported**: -- ✓ Namespace, name (required) -- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) -- ✓ Size hints (small, medium, large) -- ✓ Storage size -- ✓ Autostart flag -- ✓ Networks (Multus NetworkAttachmentDefinitions) -- ✓ Performance family (u1, o1, c1, m1) -- ✓ Instance type, preference - -**Not Currently Supported** (requires workarounds): -- ✗ Tolerations (for node taints) -- ✗ Node selectors -- ✗ Affinity/anti-affinity rules -- ✗ Resource requests/limits (beyond instance type) -- ✗ Custom labels/annotations -- ✗ SSH keys injection -- ✗ Cloud-init user data - -**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. - -**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues - ---- - -## Additional Resources - -- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) -- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) -- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) -- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) - ---- - -[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/scheduling-errors.md deleted file mode 100644 index e47c8c4f..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/scheduling-errors.md +++ /dev/null @@ -1,417 +0,0 @@ ---- -title: VM Scheduling Errors -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 -tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] -semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] -use_cases: [vm-creation, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Scheduling Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**When to use this document**: -- VM shows status `ErrorUnschedulable` after creation or start attempt -- VM events mention scheduling failures, taints, resources, or node selectors - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -## ErrorUnschedulable - -**Symptom**: VM shows status `ErrorUnschedulable` after creation - -**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**Possible Causes**: - -### 1. Node Taints (Most Common) - -Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM events for scheduling failures**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `involvedObject.name` == "" and look for messages like: -- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= -``` - -**2. Check node taints in the cluster**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' -``` - -**Common Taint Patterns**: -- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule -- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes -- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads - -**Solution - Add Tolerations to VM**: - -The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. - -**Workaround (post-creation using MCP Tools)**: - -**Step 1**: Get current VM spec - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Step 2**: Modify the returned JSON to add tolerations - -Add to `.spec.template.spec.tolerations`: -```json -{ - "tolerations": [ - { - "key": "virtualization", - "operator": "Equal", - "value": "true", - "effect": "NoSchedule" - } - ] -} -``` - -**Step 3**: Update VM with modified spec - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters**: -```json -{ - "resource": "" -} -``` - -Pass the complete modified VM resource as YAML or JSON string. - -**Step 4**: Verify tolerations were added - -Use `resources_get` again and check `.spec.template.spec.tolerations` in response. - -**Step 5**: Check if VM status improved - -Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. - -**CLI Fallback** (if MCP patch is too complex): -```bash -# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' - -# Verify tolerations -oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq - -# Check status -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**Example - Multiple Tolerations**: -```bash -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" - - key: "dedicated" - operator: "Equal" - value: "virt-workloads" - effect: "NoSchedule" -' -``` - -**Toleration Operators**: -- `Equal` - Key and value must match exactly -- `Exists` - Only key must exist (ignores value) - -**Toleration Effects**: -- `NoSchedule` - Don't schedule new pods (existing pods continue) -- `PreferNoSchedule` - Avoid scheduling if possible -- `NoExecute` - Don't schedule AND evict existing pods - -**Alternative Solutions**: -1. **Remove node taints** (if you have cluster-admin access): - ```bash - oc adm taint nodes virtualization=true:NoSchedule- - ``` - -2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits - -3. **File enhancement request** - Request tolerations support in openshift-mcp-server: - https://github.com/openshift/openshift-mcp-server/issues - ---- - -### 2. Insufficient Resources - -Not enough CPU, memory, or storage available on any node. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM resource requests**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' -``` - -**2. Check node resource availability**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. - -Alternatively, use `nodes_top` MCP tool for current resource usage. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe nodes | grep -A 5 "Allocated resources" -``` - -**3. Look for VM events mentioning "Insufficient"**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep "Insufficient" -``` - -**Example Event**: -``` -0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. -``` - -**Solutions** (Use MCP Tools First): - -1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) -2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill -3. **Delete unused VMs** - Use vm-delete skill to free up resources -4. **Check resource quotas**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters for quota**: - ```json - { - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" - } - ``` - - **Parameters for limit range**: - ```json - { - "apiVersion": "v1", - "kind": "LimitRange", - "namespace": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - oc describe limitrange -n - ``` - ---- - -### 3. Node Selector Mismatch - -VM requires specific node labels that don't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM node selector requirements**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.nodeSelector` to see required node labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' -``` - -**2. List available node labels**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.metadata.labels` for available labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes --show-labels -``` - -**3. Check if any nodes match the selector**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node", - "labelSelector": "=" -} -``` - -Should return at least one node with matching labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -l = -``` - -**Solutions** (Use MCP Tools First): - -**Option 1: Remove node selector from VM** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get current VM using `resources_get` (diagnostic step 1) -2. Remove `.spec.template.spec.nodeSelector` field -3. Update VM using `resources_create_or_update` with modified JSON - -**CLI Fallback** (JSON patch easier via CLI): -Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" -```bash -oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' -``` - -**Option 2: Add label to nodes** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get node using `resources_get` -2. Add label to `.metadata.labels` -3. Update node using `resources_create_or_update` - -⚠️ **Note**: Node labeling typically requires cluster admin privileges. - -**CLI Fallback** (simpler via CLI): -Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" -```bash -oc label node = -``` - ---- - -[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/storage-errors.md deleted file mode 100644 index 16ba584d..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/docs/troubleshooting/storage-errors.md +++ /dev/null @@ -1,1011 +0,0 @@ ---- -title: VM Storage Errors -category: kubevirt -sources: - - title: KubeVirt CDI - DataVolumes - url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ - date_accessed: 2026-02-17 - - title: Kubernetes Persistent Volumes - url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ - date_accessed: 2026-02-17 -tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] -semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] -use_cases: [vm-creation, vm-deletion, vm-cloning] -related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] -last_updated: 2026-02-17 ---- - -# VM Storage Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. - -**When to use this document**: -- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` -- Storage deletion fails after VM deletion -- DataVolume cloning operations fail -- PVC provisioning issues - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### ErrorDataVolumeNotReady - -**Symptom**: VM shows status `ErrorDataVolumeNotReady` - -**Description**: The DataVolume (persistent storage) backing the VM is not ready. - -**Possible Causes**: - -#### 1. DataVolume Still Provisioning - -Storage provisioning takes time, especially for large disks or when importing images. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Get detailed DataVolume information**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` and `.status.conditions` for provisioning details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o json -``` - -**3. Check PVC (PersistentVolumeClaim) bound status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Check `.status.phase` for each PVC (should be `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. - -#### 2. Storage Class Not Found - -The requested storage class doesn't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List available storage classes**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" -} -``` - -Review the list of available storage classes (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -``` - -**2. Check DataVolume's requested storage class**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.spec.pvc.storageClassName` in the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' -``` - -**Solution**: -1. Use a valid storage class from the cluster -2. Recreate VM with correct storage class parameter - -#### 3. Insufficient Storage Quota - -Namespace has insufficient storage quota to provision the PVC. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check resource quotas**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**2. Check storage usage**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase -``` - -**Solution**: -1. Request quota increase from cluster admin -2. Delete unused PVCs to free quota -3. Reduce VM storage size - ---- - -### ErrorPvcNotFound - -**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List PVCs in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review the list of available PVCs (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check VM's PVC references**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' -``` - -**Solution**: -- Wait for DataVolume to create the PVC -- Manually create missing PVC -- Fix VM spec to reference correct PVC name - ---- - - ---- - -### Storage Deletion Failures - -**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace - -**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. - -**Possible Causes**: -- PVC still bound to active PersistentVolume with `Retain` policy -- DataVolume still being referenced by another resource -- CDI (Containerized Data Importer) controller issues -- Storage class retention policy preventing deletion -- Finalizers on PVC/DataVolume blocking cleanup -- PVC still mounted by a pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check PVC status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check specific PVC phase**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Released` or `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o jsonpath='{.status.phase}' -``` - -**3. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**4. Check what's using the PVC**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' -``` - -**5. Check PVC finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. - -**6. Check DataVolume finalizers**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.metadata.finalizers}' -``` - -**7. Check PV reclaim policy**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume" -} -``` - -Filter results for PV where `.spec.claimRef.name` matches ``. - -To get specific PV policy: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume", - "name": "" -} -``` - -Check `.spec.persistentVolumeReclaimPolicy`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pv | grep -oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' -``` - -**Common Finalizer Patterns**: -- `kubernetes.io/pvc-protection` - Protects PVC while in use -- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer - -**Solutions** (Use MCP Tools First): - -1. **Delete DataVolume first, then PVC**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete DataVolume first (often blocks PVC deletion): - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Wait a few seconds, then delete PVC: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete datavolume -n - oc delete pvc -n - ``` - -2. **Check for pods still using PVC**: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "" - } - ``` - - Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. - - Then delete the pods using `pods_delete`: - - **Parameters**: - ```json - { - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' - oc delete pod -n - ``` - -3. **Force delete PVC** (if safe to do so): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete pvc -n --grace-period=0 --force - ``` - -4. **Remove finalizers from PVC** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PVC using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update PVC using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Remove finalizers from DataVolume** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current DataVolume using `resources_get` (diagnostic step 6) - 2. Remove items from `.metadata.finalizers` array - 3. Update DataVolume using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Change PV reclaim policy** (if PV has Retain policy): - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PV policy using `resources_get` (diagnostic step 7) - 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` - 3. Update PV using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" - ```bash - oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' - oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' - ``` - -**Storage Quota Check** (Use MCP Tools First): - -After deletion, verify storage quota is freed: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters for quota check**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.used` to verify storage quota is freed. - -**Parameters for PVC verification**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -oc get pvc -n -``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Confirm PVC is deleted: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Confirm DataVolume is deleted: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -# Should return: Error from server (NotFound) - -oc get datavolume -n -# Should return: Error from server (NotFound) -``` - ---- - -### DataVolume Cloning Failures - -**Symptom**: VM clone created successfully but DataVolume clone operation fails - -**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. - -**Possible Causes**: -- CSI driver doesn't support volume cloning -- Source PVC storage class incompatible with cloning -- Cross-namespace cloning not permitted by storage backend -- Insufficient storage quota in target namespace -- Source PVC not in `Bound` state -- Storage class doesn't have volume cloning enabled -- CDI (Containerized Data Importer) controller issues - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume clone status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and `.metadata.name`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Check DataVolume events for errors**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe datavolume -n -``` - -**3. Check DataVolume phase**: - -Use `resources_get` from step 1, extract `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -``` - -**4. Check if storage class supports cloning**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass", - "name": "" -} -``` - -Review the full YAML output for cloning-related configurations. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -o yaml | grep -A 5 -i clone -``` - -**5. Check CSI driver capabilities**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "CSIDriver" -} -``` - -Review `.items[].metadata.name` for available CSI drivers. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get csidriver -``` - -**6. Check source PVC status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Bound` for cloning to work). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**7. Check target namespace storage quota**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**8. Check CDI controller logs**: - -**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) - -First, list pods in openshift-cnv namespace: - -**Parameters for pods_list_in_namespace**: -```json -{ - "namespace": "openshift-cnv", - "labelSelector": "app.kubernetes.io/component=cdi-deployment" -} -``` - -Then get logs using `pods_log`: - -**Parameters**: -```json -{ - "namespace": "openshift-cnv", - "name": "", - "tail": 100 -} -``` - -**CLI Fallback** (if MCP unavailable or easier via CLI): -```bash -oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') -``` - -**Common Error Messages**: -- `"volume cloning is not supported"` - CSI driver lacks clone capability -- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage -- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace -- `"insufficient quota"` - Target namespace lacks storage quota -- `"source PVC not bound"` - Source PVC must be in Bound state for cloning -- `"StorageClass does not support cloning"` - Storage class configuration issue - -**Solutions** (Use MCP Tools First): - -1. **Check storage class clone support**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" - } - ``` - - For each storage class in `.items[]`, check: - - `.metadata.name` (storage class name) - - `.provisioner` (CSI driver) - - Storage classes using CSI drivers typically support cloning. Look for provisioners like: - - `csi.ovirt.org` (oVirt CSI) - - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) - - `ebs.csi.aws.com` (AWS EBS CSI) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner - ``` - -2. **Verify source PVC is bound**: - - Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pvc -n -o jsonpath='{.status.phase}' - ``` - -3. **Check target namespace quota**: - - Use `resources_list` from diagnostic step 7 to check quota. - - If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - ``` - -4. **Use snapshot-based cloning** (alternative method): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Step 1**: Create VolumeSnapshot of source PVC - - **Parameters**: - ```json - { - "apiVersion": "snapshot.storage.k8s.io/v1", - "kind": "VolumeSnapshot", - "metadata": { - "name": "-snapshot", - "namespace": "" - }, - "spec": { - "source": { - "persistentVolumeClaimName": "" - } - } - } - ``` - - **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) - - **Step 3**: Create new DataVolume from snapshot - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "metadata": { - "name": "-rootdisk", - "namespace": "" - }, - "spec": { - "source": { - "snapshot": { - "name": "-snapshot", - "namespace": "" - } - }, - "storage": { - "resources": { - "requests": { - "storage": "50Gi" - } - }, - "storageClassName": "" - } - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" - ```bash - cat <-snapshot - namespace: - spec: - source: - persistentVolumeClaimName: - EOF - - oc get volumesnapshot -snapshot -n - - cat <-rootdisk - namespace: - spec: - source: - snapshot: - name: -snapshot - namespace: - storage: - resources: - requests: - storage: 50Gi - storageClassName: - EOF - ``` - -5. **Use "new empty storage" option** (vm-clone skill): - - If cloning isn't supported, create VM with empty storage - - Manually copy data if needed - -6. **Cross-namespace cloning workaround**: - - Some storage backends require snapshot for cross-namespace cloning - - Create snapshot in source namespace, restore in target namespace (see solution 4 above) - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Check DataVolume reached Succeeded phase: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should return `Succeeded`). - -Check PVC was created and bound: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "-rootdisk" -} -``` - -Check `.status.phase` (should return `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -# Should return: Succeeded - -oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' -# Should return: Bound -``` - -**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/skills/vm-snapshot-list/SKILL.md b/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/skills/vm-snapshot-list/SKILL.md deleted file mode 100644 index 3f199f8f..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-list/environment/skills/vm-snapshot-list/SKILL.md +++ /dev/null @@ -1,402 +0,0 @@ ---- -name: vm-snapshot-list -description: | - List virtual machine snapshots across namespaces with status, age, and recovery information. - - Use when: - - "List snapshots for VM [name]" - - "Show snapshots in namespace [name]" - - "What snapshots exist for [vm]?" - - Read-only operation - no user confirmation required. - - NOT for creating/deleting snapshots (use vm-snapshot-create/delete instead). - -model: inherit -color: cyan ---- - -# /vm-snapshot-list Skill - -List virtual machine snapshots in OpenShift Virtualization. This read-only skill displays snapshot information including status, age, size, and recovery options. - -## Prerequisites - -**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools**: -- `resources_list` (from openshift-virtualization) - List VirtualMachineSnapshot resources -- `resources_get` (from openshift-virtualization) - Get snapshot details - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster (>= 4.19) -- OpenShift Virtualization operator installed -- ServiceAccount with RBAC permissions to list VirtualMachineSnapshot resources - -### Prerequisite Verification - -**Before executing, verify MCP server availability:** - -1. **Check MCP Server Configuration** - - Verify `openshift-virtualization` exists in `.mcp.json` - - If missing → Report to user with setup instructions - -2. **Check Environment Variables** - - Verify `KUBECONFIG` is set (check presence only, never expose value) - - If missing → Report to user - -**Human Notification Protocol:** - -When prerequisites fail: - -``` -❌ Cannot execute vm-snapshot-list: MCP server 'openshift-virtualization' is not available - -📋 Setup Instructions: -1. Add openshift-virtualization to .mcp.json -2. Set KUBECONFIG environment variable -3. Restart Claude Code to reload MCP servers - -🔗 Documentation: https://github.com/openshift/openshift-mcp-server -``` - -## When to Use This Skill - -**Trigger this skill when:** -- User wants to list available snapshots for recovery -- User wants to see snapshot status and age -- User wants to verify snapshot existence before restore -- User wants to identify old snapshots for deletion - -**User phrases that trigger this skill:** -- "List all snapshots for web-server VM" -- "Show snapshots in namespace production" -- "What snapshots exist?" -- "Display VM snapshots" - -**Do NOT use this skill when:** -- User wants to create a snapshot → Use `vm-snapshot-create` skill -- User wants to restore from snapshot → Use `vm-snapshot-restore` skill -- User wants to delete snapshots → Use `vm-snapshot-delete` skill - -## Workflow - -### Step 1: Gather Information - -**Required Information from User:** -1. **Namespace** - Namespace to list snapshots from -2. **VM Name** (Optional) - Filter snapshots by specific VM - -If user doesn't provide namespace, ask for it. - -### Step 2: List Snapshots - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters** (with VM filter using label selector): -```json -{ - "apiVersion": "snapshot.kubevirt.io/v1beta1", - "kind": "VirtualMachineSnapshot", - "namespace": "", - "labelSelector": "vm.kubevirt.io/name=" -} -``` - -**Parameters** (all snapshots in namespace): -```json -{ - "apiVersion": "snapshot.kubevirt.io/v1beta1", - "kind": "VirtualMachineSnapshot", - "namespace": "" -} -``` - -**Note**: The label selector `vm.kubevirt.io/name=` may not always exist. If no results are returned, fall back to listing all snapshots and filtering by checking `spec.source.name` field in the results. - -**Expected Output**: List of VirtualMachineSnapshot resources - -**Parse each snapshot to extract**: -- `metadata.name` - Snapshot name -- `metadata.namespace` - Namespace -- `metadata.creationTimestamp` - Creation time -- `spec.source.name` - VM name -- `status.phase` - Status (InProgress, Succeeded, Failed) -- `status.readyToUse` - Ready for restore (true/false) - -**Error Handling**: -- If namespace not found → Report error -- If permission denied → Report RBAC error -- If no snapshots found → Report "No snapshots found" - -### Step 3: Report Snapshot List - -**If snapshots found:** - -```markdown -## VM Snapshots - -**Namespace**: `` - -**VM**: `` - - -### Available Snapshots - -| Snapshot Name | VM Name | Status | Created | Age | ReadyToUse | -|---------------|---------|--------|---------|-----|------------| -| `pre-upgrade-snapshot` | `database-01` | Succeeded ✓ | 2024-01-15 10:30 | 2 days | true | -| `backup-snapshot` | `database-01` | Succeeded ✓ | 2024-01-10 08:00 | 7 days | true | -| `test-snapshot` | `web-server` | Succeeded ✓ | 2024-01-14 14:20 | 3 days | true | - -**Total Snapshots**: 3 - ---- - -### Snapshot Details - -**Snapshot: `pre-upgrade-snapshot`** -- **VM**: `database-01` -- **Status**: Succeeded ✓ -- **Created**: 2024-01-15 10:30:00 UTC -- **Age**: 2 days -- **Ready to Use**: true - -**Snapshot: `backup-snapshot`** -- **VM**: `database-01` -- **Status**: Succeeded ✓ -- **Created**: 2024-01-10 08:00:00 UTC -- **Age**: 7 days -- **Ready to Use**: true - -**Snapshot: `test-snapshot`** -- **VM**: `web-server` -- **Status**: Succeeded ✓ -- **Created**: 2024-01-14 14:20:00 UTC -- **Age**: 3 days -- **Ready to Use**: true - ---- - -### Actions - -**To restore from a snapshot:** -``` -"Restore VM from snapshot " -``` - -**To delete a snapshot:** -``` -"Delete snapshot " -``` - -**To create a new snapshot:** -``` -"Create snapshot of VM " -``` -``` - -**If no snapshots found:** - -```markdown -## VM Snapshots - -**Namespace**: `` - -**VM**: `` - - -**No snapshots found.** - - -No snapshots exist for VM `` in namespace ``. - - -No snapshots exist in namespace ``. - - -**To create a snapshot:** -``` -"Create snapshot of VM " -``` -``` - -## Common Issues - -### Issue 1: Permission Denied - -**Error**: "Forbidden: User lacks permissions to list virtualmachinesnapshots" - -**Cause**: Missing RBAC permissions for listing snapshots. - -**Solution:** -1. Check permissions: `oc auth can-i list virtualmachinesnapshots -n ` -2. Contact cluster admin to grant list/get permissions for virtualmachinesnapshots -3. Try listing in a different namespace where you have permissions - -### Issue 2: No Snapshots Found - -**Error**: "No snapshots exist in namespace ``" - -**Cause**: Namespace has no snapshots, or wrong namespace. - -**Solution:** -1. Verify correct namespace name -2. List snapshots without VM filter to see all snapshots -3. Check other namespaces: Use `namespaces_list` to see available namespaces -4. Check if snapshots were recently deleted: Use `events_list` in namespace - -### Issue 3: Snapshot Shows Failed Status - -**Error**: Snapshot listed but `status.phase: Failed` or `readyToUse: false` - -**Cause**: Snapshot creation failed due to storage issues, hot-plugged volumes, or missing VolumeSnapshotClass. - -**Solution:** -1. Get snapshot details: Use `resources_get` to check `status.conditions` for error messages -2. Check cluster events: Use `events_list` for snapshot-related errors -3. Common fixes: - - VolumeSnapshotClass missing: Contact cluster admin - - Hot-plugged volumes: Stop VM, persist volumes, recreate snapshot - - Failed permanently: Delete and recreate snapshot - -## Dependencies - -### Required MCP Servers -- `openshift-virtualization` - OpenShift MCP server with kubevirt toolset - -### Required MCP Tools -- `resources_list` (from openshift-virtualization) - List Kubernetes resources by type - - Used for: Listing VirtualMachineSnapshot resources - - Parameters: apiVersion, kind, namespace, labelSelector (optional) - - Source: https://github.com/openshift/openshift-mcp-server - -### Related Skills -- `vm-snapshot-create` - Create VM snapshots -- `vm-snapshot-restore` - Restore VMs from snapshots -- `vm-snapshot-delete` - Delete VM snapshots -- `vm-inventory` - List VMs before creating snapshots - -### Reference Documentation -- [OpenShift Virtualization Snapshots](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt-managing-vm-snapshots) -- [KubeVirt VM Snapshots](https://kubevirt.io/user-guide/operations/snapshot_restore_api/) - -## Security Considerations - -- **RBAC Enforcement**: Requires list permissions for VirtualMachineSnapshot resources -- **Namespace Isolation**: Only lists snapshots within specified namespace -- **Read-Only Operation**: No modifications or destructive actions -- **KUBECONFIG Security**: Credentials never exposed in output - -## Example Usage - -### Example 1: List Snapshots for Specific VM - -``` -User: "List all snapshots for database-01" - -Agent: "Which namespace is database-01 in?" - -User: "production" - -Agent: [Invokes vm-snapshot-list skill] - [Lists snapshots with VM filter] - -## VM Snapshots - -**Namespace**: `production` -**VM**: `database-01` - -### Available Snapshots - -| Snapshot Name | VM Name | Status | Created | Age | ReadyToUse | -|---------------|---------|--------|---------|-----|------------| -| `database-01-pre-upgrade` | `database-01` | Succeeded ✓ | 2024-01-15 10:30 | 2 hours | true | -| `database-01-daily-backup` | `database-01` | Succeeded ✓ | 2024-01-14 02:00 | 1 day | true | - -**Total Snapshots**: 2 - ---- - -### Snapshot Details - -**Snapshot: `database-01-pre-upgrade`** -- **VM**: `database-01` -- **Status**: Succeeded ✓ -- **Created**: 2024-01-15 10:30:00 UTC -- **Age**: 2 hours -- **Ready to Use**: true - -**Snapshot: `database-01-daily-backup`** -- **VM**: `database-01` -- **Status**: Succeeded ✓ -- **Created**: 2024-01-14 02:00:00 UTC -- **Age**: 1 day -- **Ready to Use**: true - ---- - -### Actions - -**To restore from a snapshot:** -"Restore VM database-01 from snapshot " - -**To delete a snapshot:** -"Delete snapshot " - -**To create a new snapshot:** -"Create snapshot of VM database-01" -``` - -### Example 2: List All Snapshots in Namespace - -``` -User: "Show me all snapshots in namespace production" - -Agent: [Invokes vm-snapshot-list skill] - [Lists all snapshots without VM filter] - -## VM Snapshots - -**Namespace**: `production` - -### Available Snapshots - -| Snapshot Name | VM Name | Status | Created | Age | ReadyToUse | -|---------------|---------|--------|---------|-----|------------| -| `database-01-pre-upgrade` | `database-01` | Succeeded ✓ | 2024-01-15 10:30 | 2 hours | true | -| `database-01-daily-backup` | `database-01` | Succeeded ✓ | 2024-01-14 02:00 | 1 day | true | -| `web-server-backup` | `web-server` | Succeeded ✓ | 2024-01-13 03:00 | 2 days | true | - -**Total Snapshots**: 3 - -VMs with snapshots: `database-01` (2), `web-server` (1) -``` - -### Example 3: No Snapshots Found - -``` -User: "List snapshots for test-vm" - -Agent: "Which namespace is test-vm in?" - -User: "dev" - -Agent: [Invokes vm-snapshot-list skill] - [Lists snapshots - none found] - -## VM Snapshots - -**Namespace**: `dev` -**VM**: `test-vm` - -**No snapshots found.** - -No snapshots exist for VM `test-vm` in namespace `dev`. - -**To create a snapshot:** -"Create snapshot of VM test-vm" -``` diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/.ai-index/semantic-index.json deleted file mode 100644 index 2512c2a3..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/.ai-index/semantic-index.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "index_version": "1.0.0", - "last_updated": "2026-02-17", - "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", - - "documents": [ - { - "file": "INDEX.md", - "title": "VM Troubleshooting Guide - Index", - "category": "navigation", - "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], - "use_cases": ["navigation", "tool-reference", "general-guidance"], - "token_estimate": 450, - "skills": ["all"] - }, - { - "file": "scheduling-errors.md", - "title": "VM Scheduling Errors", - "category": "scheduling", - "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], - "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], - "errors_covered": [ - "ErrorUnschedulable - Node Taints", - "ErrorUnschedulable - Insufficient Resources", - "ErrorUnschedulable - Node Selector Mismatch" - ], - "token_estimate": 820, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "storage-errors.md", - "title": "VM Storage Errors", - "category": "storage", - "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], - "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], - "errors_covered": [ - "ErrorDataVolumeNotReady - DataVolume Still Provisioning", - "ErrorDataVolumeNotReady - Storage Class Not Found", - "ErrorDataVolumeNotReady - Insufficient Storage Quota", - "ErrorPvcNotFound", - "Storage Deletion Failures", - "DataVolume Cloning Failures" - ], - "token_estimate": 1350, - "skills": ["vm-creator", "vm-delete", "vm-clone"] - }, - { - "file": "lifecycle-errors.md", - "title": "VM Lifecycle Errors", - "category": "lifecycle", - "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], - "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], - "errors_covered": [ - "VM Stuck in Terminating State", - "VM Won't Start (Non-Scheduling Issues)", - "VM Won't Stop" - ], - "token_estimate": 1150, - "skills": ["vm-delete", "vm-lifecycle-manager"] - }, - { - "file": "runtime-errors.md", - "title": "VM Runtime Errors", - "category": "runtime", - "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], - "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], - "errors_covered": [ - "CrashLoopBackOff" - ], - "token_estimate": 850, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "network-errors.md", - "title": "VM Network Errors", - "category": "networking", - "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], - "use_cases": ["vm-creation", "network-troubleshooting"], - "errors_covered": [ - "Network Attachment Failures" - ], - "token_estimate": 580, - "skills": ["vm-creator"] - } - ], - - "skill_to_docs_mapping": { - "vm-creator": { - "primary_docs": ["scheduling-errors.md", "storage-errors.md"], - "secondary_docs": ["runtime-errors.md", "network-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3650, - "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" - }, - "vm-delete": { - "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 2950, - "description": "VM deletion failures - stuck terminating, storage cleanup" - }, - "vm-clone": { - "primary_docs": ["storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 1800, - "description": "VM cloning failures - DataVolume cloning, storage provisioning" - }, - "vm-lifecycle-manager": { - "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], - "secondary_docs": ["runtime-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3270, - "description": "VM start/stop failures - lifecycle management, scheduling" - }, - "vm-inventory": { - "primary_docs": [], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 450, - "description": "General guidance for status interpretation" - } - }, - - "error_to_docs_mapping": { - "ErrorUnschedulable": "scheduling-errors.md", - "ErrorDataVolumeNotReady": "storage-errors.md", - "ErrorPvcNotFound": "storage-errors.md", - "Terminating": "lifecycle-errors.md", - "CrashLoopBackOff": "runtime-errors.md", - "NetworkAttachmentFailed": "network-errors.md" - }, - - "cross_references": { - "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], - "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], - "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], - "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], - "network-errors.md": ["scheduling-errors.md"] - }, - - "progressive_disclosure_strategy": { - "description": "Load only the minimum documentation needed for current task", - "pattern": "error_type → primary_doc → cross_references (if needed)", - "token_savings": "29% average reduction vs single-file (based on rh-sre data)", - "navigation_overhead_reduction": "85% vs single-file navigation" - } -} diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/INDEX.md b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/INDEX.md deleted file mode 100644 index a2e76963..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/INDEX.md +++ /dev/null @@ -1,332 +0,0 @@ ---- -title: VM Troubleshooting Guide - Index -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 - - title: OpenShift Virtualization - Virtual Machine Status - url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html - date_accessed: 2026-02-06 - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] -semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] -use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] -last_updated: 2026-02-17 ---- - -# VM Troubleshooting Guide - Index - -## Overview - -This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: - -- **Creation**: Scheduling failures, storage provisioning issues -- **Lifecycle**: Start/stop failures, stuck states -- **Deletion**: Resources stuck in Terminating, storage cleanup failures -- **Cloning**: DataVolume cloning errors, cross-namespace issues -- **Networking**: Secondary network attachment failures -- **Runtime**: CrashLoopBackOff, guest OS boot failures - -This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. - ---- - -## 🗂️ Troubleshooting Categories - -The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: - -### 1. [Scheduling Errors](scheduling-errors.md) -**When to use**: VM fails to schedule on any node - -**Errors covered**: -- ErrorUnschedulable - Node Taints -- ErrorUnschedulable - Insufficient Resources -- ErrorUnschedulable - Node Selector Mismatch - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 2. [Storage Errors](storage-errors.md) -**When to use**: VM has storage provisioning, deletion, or cloning issues - -**Errors covered**: -- ErrorDataVolumeNotReady (all 3 subsections) -- ErrorPvcNotFound -- Storage Deletion Failures -- DataVolume Cloning Failures - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### 3. [Lifecycle Errors](lifecycle-errors.md) -**When to use**: VM has start/stop/termination issues - -**Errors covered**: -- VM Stuck in Terminating State -- VM Won't Start (Non-Scheduling Issues) -- VM Won't Stop - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### 4. [Runtime Errors](runtime-errors.md) -**When to use**: VM repeatedly crashes or fails at runtime - -**Errors covered**: -- CrashLoopBackOff - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 5. [Network Errors](network-errors.md) -**When to use**: VM has secondary network attachment failures - -**Errors covered**: -- Network Attachment Failures (Multus, SR-IOV) - -**Skills that use this**: vm-create - ---- - -## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last - -### Critical Principle: MCP-First Approach - -**All diagnostic procedures in this guide follow the MCP-first pattern**: - -``` -1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) -2. ⚠️ IF FAILS: Ask user permission to use CLI command -3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval -``` - -### Why MCP Tools First? - -- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources -- **Consistency**: Same interface across all operations -- **Better Error Handling**: MCP tools return structured errors -- **Audit Trail**: MCP tool usage is logged and trackable - -### Command Pattern Examples - -Throughout this guide, you'll see diagnostic steps formatted like this: - -**✅ CORRECT Pattern (MCP First)**: - -```markdown -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Extract**: `.metadata.finalizers` from returned JSON - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` -``` - -### Available MCP Tools - -The `openshift-virtualization` MCP server provides these tools: - -**Resource Operations**: -- `resources_get` - Get specific resource (replaces `oc get `) -- `resources_list` - List resources (replaces `oc get `) -- `resources_delete` - Delete resource (replaces `oc delete`) -- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) - -**Pod Operations**: -- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) -- `pods_get` - Get pod details (replaces `oc get pod`) -- `pods_log` - Get pod logs (replaces `oc logs`) -- `pods_exec` - Execute in pod (replaces `oc exec`) -- `pods_delete` - Delete pod (replaces `oc delete pod`) -- `pods_top` - Pod resource usage (replaces `oc top pods`) - -**Events & Monitoring**: -- `events_list` - List events (replaces `oc get events`) -- `nodes_top` - Node resource usage (replaces `oc top nodes`) -- `nodes_log` - Node logs (replaces `oc adm node-logs`) -- `nodes_stats_summary` - Detailed node stats - -**VM Operations** (KubeVirt toolset): -- `vm_create` - Create VMs -- `vm_lifecycle` - Start/stop/restart VMs - -**Namespaces**: -- `namespaces_list` - List namespaces (replaces `oc get namespaces`) -- `projects_list` - List OpenShift projects (replaces `oc get projects`) - -### When CLI Commands Are Required - -Some operations have **NO MCP equivalent** and require CLI: - -- `virtctl` commands (console, VNC access) -- `oc debug node` (node debugging) -- `oc auth can-i` (permission checks) -- `oc adm taint` (node taint management) -- Complex JSON patch operations - -For these, the guide will note: **"CLI Only - No MCP equivalent"** - -### Quick Reference: CLI → MCP Mapping - -| CLI Command | MCP Tool Equivalent | -|-------------|---------------------| -| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | -| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | -| `oc logs -n ` | `pods_log` with name="", namespace="" | -| `oc get events -n ` | `events_list` with namespace="" | -| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | - -**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. - -### How to Read Diagnostic Sections - -Each error section includes: -1. **Symptom** - What you observe -2. **Description** - What's happening -3. **Possible Causes** - Why it's happening -4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback -5. **Solutions** - **MCP tools first**, then CLI fallback -6. **Verification** - **MCP tools first**, then CLI fallback - -**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. - ---- - -## 🔍 Quick Navigation by Skill - -**vm-create**: -- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics -- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning -- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff -- [Network Errors](network-errors.md) - Network attachment failures - -**vm-delete**: -- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state -- [Storage Errors](storage-errors.md) - Storage deletion failures - -**vm-clone**: -- [Storage Errors](storage-errors.md) - DataVolume cloning failures - -**vm-lifecycle-manager**: -- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop -- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling - -**vm-inventory**: -- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed - ---- - -## 📊 Documentation Coverage & Maintenance - -### Current Coverage - -This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: - -- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) -- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures -- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop -- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) -- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems - -**Total errors documented**: 12 error types across 6 categories - ---- - -### Encountering Undocumented Errors - -#### For AI Agents (Claude Code) - -If you encounter an error **not documented** in the categories above: - -1. **Report to user** with all available details (error message, affected resources, namespace) -2. **Provide best-effort diagnostics** using MCP tools: - - `resources_get` to inspect resource status - - `pods_log` to check virt-launcher or compute container logs - - `events_list` to view Kubernetes events -3. **Suggest documentation update**: - ``` - ⚠️ This error is not yet documented in the troubleshooting guide. - - **Error**: - **Affected resource**: / - - I recommend adding this error to the troubleshooting guide: - - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file - - If it's a new error class → Create new category file in docs/troubleshooting/ - - Would you like me to help document this error for future reference? - ``` - -4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs - ---- - -#### For Human Contributors - -To document a new error: - -1. **Determine the category**: - - Scheduling issue → `scheduling-errors.md` - - Storage problem → `storage-errors.md` - - Lifecycle/start/stop → `lifecycle-errors.md` - - Runtime crash → `runtime-errors.md` - - Network issue → `network-errors.md` - - New error class → Create new category file (e.g., `security-errors.md`) - -2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): - - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback - - **Solutions**: MCP Tool approach → Implementation → CLI Fallback - - **Verification**: MCP Tool checks → Expected results → CLI Fallback - -3. **Update semantic index**: - - Edit `.ai-index/semantic-index.json` - - Add error to `error_to_docs_mapping` - - Update relevant skill's `primary_docs` or `secondary_docs` - - Increment token estimates if needed - -4. **Reference in skill files**: - - Update skill's Reference Documentation section - - Ensure skill's Document Consultation steps point to the right category - -5. **Update this INDEX.md**: - - Add error to the appropriate category section (lines 50-120) - - Update "Current Coverage" list above - ---- - -## 📚 Additional Resources - -- [rh-virt README](../../README.md) - MCP server setup and skill documentation -- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources -- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/lifecycle-errors.md deleted file mode 100644 index 6167fcc8..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/lifecycle-errors.md +++ /dev/null @@ -1,869 +0,0 @@ ---- -title: VM Lifecycle Errors -category: kubevirt -sources: - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 -tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] -semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] -use_cases: [vm-deletion, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Lifecycle Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM lifecycle issues including start/stop failures and stuck termination states. - -**When to use this document**: -- VM stuck in `Terminating` state -- VM won't start (runStrategy is Always but VM never reaches Running) -- VM won't stop (runStrategy is Halted but VM never reaches Stopped) - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### VM Stuck in Terminating State - -**Symptom**: VM shows status `Terminating` but deletion never completes - -**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). - -**Possible Causes**: -- Finalizers blocking deletion -- PVC/DataVolume still attached and preventing cleanup -- VirtualMachineInstance (VMI) not terminating properly -- Custom controllers or operators blocking deletion -- Stuck virt-launcher pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check finalizers on the VM**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` - -**2. Check if VMI still exists**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name containing "virt-launcher-". - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check events for deletion issues**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results where `involvedObject.name` == "" and sort by timestamp. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' -``` - -**5. Check VM deletion timestamp**: - -Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. - -If present, VM is in deletion process. If null, VM is not being deleted. - -**Common Finalizer Patterns**: -- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer -- `foregroundDeletion` - Waits for dependent objects to be deleted -- Custom finalizers from operators - -**Solutions** (Use MCP Tools First): - -1. **Wait for dependent resources** (safest approach): - - Use `resources_list` to check if PVCs, DataVolumes are still deleting - - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) - -2. **Force delete VMI** (if VMI is stuck): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable - requires explicit user permission): - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -3. **Force delete virt-launcher pod** (if pod is stuck): - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - # Find the virt-launcher pod name first - oc get pods -n | grep virt-launcher- - - # Then delete it - oc delete pod virt-launcher--xxx -n --force --grace-period=0 - ``` - -4. **Remove finalizers** (⚠️ dangerous - use only as last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` - 2. Remove items from `.metadata.finalizers` array - 3. Update VM using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch not easily done via MCP): - ```bash - # This operation is complex for MCP - may need CLI - oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Check for protection labels** (vm-delete skill specific): - - Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. - - If value is "true", the vm-delete skill refuses deletion (this is expected behavior). - -**Verification**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error if deletion successful. - -**CLI Fallback**: -```bash -oc get vm -n -# Should return: Error from server (NotFound) -``` - ---- - - ---- - -### VM Won't Start (Non-Scheduling Issues) - -**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state - -**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. - -**Possible Causes**: -- Guest OS kernel panic or boot failure -- Cloud-init configuration errors -- Missing or corrupted disk image -- Insufficient memory for guest OS to boot -- QEMU/KVM virtualization errors -- VirtualMachineInstance (VMI) creation failures -- virt-launcher pod crashes - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VMI (VirtualMachineInstance) status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and overall VMI state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**2. Check VMI conditions for errors**: - -Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. View virt-launcher pod logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -First, get pod name from step 3, then: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -Look for QEMU errors, memory allocation failures, device errors. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**5. Check virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. - -Alternatively, use `pods_get` to get full pod details: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**6. Access VM console to see guest OS boot messages**: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -Look for kernel panic, initramfs errors, filesystem errors. - -**7. Check VM events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep -A 20 "Events:" -``` - -**Common Error Patterns**: - -1. **Guest OS Boot Failure**: - - Console shows kernel panic - - Guest hangs at GRUB or boot loader - - Cloud-init errors during first boot - -2. **Insufficient Memory**: - - Guest OS kills processes due to OOM - - VMI logs show memory allocation errors - -3. **QEMU Crashes**: - - virt-launcher logs show QEMU segfaults - - VMI repeatedly restarts - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for kernel panic, initramfs errors, filesystem errors. - -2. **Check virt-launcher pod logs for QEMU errors**: - - Use `pods_log` from diagnostic step 4 to view logs. - - Look for: - - "qemu-system-x86_64: ..." errors - - Memory allocation failures - - Device errors - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx - ``` - -3. **Increase memory if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check VMI memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -4. **Verify disk image integrity**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check PVC is bound: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - If using container disk, verify image exists and is accessible (check VMI spec). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get pvc -n - ``` - -5. **Check cloud-init configuration** (if applicable): - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. - - Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq - ``` - -6. **Restart VMI** (soft reset): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete VMI (VM controller will recreate it): - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for new VMI to start (use `resources_get` to check status). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n -w - ``` - -7. **Check virtualization extensions** (KVM): - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - - # In debug shell: - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should eventually return `Running`). - -Check VMI is running: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should show `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should eventually return: Running - -oc get vmi -n -# Should show: Running -``` - ---- - -### VM Won't Stop - -**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state - -**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. - -**Possible Causes**: -- Guest OS not responding to ACPI shutdown signal -- virt-launcher pod stuck and not terminating -- VirtualMachineInstance (VMI) deletion blocked by finalizers -- Guest shutdown scripts hanging -- Filesystem sync issues in guest OS - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (might show `Stopping`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**2. Check VMI status and deletion timestamp**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check VMI events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vmi -n | grep -A 10 "Events:" -``` - -**5. Check VMI finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.metadata.finalizers}' -``` - -**6. Check if guest is responsive** (if VMI still exists): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Patterns**: -- VMI shows `deletionTimestamp` but never actually deletes -- virt-launcher pod in `Terminating` state -- VM runStrategy is `Halted` but printableStatus shows `Stopping` - -**Solutions** (Use MCP Tools First): - -1. **Wait for graceful shutdown** (default: 30 seconds): - - Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. - -2. **Force stop by deleting VMI**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - This is the standard way to force-stop a VM. - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n - # Should return: Error from server (NotFound) - ``` - -3. **Force delete VMI with grace period** (if VMI won't delete): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -4. **Force delete virt-launcher pod**: - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find the pod using `pods_list_in_namespace` from diagnostic step 3. - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - ⚠️ **Note**: For force deletion with grace period, use CLI fallback. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') - oc delete pod $POD_NAME -n --force --grace-period=0 - ``` - -5. **Remove VMI finalizers** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VMI using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update VMI using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Patch VM runStrategy directly** (ensure consistency): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` (diagnostic step 1) - 2. Set `.spec.runStrategy` to `"Halted"` - 3. Update VM using `resources_create_or_update` with modified JSON - - **CLI Fallback** (merge patch easier via CLI): - Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Stopped` or `Halted`). - -Verify VMI is gone: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Verify virt-launcher pod is gone: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for pods matching `virt-launcher-`. Should return no results. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Stopped or Halted - -oc get vmi -n -# Should return: Error from server (NotFound) - -oc get pods -n | grep virt-launcher- -# Should return: No resources found -``` - -**Prevention**: -- Ensure guest OS has ACPI support enabled -- Use proper shutdown commands in guest OS -- Avoid forceful stops unless necessary (can corrupt guest filesystem) - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/network-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/network-errors.md deleted file mode 100644 index 82fa313c..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/network-errors.md +++ /dev/null @@ -1,429 +0,0 @@ ---- -title: VM Network Errors -category: kubevirt -sources: - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] -semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] -use_cases: [vm-creation, network-troubleshooting] -related_docs: [INDEX.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Network Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. - -**When to use this document**: -- VM created successfully but secondary networks not attached -- NetworkAttachmentDefinition not found errors -- Multus CNI failures -- SR-IOV device attachment issues - -**Skills that use this**: vm-create - ---- - -### Network Attachment Failures - -**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working - -**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). - -**Possible Causes**: -- NetworkAttachmentDefinition doesn't exist in the namespace -- Multus CNI not installed or not configured on cluster -- Namespace mismatch (NAD in different namespace than VM) -- Interface name conflicts in VM spec -- Bridge/network configuration errors in NAD -- SR-IOV device not available (if using SR-IOV) - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List NetworkAttachmentDefinitions in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "" -} -``` - -Review `.items[].metadata.name` for available NADs. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -n -``` - -**2. List NetworkAttachmentDefinitions in all namespaces**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition" -} -``` - -Omit `namespace` parameter to list across all namespaces. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -A -``` - -**3. Check specific NetworkAttachmentDefinition**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "", - "name": "" -} -``` - -Review `.spec.config` for CNI configuration. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definition -n -o yaml -``` - -**4. Check VM network configuration**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.networks` to see network references. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq -``` - -**5. Check VM domain interfaces**: - -Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq -``` - -**6. Check virt-launcher pod network annotations** (shows actual attachments): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**7. Check Multus is installed**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "openshift-multus" -} -``` - -Should show Multus CNI pods running. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n openshift-multus -``` - -**8. Check for errors in virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n -``` - -**Common Error Messages**: -- `"network-attachment-definition not found"` - NAD doesn't exist in namespace -- `"multus CNI not configured"` - Multus not installed or misconfigured -- `"interface name conflict"` - Duplicate interface names in VM spec -- `"failed to add network"` - CNI plugin error (check NAD config) -- `"no available devices"` - SR-IOV device not available (if using SR-IOV) - -**Solutions** (Use MCP Tools First): - -1. **Verify NetworkAttachmentDefinition exists in correct namespace**: - - Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. - - If NAD is in different namespace, copy it to VM namespace: - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get NAD from source namespace using `resources_get` - 2. Modify `.metadata.namespace` to target namespace - 3. Create NAD in target namespace using `resources_create_or_update` - - **CLI Fallback** (stream processing easier via CLI): - Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" - ```bash - oc get network-attachment-definition -n -o yaml | \ - sed "s/namespace: /namespace: /" | \ - oc apply -f - - ``` - -2. **Create missing NetworkAttachmentDefinition**: - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - Example: Linux bridge network - - **Parameters**: - ```json - { - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "metadata": { - "name": "vlan100", - "namespace": "" - }, - "spec": { - "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" - ```bash - cat < - spec: - config: '{ - "cniVersion": "0.3.1", - "type": "bridge", - "bridge": "br1", - "vlan": 100, - "ipam": { - "type": "host-local", - "subnet": "192.168.100.0/24" - } - }' - EOF - ``` - -3. **Check Multus CNI installation**: - - Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. - - To check cluster network operator: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "config.openshift.io/v1", - "kind": "ClusterOperator" - } - ``` - - Filter for `network` operator. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n openshift-multus - oc get clusteroperators network - ``` - -4. **Fix interface name conflicts** (if VM has duplicate names): - - Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. - - Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. - - **CLI Fallback** (interactive edit easier via CLI): - Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' - oc edit vm -n - ``` - -5. **Validate NAD configuration syntax**: - - Use `resources_get` from diagnostic step 3, extract `.spec.config`. - - Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get network-attachment-definition -n -o jsonpath='{.spec.config}' - ``` - -6. **Check SR-IOV device availability** (if using SR-IOV networks): - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - List SR-IOV network node policies: - - **Parameters**: - ```json - { - "apiVersion": "sriovnetwork.openshift.io/v1", - "kind": "SriovNetworkNodePolicy", - "namespace": "openshift-sriov-network-operator" - } - ``` - - Check SR-IOV device plugin pods: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "openshift-sriov-network-operator" - } - ``` - - Filter for pods with "device-plugin" in name. - - Check available SR-IOV devices on node: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "Node", - "name": "" - } - ``` - - Review `.status.allocatable` for SR-IOV resources. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get sriovnetworknodepolicy -n openshift-sriov-network-operator - oc get pods -n openshift-sriov-network-operator | grep device-plugin - oc describe node | grep -A 10 "Allocatable:" - ``` - -7. **Recreate VM with corrected network configuration** (if needed): - - If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -After remediation, check virt-launcher pod network status: - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -Should show all attached networks with status. Example output: -```json -[ - { - "name": "openshift-sdn", - "interface": "eth0", - "ips": ["10.128.2.10"], - "default": true - }, - { - "name": "vlan100", - "interface": "net1", - "ips": ["192.168.100.5"] - } -] -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**Check from inside VM** (via console): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# In guest OS: -ip addr show -# Should show all network interfaces (eth0, net1, etc.) -``` - -**Common Network Types**: -- **Linux Bridge**: Layer 2 bridge for VLAN networks -- **SR-IOV**: High-performance direct device assignment -- **macvlan**: MAC-based VLAN for container networks -- **OVN-Kubernetes**: OpenShift native overlay network - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/runtime-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/runtime-errors.md deleted file mode 100644 index 804d3cbc..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/runtime-errors.md +++ /dev/null @@ -1,616 +0,0 @@ ---- -title: VM Runtime Errors -category: kubevirt -sources: - - title: Kubernetes Pod Lifecycle - url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ - date_accessed: 2026-02-17 -tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] -semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] -use_cases: [vm-creation, vm-lifecycle, diagnostics] -related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Runtime Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. - -**When to use this document**: -- VM or virt-launcher pod shows `CrashLoopBackOff` status -- virt-launcher pod repeatedly restarting -- Guest OS kernel panics on boot - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### CrashLoopBackOff - -**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting - -**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. - -**Possible Causes**: -- Guest OS kernel panic on boot -- Insufficient resources (memory/CPU) for guest OS -- Corrupted disk image or filesystem -- QEMU/libvirt crashes due to configuration errors -- Missing or incompatible device drivers in guest -- Resource limits too low for virt-launcher pod -- Virtualization features (KVM) not available on node - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check virt-launcher pod restart count**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# Look at RESTARTS column -``` - -**2. View recent crash logs** (previous container instance): - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "previous": true, - "tail": 100 -} -``` - -Look for QEMU errors, kernel panics, or segfaults. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n virt-launcher--xxx --previous -``` - -**3. Check current virt-launcher logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. - -**CLI Fallback** (if MCP unavailable or all containers needed): -```bash -oc logs -n virt-launcher--xxx --all-containers -``` - -**4. Check VMI conditions for crash details**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Extract `.status.conditions` for crash details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**5. Check pod events for crash reasons**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" -``` - -**6. Check pod resource limits**: - -**MCP Tool**: `pods_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -Extract `.spec.containers[0].resources` for resource limits. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq -``` - -**7. Check node kubelet logs for OOM kills**: - -⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -oc adm node-logs -u kubelet | grep -i oom -``` - -**8. Access guest console** (if VM briefly starts): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Crash Patterns**: - -1. **Guest Kernel Panic**: - - Console logs show kernel panic messages - - Guest crashes immediately after boot - - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" - -2. **OOM (Out of Memory)**: - - Pod killed with reason: `OOMKilled` - - Guest runs out of memory during boot or operation - - virt-launcher logs show memory allocation failures - -3. **QEMU Crash**: - - virt-launcher logs show QEMU segmentation fault - - Symptoms: "qemu-system-x86_64: terminated by signal" - - Configuration incompatibility or QEMU bug - -4. **Disk Image Corruption**: - - Guest cannot boot from disk - - Filesystem errors in guest console - - DataVolume import failed - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for kernel panic or boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for: - - Kernel panic messages - - Initramfs errors - - Filesystem mounting failures - - Missing device errors - -2. **Review virt-launcher crash logs**: - - Use `pods_log` with `previous: true` from diagnostic step 2. - - Look for: - - QEMU command line errors - - Device initialization failures - - Memory allocation errors - - Signal termination (SIGSEGV, SIGABRT) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx --previous - ``` - -3. **Check for OOM (Out of Memory) kills**: - - **MCP Tool**: `pods_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - Extract `.status.containerStatuses[0].lastState.terminated.reason`. - - If returns `"OOMKilled"`: - - Option 1: Increase virt-launcher memory limits - - Option 2: Decrease guest memory allocation - - Option 3: Use smaller instance type - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' - ``` - -4. **Increase resources if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check current memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -5. **Verify disk image integrity**: - - **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters for list**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" - } - ``` - - **Parameters for specific DV**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check `.status.phase` (should be `Succeeded`). - - If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get datavolume -n -o jsonpath='{.status.phase}' - ``` - -6. **Check virtualization (KVM) availability**: - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -7. **Simplify VM configuration** (eliminate variables): - - Try creating minimal VM using vm-create skill with: - - Small instance type - - No secondary networks - - Simple container disk (e.g., Fedora) - - No cloud-init - - If minimal VM works, add features back one by one. - -8. **Recreate VM with different workload** (test disk image): - - If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) - -After remediation, check pod restart count stops increasing: - -**Parameters for pods**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). - -Check VM reaches Running state: - -**Parameters for VM**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# RESTARTS should stabilize - -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Running -``` - -Verify guest is responsive: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# Should show login prompt or OS console -``` - -**Advanced Debugging**: - -**MCP Tool**: `pods_exec` (from openshift-virtualization) - -Check libvirt domain XML: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["virsh", "dumpxml", "1"] -} -``` - -Check QEMU process: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["ps", "aux"] -} -``` - -Filter output for "qemu" process. - -**CLI Fallback** (if MCP unavailable): -```bash -oc exec -n virt-launcher--xxx -- virsh dumpxml 1 -oc exec -n virt-launcher--xxx -- ps aux | grep qemu -``` - -**Prevention**: -- Start with minimal VM configuration and add complexity gradually -- Use recommended instance types for your workload -- Test disk images before deploying to production -- Ensure nodes have adequate resources and KVM support -- Monitor resource usage and set appropriate limits - ---- - -## Workaround Patterns for MCP Tool Limitations - -### General Pattern: Diagnose → Propose → Confirm → Execute - -When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): - -**1. Diagnose** the root cause using Kubernetes commands -```bash -oc describe vm -n -oc get events -n --field-selector involvedObject.name= -``` - -**2. Propose** a manual workaround to the user with clear explanation -- Explain why the issue occurred -- Show what will be changed -- List alternative options - -**3. Confirm** with user before executing (Human-in-the-Loop) -- Wait for explicit "yes" or "apply workaround" -- Never auto-execute modifications - -**4. Execute** the workaround using oc/kubectl -```bash -oc patch vm -n --type=merge -p '...' -``` - -**5. Verify** the fix was successful -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**6. Document** the limitation and suggest filing enhancement request -- Note this is temporary until MCP tool is enhanced -- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues - -### Example: Adding Tolerations Workaround - -**Diagnostic Output**: -```markdown -## ⚠️ VM Scheduling Issue Detected - -**Root Cause**: Node taints prevent VM scheduling - -**Details**: -- Found 3 nodes with taint: `virtualization=true:NoSchedule` -- VM spec does not include matching tolerations -- This prevents VM scheduling on virtualization-dedicated nodes -``` - -**Proposed Workaround**: -```bash -oc patch vm web-server -n vms --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' -``` - -**User Confirmation Required**: -``` -How would you like to proceed? -- "apply workaround" - I'll patch the VM with tolerations -- "manual" - I'll provide instructions for you to apply manually -- "cancel" - Delete the VM and abort creation -``` - -**After Execution**: -```markdown -## ✓ Workaround Applied Successfully - -**Action**: Added tolerations for taint `virtualization=true:NoSchedule` -**New Status**: Stopped (VM can now be scheduled) - -**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. -Future VMs in this cluster will need the same fix until the tool is enhanced. -``` - ---- - -## VM Status Reference - -### Status Values - -| Status | Meaning | Action Required | -|--------|---------|-----------------| -| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | -| `Running` | VM is running | Normal | -| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | -| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | -| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | -| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | -| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | -| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | -| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | -| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | - -### Checking VM Status - -```bash -# Get printable status -oc get vm -n -o jsonpath='{.status.printableStatus}' - -# Get detailed status and conditions -oc get vm -n -o jsonpath='{.status}' | jq - -# Watch status changes in real-time -oc get vm -n -w -``` - ---- - -## Best Practices for Agents - -When implementing diagnostic workflows: - -1. **Always verify VM status** after creation (wait 5-10 seconds first) -2. **Consult this document** when encountering error status values -3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) -4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) -5. **Respect human-in-the-loop** for all VM modifications -6. **Document temporary workarounds** and their limitations clearly -7. **Suggest filing issues** for missing MCP tool features - -### Document Consultation Pattern - -```markdown -**Document Consultation** (REQUIRED): -1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes -2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." -``` - ---- - -## Known MCP Tool Limitations - -### vm_create tool - -**Currently Supported**: -- ✓ Namespace, name (required) -- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) -- ✓ Size hints (small, medium, large) -- ✓ Storage size -- ✓ Autostart flag -- ✓ Networks (Multus NetworkAttachmentDefinitions) -- ✓ Performance family (u1, o1, c1, m1) -- ✓ Instance type, preference - -**Not Currently Supported** (requires workarounds): -- ✗ Tolerations (for node taints) -- ✗ Node selectors -- ✗ Affinity/anti-affinity rules -- ✗ Resource requests/limits (beyond instance type) -- ✗ Custom labels/annotations -- ✗ SSH keys injection -- ✗ Cloud-init user data - -**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. - -**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues - ---- - -## Additional Resources - -- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) -- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) -- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) -- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) - ---- - -[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/scheduling-errors.md deleted file mode 100644 index e47c8c4f..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/scheduling-errors.md +++ /dev/null @@ -1,417 +0,0 @@ ---- -title: VM Scheduling Errors -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 -tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] -semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] -use_cases: [vm-creation, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Scheduling Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**When to use this document**: -- VM shows status `ErrorUnschedulable` after creation or start attempt -- VM events mention scheduling failures, taints, resources, or node selectors - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -## ErrorUnschedulable - -**Symptom**: VM shows status `ErrorUnschedulable` after creation - -**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**Possible Causes**: - -### 1. Node Taints (Most Common) - -Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM events for scheduling failures**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `involvedObject.name` == "" and look for messages like: -- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= -``` - -**2. Check node taints in the cluster**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' -``` - -**Common Taint Patterns**: -- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule -- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes -- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads - -**Solution - Add Tolerations to VM**: - -The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. - -**Workaround (post-creation using MCP Tools)**: - -**Step 1**: Get current VM spec - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Step 2**: Modify the returned JSON to add tolerations - -Add to `.spec.template.spec.tolerations`: -```json -{ - "tolerations": [ - { - "key": "virtualization", - "operator": "Equal", - "value": "true", - "effect": "NoSchedule" - } - ] -} -``` - -**Step 3**: Update VM with modified spec - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters**: -```json -{ - "resource": "" -} -``` - -Pass the complete modified VM resource as YAML or JSON string. - -**Step 4**: Verify tolerations were added - -Use `resources_get` again and check `.spec.template.spec.tolerations` in response. - -**Step 5**: Check if VM status improved - -Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. - -**CLI Fallback** (if MCP patch is too complex): -```bash -# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' - -# Verify tolerations -oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq - -# Check status -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**Example - Multiple Tolerations**: -```bash -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" - - key: "dedicated" - operator: "Equal" - value: "virt-workloads" - effect: "NoSchedule" -' -``` - -**Toleration Operators**: -- `Equal` - Key and value must match exactly -- `Exists` - Only key must exist (ignores value) - -**Toleration Effects**: -- `NoSchedule` - Don't schedule new pods (existing pods continue) -- `PreferNoSchedule` - Avoid scheduling if possible -- `NoExecute` - Don't schedule AND evict existing pods - -**Alternative Solutions**: -1. **Remove node taints** (if you have cluster-admin access): - ```bash - oc adm taint nodes virtualization=true:NoSchedule- - ``` - -2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits - -3. **File enhancement request** - Request tolerations support in openshift-mcp-server: - https://github.com/openshift/openshift-mcp-server/issues - ---- - -### 2. Insufficient Resources - -Not enough CPU, memory, or storage available on any node. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM resource requests**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' -``` - -**2. Check node resource availability**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. - -Alternatively, use `nodes_top` MCP tool for current resource usage. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe nodes | grep -A 5 "Allocated resources" -``` - -**3. Look for VM events mentioning "Insufficient"**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep "Insufficient" -``` - -**Example Event**: -``` -0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. -``` - -**Solutions** (Use MCP Tools First): - -1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) -2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill -3. **Delete unused VMs** - Use vm-delete skill to free up resources -4. **Check resource quotas**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters for quota**: - ```json - { - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" - } - ``` - - **Parameters for limit range**: - ```json - { - "apiVersion": "v1", - "kind": "LimitRange", - "namespace": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - oc describe limitrange -n - ``` - ---- - -### 3. Node Selector Mismatch - -VM requires specific node labels that don't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM node selector requirements**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.nodeSelector` to see required node labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' -``` - -**2. List available node labels**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.metadata.labels` for available labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes --show-labels -``` - -**3. Check if any nodes match the selector**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node", - "labelSelector": "=" -} -``` - -Should return at least one node with matching labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -l = -``` - -**Solutions** (Use MCP Tools First): - -**Option 1: Remove node selector from VM** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get current VM using `resources_get` (diagnostic step 1) -2. Remove `.spec.template.spec.nodeSelector` field -3. Update VM using `resources_create_or_update` with modified JSON - -**CLI Fallback** (JSON patch easier via CLI): -Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" -```bash -oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' -``` - -**Option 2: Add label to nodes** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get node using `resources_get` -2. Add label to `.metadata.labels` -3. Update node using `resources_create_or_update` - -⚠️ **Note**: Node labeling typically requires cluster admin privileges. - -**CLI Fallback** (simpler via CLI): -Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" -```bash -oc label node = -``` - ---- - -[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/storage-errors.md b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/storage-errors.md deleted file mode 100644 index 16ba584d..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/docs/troubleshooting/storage-errors.md +++ /dev/null @@ -1,1011 +0,0 @@ ---- -title: VM Storage Errors -category: kubevirt -sources: - - title: KubeVirt CDI - DataVolumes - url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ - date_accessed: 2026-02-17 - - title: Kubernetes Persistent Volumes - url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ - date_accessed: 2026-02-17 -tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] -semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] -use_cases: [vm-creation, vm-deletion, vm-cloning] -related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] -last_updated: 2026-02-17 ---- - -# VM Storage Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. - -**When to use this document**: -- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` -- Storage deletion fails after VM deletion -- DataVolume cloning operations fail -- PVC provisioning issues - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### ErrorDataVolumeNotReady - -**Symptom**: VM shows status `ErrorDataVolumeNotReady` - -**Description**: The DataVolume (persistent storage) backing the VM is not ready. - -**Possible Causes**: - -#### 1. DataVolume Still Provisioning - -Storage provisioning takes time, especially for large disks or when importing images. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Get detailed DataVolume information**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` and `.status.conditions` for provisioning details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o json -``` - -**3. Check PVC (PersistentVolumeClaim) bound status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Check `.status.phase` for each PVC (should be `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. - -#### 2. Storage Class Not Found - -The requested storage class doesn't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List available storage classes**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" -} -``` - -Review the list of available storage classes (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -``` - -**2. Check DataVolume's requested storage class**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.spec.pvc.storageClassName` in the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' -``` - -**Solution**: -1. Use a valid storage class from the cluster -2. Recreate VM with correct storage class parameter - -#### 3. Insufficient Storage Quota - -Namespace has insufficient storage quota to provision the PVC. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check resource quotas**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**2. Check storage usage**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase -``` - -**Solution**: -1. Request quota increase from cluster admin -2. Delete unused PVCs to free quota -3. Reduce VM storage size - ---- - -### ErrorPvcNotFound - -**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List PVCs in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review the list of available PVCs (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check VM's PVC references**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' -``` - -**Solution**: -- Wait for DataVolume to create the PVC -- Manually create missing PVC -- Fix VM spec to reference correct PVC name - ---- - - ---- - -### Storage Deletion Failures - -**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace - -**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. - -**Possible Causes**: -- PVC still bound to active PersistentVolume with `Retain` policy -- DataVolume still being referenced by another resource -- CDI (Containerized Data Importer) controller issues -- Storage class retention policy preventing deletion -- Finalizers on PVC/DataVolume blocking cleanup -- PVC still mounted by a pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check PVC status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check specific PVC phase**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Released` or `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o jsonpath='{.status.phase}' -``` - -**3. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**4. Check what's using the PVC**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' -``` - -**5. Check PVC finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. - -**6. Check DataVolume finalizers**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.metadata.finalizers}' -``` - -**7. Check PV reclaim policy**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume" -} -``` - -Filter results for PV where `.spec.claimRef.name` matches ``. - -To get specific PV policy: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume", - "name": "" -} -``` - -Check `.spec.persistentVolumeReclaimPolicy`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pv | grep -oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' -``` - -**Common Finalizer Patterns**: -- `kubernetes.io/pvc-protection` - Protects PVC while in use -- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer - -**Solutions** (Use MCP Tools First): - -1. **Delete DataVolume first, then PVC**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete DataVolume first (often blocks PVC deletion): - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Wait a few seconds, then delete PVC: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete datavolume -n - oc delete pvc -n - ``` - -2. **Check for pods still using PVC**: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "" - } - ``` - - Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. - - Then delete the pods using `pods_delete`: - - **Parameters**: - ```json - { - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' - oc delete pod -n - ``` - -3. **Force delete PVC** (if safe to do so): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete pvc -n --grace-period=0 --force - ``` - -4. **Remove finalizers from PVC** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PVC using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update PVC using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Remove finalizers from DataVolume** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current DataVolume using `resources_get` (diagnostic step 6) - 2. Remove items from `.metadata.finalizers` array - 3. Update DataVolume using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Change PV reclaim policy** (if PV has Retain policy): - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PV policy using `resources_get` (diagnostic step 7) - 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` - 3. Update PV using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" - ```bash - oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' - oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' - ``` - -**Storage Quota Check** (Use MCP Tools First): - -After deletion, verify storage quota is freed: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters for quota check**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.used` to verify storage quota is freed. - -**Parameters for PVC verification**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -oc get pvc -n -``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Confirm PVC is deleted: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Confirm DataVolume is deleted: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -# Should return: Error from server (NotFound) - -oc get datavolume -n -# Should return: Error from server (NotFound) -``` - ---- - -### DataVolume Cloning Failures - -**Symptom**: VM clone created successfully but DataVolume clone operation fails - -**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. - -**Possible Causes**: -- CSI driver doesn't support volume cloning -- Source PVC storage class incompatible with cloning -- Cross-namespace cloning not permitted by storage backend -- Insufficient storage quota in target namespace -- Source PVC not in `Bound` state -- Storage class doesn't have volume cloning enabled -- CDI (Containerized Data Importer) controller issues - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume clone status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and `.metadata.name`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Check DataVolume events for errors**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe datavolume -n -``` - -**3. Check DataVolume phase**: - -Use `resources_get` from step 1, extract `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -``` - -**4. Check if storage class supports cloning**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass", - "name": "" -} -``` - -Review the full YAML output for cloning-related configurations. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -o yaml | grep -A 5 -i clone -``` - -**5. Check CSI driver capabilities**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "CSIDriver" -} -``` - -Review `.items[].metadata.name` for available CSI drivers. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get csidriver -``` - -**6. Check source PVC status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Bound` for cloning to work). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**7. Check target namespace storage quota**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**8. Check CDI controller logs**: - -**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) - -First, list pods in openshift-cnv namespace: - -**Parameters for pods_list_in_namespace**: -```json -{ - "namespace": "openshift-cnv", - "labelSelector": "app.kubernetes.io/component=cdi-deployment" -} -``` - -Then get logs using `pods_log`: - -**Parameters**: -```json -{ - "namespace": "openshift-cnv", - "name": "", - "tail": 100 -} -``` - -**CLI Fallback** (if MCP unavailable or easier via CLI): -```bash -oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') -``` - -**Common Error Messages**: -- `"volume cloning is not supported"` - CSI driver lacks clone capability -- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage -- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace -- `"insufficient quota"` - Target namespace lacks storage quota -- `"source PVC not bound"` - Source PVC must be in Bound state for cloning -- `"StorageClass does not support cloning"` - Storage class configuration issue - -**Solutions** (Use MCP Tools First): - -1. **Check storage class clone support**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" - } - ``` - - For each storage class in `.items[]`, check: - - `.metadata.name` (storage class name) - - `.provisioner` (CSI driver) - - Storage classes using CSI drivers typically support cloning. Look for provisioners like: - - `csi.ovirt.org` (oVirt CSI) - - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) - - `ebs.csi.aws.com` (AWS EBS CSI) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner - ``` - -2. **Verify source PVC is bound**: - - Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pvc -n -o jsonpath='{.status.phase}' - ``` - -3. **Check target namespace quota**: - - Use `resources_list` from diagnostic step 7 to check quota. - - If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - ``` - -4. **Use snapshot-based cloning** (alternative method): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Step 1**: Create VolumeSnapshot of source PVC - - **Parameters**: - ```json - { - "apiVersion": "snapshot.storage.k8s.io/v1", - "kind": "VolumeSnapshot", - "metadata": { - "name": "-snapshot", - "namespace": "" - }, - "spec": { - "source": { - "persistentVolumeClaimName": "" - } - } - } - ``` - - **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) - - **Step 3**: Create new DataVolume from snapshot - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "metadata": { - "name": "-rootdisk", - "namespace": "" - }, - "spec": { - "source": { - "snapshot": { - "name": "-snapshot", - "namespace": "" - } - }, - "storage": { - "resources": { - "requests": { - "storage": "50Gi" - } - }, - "storageClassName": "" - } - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" - ```bash - cat <-snapshot - namespace: - spec: - source: - persistentVolumeClaimName: - EOF - - oc get volumesnapshot -snapshot -n - - cat <-rootdisk - namespace: - spec: - source: - snapshot: - name: -snapshot - namespace: - storage: - resources: - requests: - storage: 50Gi - storageClassName: - EOF - ``` - -5. **Use "new empty storage" option** (vm-clone skill): - - If cloning isn't supported, create VM with empty storage - - Manually copy data if needed - -6. **Cross-namespace cloning workaround**: - - Some storage backends require snapshot for cross-namespace cloning - - Create snapshot in source namespace, restore in target namespace (see solution 4 above) - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Check DataVolume reached Succeeded phase: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should return `Succeeded`). - -Check PVC was created and bound: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "-rootdisk" -} -``` - -Check `.status.phase` (should return `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -# Should return: Succeeded - -oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' -# Should return: Bound -``` - -**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/skills/vm-snapshot-restore/SKILL.md b/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/skills/vm-snapshot-restore/SKILL.md deleted file mode 100644 index f4e2fb19..00000000 --- a/evaluation/with_skills/rh-virt__vm-snapshot-restore/environment/skills/vm-snapshot-restore/SKILL.md +++ /dev/null @@ -1,495 +0,0 @@ ---- -name: vm-snapshot-restore -description: | - Restore virtual machines from snapshots with strict safety confirmations to prevent data loss. - - Use when: - - "Restore VM [name] from snapshot [snapshot-name]" - - "Roll back VM [name] to snapshot" - - "Recover VM [name] from backup" - - CRITICAL: Requires VM to be stopped and typed snapshot name confirmation before restore. - - NOT for creating snapshots (use vm-snapshot-create instead). - -model: inherit -color: red ---- - -# /vm-snapshot-restore Skill - -Restore virtual machines from snapshots in OpenShift Virtualization. **CRITICAL**: This operation replaces current VM state with snapshot data. ALL changes since the snapshot will be LOST. - -**Implementation Note**: This skill uses generic Kubernetes resource tools (`resources_create_or_update`) to create VirtualMachineRestore resources. Dedicated restore tools do not currently exist in the openshift-virtualization MCP server. - -## Prerequisites - -**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools**: -- `resources_create_or_update` (from openshift-virtualization) - Create VirtualMachineRestore -- `resources_get` (from openshift-virtualization) - Verify VM/snapshot exists, monitor restore -- `vm_lifecycle` (from openshift-virtualization) - Stop VM if running - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster (>= 4.19) -- OpenShift Virtualization operator installed -- ServiceAccount with RBAC permissions to create VirtualMachineRestore resources - -## When to Use This Skill - -**Trigger this skill when:** -- User wants to restore a VM to a previous state -- User wants to recover from failed changes/upgrades -- User explicitly requests snapshot restore - -**User phrases that trigger this skill:** -- "Restore VM api-server from snapshot snapshot-20240115" -- "Roll back database-01 to pre-upgrade snapshot" -- "Recover VM web-server from backup" - -**Do NOT use this skill when:** -- User wants to create snapshots → Use `vm-snapshot-create` skill -- User wants to list snapshots → Use `vm-snapshot-list` skill -- User wants to clone a VM → Use `vm-clone` skill - -## Workflow - -### Step 1: Gather Restore Information - -**Required Information from User:** -1. **VM Name** - VM to restore -2. **Namespace** - Namespace where VM exists -3. **Snapshot Name** - Snapshot to restore from - -If any information missing, ask for it. - -### Step 2: Verify VM Exists - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Error Handling**: -- If VM not found → Report error -- If permission denied → Report RBAC error - -### Step 3: Check VM Running State - -**From the VM resource in Step 2**, check `status.printableStatus`. - -**If VM is Running:** -```markdown -⚠️ VM Must Be Stopped Before Restore - -**VM**: `` (namespace: ``) -**Status**: Running - -**Safety Requirement**: VMs must be stopped before restore to prevent data corruption. - -**Options:** -1. "stop-and-restore" - Stop the VM first, then restore from snapshot -2. "cancel" - Cancel restore operation - -How would you like to proceed? -``` - -**Wait for user response.** - -- If "stop-and-restore" → Stop VM using vm_lifecycle, then continue -- If "cancel" → Stop workflow - -### Step 4: Verify Snapshot Exists - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "snapshot.kubevirt.io/v1beta1", - "kind": "VirtualMachineSnapshot", - "namespace": "", - "name": "" -} -``` - -**If snapshot not found:** -```markdown -❌ Snapshot Not Found - -**Snapshot**: `` does not exist in namespace ``. - -**To list available snapshots:** -"List snapshots for VM " - -Restore operation cancelled. -``` - -**STOP workflow**. - -**Extract snapshot details:** -- `metadata.creationTimestamp` - Creation time -- `status.phase` - Must be "Succeeded" -- `status.readyToUse` - Must be `true` -- `spec.source.name` - Verify it matches the VM name - -**If snapshot status is not Ready:** -```markdown -❌ Snapshot Not Ready - -**Snapshot**: `` -**Status**: -**Ready to Use**: - -Snapshot is not ready for restore. Only snapshots with "Succeeded" phase and readyToUse=true can be used. - -Restore operation cancelled. -``` - -**STOP workflow**. - -### Step 5: Present Restore Preview and Get Typed Confirmation - -**CRITICAL: User must type the snapshot name to confirm.** - -```markdown -## 🔴 VM RESTORE - Data Loss Warning - -**⚠️ THIS WILL REPLACE CURRENT VM STATE WITH SNAPSHOT DATA ⚠️** - -### What Will Happen - -**VM to Restore**: `` (namespace: ``) -**Snapshot to Restore From**: `` - -**Current VM State** (WILL BE LOST): -- **Last Modified**: -- **Changes Since Snapshot**: ALL changes made after WILL BE PERMANENTLY LOST - -**Snapshot State** (WILL BE RESTORED): -- **Created**: -- **Age**: - -**Time Range of Data Loss**: -- **⚠️ ALL CHANGES in the last WILL BE LOST ⚠️** - -### What Will Be Restored -- ✓ VM configuration (from snapshot time) -- ✓ Disk data (from snapshot time) - -### What Will Be Lost -- ✗ **ALL disk changes** made after -- ✗ **ALL configuration changes** made after - ---- - -**⚠️ CRITICAL: This restore is permanent. Current VM state cannot be recovered unless you create a snapshot now.** - -**To proceed with restore, type the snapshot name exactly as shown:** - -Type `` to confirm: _____ -``` - -**Wait for user to type the snapshot name.** - -**Validation:** -- Compare user input with snapshot name (case-sensitive, exact match) -- **If match**: Proceed to Step 6 -- **If mismatch**: Cancel operation - -**On mismatch:** -```markdown -❌ Confirmation Failed - -**You typed**: `` -**Expected**: `` - -Names do not match. Restore cancelled for safety. - -Operation cancelled. Current VM state preserved. -``` - -**STOP workflow**. - -### Step 6: Final Confirmation Before Restore - -**After typed verification succeeds**, ask for final explicit confirmation. - -```markdown -## ✓ Typed Verification Passed - -**Confirmation received for snapshot**: `` - -### Ready to Restore - -**VM**: `` (namespace: ``) -**From Snapshot**: `` - -**Impact**: -- Current VM state will be replaced with snapshot state -- All changes in the last will be permanently lost - ---- - -**Proceed with VM restore? This action cannot be undone.** -- Type "yes" to execute restore -- Type "cancel" to abort - -Your choice: _____ -``` - -**Wait for user response.** - -**Handle response:** -- If "yes" → Proceed to Step 7 (execute restore) -- If "cancel", "no", "wait", or anything else → Cancel operation - -**On cancellation:** -```markdown -Restore operation cancelled by user. Current VM state preserved. -``` - -**STOP workflow**. - -### Step 7: Execute Restore - -**ONLY PROCEED AFTER**: -- ✓ VM verified (exists, stopped) -- ✓ Snapshot verified (exists, ready) -- ✓ User typed snapshot name correctly -- ✓ User confirmed "yes" - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Construct VirtualMachineRestore YAML:** - -```yaml -apiVersion: snapshot.kubevirt.io/v1beta1 -kind: VirtualMachineRestore -metadata: - name: - namespace: -spec: - target: - apiGroup: kubevirt.io - kind: VirtualMachine - name: - virtualMachineSnapshotName: -``` - -**Generate restore name**: -- Format: `restore--` -- Example: `restore-database-01-20260218-143500` - -**Parameters**: -```json -{ - "resource": "apiVersion: snapshot.kubevirt.io/v1beta1\nkind: VirtualMachineRestore\nmetadata:\n name: \n namespace: \nspec:\n target:\n apiGroup: kubevirt.io\n kind: VirtualMachine\n name: \n virtualMachineSnapshotName: " -} -``` - -**Report progress:** -```markdown -🔄 Restoring VM from snapshot... -⏳ This may take several minutes... -``` - -### Step 8: Monitor Restore Progress - -**Use `resources_get` to monitor VirtualMachineRestore status.** - -Check `status.complete`: -- `true` → Restore completed -- `false` → Restore in progress - -**Wait up to 10 minutes for restore to complete.** - -### Step 9: Report Restore Results - -**On success:** - -```markdown -## ✓ VM Restored Successfully - -**VM**: `` (namespace: ``) -**Restored From**: Snapshot `` - -### Restore Details -- **Snapshot Created**: -- **Restore Completed**: -- **VM Status**: Stopped (ready to start) - -### Data Loss Confirmation -- ⚠️ All changes made after have been lost - -### Next Steps - -**To start the restored VM:** -"Start VM in namespace " -``` - -**On failure:** - -```markdown -## ❌ VM Restore Failed - -**Error**: - -**VM**: `` -**Snapshot**: `` - -**Current VM State**: UNKNOWN - may be partially restored or unchanged - -**CRITICAL**: Do not start VM until restore issue is resolved - -**Recovery Options:** -1. Try restore again after resolving the error -2. Restore from a different snapshot -3. Contact cluster admin for investigation -``` - -## Dependencies - -### Required MCP Servers -- `openshift-virtualization` - OpenShift MCP server with kubevirt toolset - -### Required MCP Tools -- `resources_create_or_update` (from openshift-virtualization) - Create VirtualMachineRestore -- `resources_get` (from openshift-virtualization) - Verify and monitor -- `vm_lifecycle` (from openshift-virtualization) - Stop VM if running - -### Related Skills -- `vm-snapshot-list` - List snapshots before restore -- `vm-snapshot-create` - Create snapshots before risky operations -- `vm-snapshot-delete` - Delete old snapshots -- `vm-lifecycle-manager` - Start VM after restore - -### Reference Documentation - -**Official Red Hat Documentation:** -- [OpenShift Virtualization Snapshots - OpenShift 4.20](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt-managing-vm-snapshots) - -**Upstream Documentation:** -- [KubeVirt VM Snapshots](https://kubevirt.io/user-guide/operations/snapshot_restore_api/) - -## Critical: Human-in-the-Loop Requirements - -**IMPORTANT:** This skill performs DESTRUCTIVE operations. You MUST: - -1. **Before Restoring Snapshots** (CRITICAL - Data Loss Risk) - - **REQUIRE VM to be stopped first** if currently running - - Display what will be lost (current VM state since snapshot) - - Show snapshot details (creation time, age) - - **Require typed confirmation** - user must type snapshot name exactly - - Ask: "Proceed with restore? This will replace current VM state. (yes/cancel)" - - Wait for explicit "yes" - -2. **Never Auto-Execute** - - **NEVER restore without user confirmation** - - **NEVER restore to running VMs** without stopping first - - **NEVER skip typed verification for restore operations** - -**Why This Matters:** -- **Data Loss on Restore**: Restoring replaces current VM state - all changes since snapshot are PERMANENTLY LOST -- **No Undo**: Restore cannot be reversed - current data cannot be recovered -- **Typed Confirmation**: Prevents accidental restores to wrong snapshots - -## Common Issues - -### Issue 1: Restore Fails - Insufficient Storage Capacity - -**Error**: "Failed to restore: insufficient storage capacity" or "PVC provisioning failed" - -**Cause**: The namespace doesn't have enough storage quota or the storage backend is full. - -**Solution:** -1. Check namespace storage quota: `resources_list` with kind="ResourceQuota" -2. Check PVC status: `resources_list` for PersistentVolumeClaims -3. Delete unnecessary snapshots: Use vm-snapshot-delete skill -4. Request quota increase: Contact cluster admin -5. Retry restore once storage is available - -### Issue 2: Restore Stuck in Progress - -**Error**: VirtualMachineRestore status shows `complete: false` for extended period - -**Cause**: The storage backend is slow, the snapshot is corrupted, or there's a CSI driver issue. - -**Solution:** -1. Check VirtualMachineRestore `status.conditions` for detailed error messages -2. Verify snapshot is "Succeeded": Use vm-snapshot-list skill -3. Wait longer: Large VMs may take 10+ minutes to restore -4. Cancel and retry: Delete VirtualMachineRestore resource and try again -5. Try alternative snapshot if restore continues to fail -6. Check CSI driver logs (requires cluster admin access) - -### Issue 3: User Typed Wrong Snapshot Name - -**Error**: "Confirmation Failed - Names do not match" - -**Cause**: User made a typo when typing the snapshot name for confirmation. - -**Solution:** -1. Retry with exact name: Copy-paste the snapshot name from the previous message -2. Check capitalization: Snapshot names are case-sensitive -3. Check for spaces: Ensure no leading/trailing spaces in the typed name -4. Use vm-snapshot-list skill to see all available snapshots and verify the correct name - -**This safety mechanism is intentional** - it prevents accidental data loss from restoring to the wrong snapshot. - -## Security Considerations - -- **Data Protection**: Restore requires TWO confirmations (typed + yes) to prevent accidental data loss -- **VM Stop Requirement**: VMs must be stopped before restore -- **Typed Verification**: User must type exact snapshot name -- **RBAC Enforcement**: Requires permissions for VirtualMachineRestore resources -- **Namespace Isolation**: Restore scoped to namespace boundaries -- **Audit Trail**: All restore operations logged in Kubernetes API audit logs -- **No Rollback**: Restore is permanent - current state cannot be recovered after restore - -## Example Usage - -**User**: "The database upgrade failed. Restore database-01 from snapshot pre-upgrade-snapshot" - -**Agent**: "Which namespace is database-01 in?" - -**User**: "production" - -**Agent**: [Verifies VM - Running status detected] -"⚠️ VM Must Be Stopped Before Restore. Options: 1) stop-and-restore 2) cancel" - -**User**: "stop-and-restore" - -**Agent**: [Stops VM, verifies snapshot exists] -"🔴 VM RESTORE - Data Loss Warning -- VM: database-01 (production) -- Snapshot: pre-upgrade-snapshot (created 2026-02-18 10:00, age 6h 30m) -- ⚠️ ALL CHANGES in last 6h 30m WILL BE LOST -Type `pre-upgrade-snapshot` to confirm: _____" - -**User**: "pre-upgrade-snapshot" - -**Agent**: "✓ Typed Verification Passed. Proceed with restore? (yes/cancel)" - -**User**: "yes" - -**Agent**: "🔄 Restoring VM from snapshot... ⏳ This may take several minutes..." -[Monitors progress] -"✓ VM Restored Successfully -- VM: database-01 (production) -- Restored From: pre-upgrade-snapshot (created 2026-02-18 10:00) -- Status: Stopped (ready to start) -- ⚠️ All changes after 2026-02-18 10:00 have been lost - -To start: 'Start VM database-01 in namespace production'" diff --git a/evaluation/without_skills/ocp-admin__cluster-report/environment/docs/multi-cluster-auth.md b/evaluation/without_skills/ocp-admin__cluster-report/environment/docs/multi-cluster-auth.md deleted file mode 100644 index e187471b..00000000 --- a/evaluation/without_skills/ocp-admin__cluster-report/environment/docs/multi-cluster-auth.md +++ /dev/null @@ -1,248 +0,0 @@ -# Multi-Cluster Authentication with Service Account Tokens - -Set up non-interactive, long-lived authentication for running `cluster-report` across many OpenShift clusters without repeated `oc login` sessions. - -## Overview - -The `cluster-report` skill requires valid kubeconfig contexts for every cluster it reports on. Interactive `oc login --web` opens a browser for each cluster and produces tokens that expire in ~24 hours which make it difficult to do at scale. - -**Solution**: Create a read-only ServiceAccount on each cluster with a non-expiring token. A builder script assembles these tokens into a single merged kubeconfig that the skill uses unchanged. - -## Prerequisites - -- `oc` or `kubectl` CLI -- `python3` (stdlib only, no extra packages) -- `cluster-admin` access on each target cluster (one-time setup only) - -## Quick Start (Automated) - -If you're currently logged into all the clusters you would like to get a report for via `oc login`: - -```bash -# Step 1: Setup — applies RBAC to each cluster, extracts SA tokens -python3 ocp-admin/scripts/cluster-report/build-kubeconfig.py setup --all-contexts - -# Step 2: Build — assembles a merged kubeconfig from the inventory -python3 ocp-admin/scripts/cluster-report/build-kubeconfig.py \ - build --clusters ~/.ocp-clusters/clusters.json --verify - -# Step 3: Use — export and run the skill -export KUBECONFIG=/tmp/cluster-report-kubeconfig -# Then in Claude Code use the skill: /cluster-report -``` - -After the one-time setup, only Steps 2–3 are needed for future report sessions. - -## Manual Setup (Per Cluster) - -If you prefer to set up each cluster individually: - -### 1. Apply RBAC - -> **Required permissions**: The manifest creates cluster-scoped resources (ClusterRole, ClusterRoleBinding), so the user applying it needs `cluster-admin` privileges. This is a one-time setup step. - -```bash -oc login -oc apply -f ocp-admin/scripts/cluster-report/cluster-reporter-rbac.yaml -``` - -This creates: - -- Namespace `cluster-reporter-system` -- ServiceAccount `cluster-reporter` with a read-only ClusterRole -- ClusterRoleBinding `cluster-reporter-binding` (binds the SA to the ClusterRole) -- Token Secret `cluster-reporter-token` (non-expiring) - -### 2. Extract the Token - -```bash -oc get secret cluster-reporter-token -n cluster-reporter-system \ - -o jsonpath='{.data.token}' | base64 -d -``` - -Save this token securely. It grants read-only access to nodes, pods, namespaces, projects, cluster version, and metrics. - -> **AI Safety**: Never display token values in conversation output. Verify tokens are set, but never print or echo their contents. - -### 3. Add to Inventory File - -Create or edit `~/.ocp-clusters/clusters.json`: - -```json -{ - "clusters": [ - { - "name": "prod-us-east", - "api_url": "https://api.prod-us-east.example.com:6443", - "token": "sha256~your-token-here" - } - ] -} -``` - -Set permissions: `chmod 600 ~/.ocp-clusters/clusters.json` - -### 4. Build Kubeconfig - -```bash -python3 ocp-admin/scripts/cluster-report/build-kubeconfig.py \ - build --clusters ~/.ocp-clusters/clusters.json --output ~/.kube/cluster-report-kubeconfig -``` - -## RBAC Permissions - -The `cluster-reporter-readonly` ClusterRole grants the minimum permissions required by the `cluster-report` skill: - - -| Resource | API Group | Verbs | Used By | -| ----------------------- | -------------------- | --------- | ------------------------------------------------------------- | -| nodes, namespaces, pods | core | get, list | `nodes_top`, `resources_list`, `namespaces_list`, `pods_list` | -| clusterversions | config.openshift.io | get | `resources_get` (OpenShift verification) | -| projects | project.openshift.io | list | `projects_list` | -| nodes, pods (metrics) | metrics.k8s.io | get, list | `nodes_top` | - - -No create, update, delete, or watch permissions are granted. - -## Clusters Inventory Format - -The inventory file (`clusters.json`) supports two token modes: - -### Inline Tokens (Simple) - -```json -{ - "clusters": [ - { - "name": "prod-us-east", - "api_url": "https://api.prod-us-east.example.com:6443", - "token": "sha256~abc123..." - } - ] -} -``` - -The file itself contains secrets — keep it out of git and set `chmod 600`. - -### Environment Variable References (More Secure) - -```json -{ - "clusters": [ - { - "name": "prod-us-east", - "api_url": "https://api.prod-us-east.example.com:6443", - "token_env": "CLUSTER_TOKEN_PROD_US_EAST" - } - ] -} -``` - -The file contains no secrets. Load tokens into environment variables from your secrets manager before running `--build`. - -### Optional: CA Certificate - -```json -{ - "clusters": [ - { - "name": "prod-us-east", - "api_url": "https://api.prod-us-east.example.com:6443", - "token": "sha256~abc123...", - "ca_cert": "/path/to/prod-us-east-ca.crt" - } - ] -} -``` - -If `ca_cert` is omitted, TLS verification is skipped (`--insecure-skip-tls-verify`). - -## Script Reference - -### `setup` Subcommand - -```bash -python3 build-kubeconfig.py setup [OPTIONS] -``` - - -| Flag | Description | Default | -| --------------------------- | ----------------------------- | ------------------------------- | -| `--all-contexts` | Setup all kubeconfig contexts | Lists contexts and exits | -| `--contexts ctx1,ctx2` | Setup only specified contexts | — | -| `--output-inventory ` | Inventory file path | `~/.ocp-clusters/clusters.json` | - - -Behavior: - -- Applies `cluster-reporter-rbac.yaml` to each cluster -- Waits up to 15 seconds for the token Secret to populate -- Extracts and saves the token to the inventory file -- Skips unreachable clusters with an error message -- Appends to existing inventory (deduplicates by name) - -### `build` Subcommand - -```bash -python3 build-kubeconfig.py build --clusters [OPTIONS] -``` - - -| Flag | Description | Default | -| ------------------- | -------------------------------- | -------------------------------- | -| `--clusters ` | Inventory file path (required) | — | -| `--output ` | Kubeconfig output path | `/tmp/cluster-report-kubeconfig` | -| `--verify` | Test each context after building | Off | - - -Behavior: - -- Reads inventory, resolves tokens (inline or env var) -- Builds kubeconfig with `kubectl config set-cluster/set-credentials/set-context` -- Partial success: continues on individual failures -- `--verify` tests each context with `cluster-info` -- Outputs JSON summary with success/error counts - -## Token Rotation - -SA token Secrets do not expire, but you may want to rotate them periodically: - -```bash -oc delete secret cluster-reporter-token -n cluster-reporter-system -oc apply -f ocp-admin/scripts/cluster-report/cluster-reporter-rbac.yaml - -oc get secret cluster-reporter-token -n cluster-reporter-system \ - -o jsonpath='{.data.token}' | base64 -d - -python3 build-kubeconfig.py build --clusters ~/.ocp-clusters/clusters.json --verify -``` - -To detect expired or invalid tokens: - -```bash -python3 build-kubeconfig.py build --clusters ~/.ocp-clusters/clusters.json --verify -``` - -## Security Best Practices - -1. **Never commit tokens to git** — add `clusters.json` to `.gitignore` -2. **File permissions** — `chmod 600` on both `clusters.json` and the generated kubeconfig -3. **Prefer `token_env`** — store actual tokens in a secrets manager, not in files -4. **Minimum RBAC** — the ClusterRole grants read-only access only -5. **Dedicated namespace** — the SA lives in `cluster-reporter-system`, not `kube-system` -6. **Generated kubeconfig is ephemeral** — `/tmp/` is fine for session use; for persistent storage use `~/.kube/` with `chmod 600` -7. **Never display tokens in AI conversations** — verify tokens are set but never print, echo, or expose their values in output - -## Troubleshooting - - -| Problem | Cause | Fix | -| ---------------------------------------- | ----------------------------------------- | ------------------------------------------------------------- | -| `--setup` skips a cluster | Not logged in or auth expired | `oc login ` first, then re-run setup | -| `--verify` fails for a cluster | Token expired or Secret deleted | Re-run `--setup --contexts ` for that cluster | -| `cluster-report` shows 401 for a cluster | Token invalid | Same as above — re-run setup for that cluster | -| `cluster-report` shows 403 | SA missing permissions | Re-apply `cluster-reporter-rbac.yaml` on that cluster | -| Token Secret not populated | Token controller slow or SA doesn't exist | Wait and retry; verify SA exists in `cluster-reporter-system` | -| `--build` says "env var not set" | Using `token_env` but env not loaded | Export the token env vars before running `--build` | - - diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/common-issues.md b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/examples/model-deploy.md b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/examples/model-deploy.md deleted file mode 100644 index dcd79e44..00000000 --- a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/examples/model-deploy.md +++ /dev/null @@ -1,160 +0,0 @@ -# /model-deploy Examples - -Example interactions demonstrating the model-deploy skill workflow. - -## Example 1: Deploy Llama 3.1 8B with vLLM - -``` -User: "Deploy Llama 3.1 8B on my cluster" - -Agent: [Invokes /model-deploy skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand runtime capabilities. - -Step 1 of 10: Gathering deployment information. - -| Setting | Value | Source | -|---------|-------|--------| -| Model | Llama 3.1 8B Instruct | user input | -| Runtime | TBD | auto-detect | -| Namespace | ? | needs input | -| Model Source | ? | needs input | -| Deployment Mode | Serverless | default | - -What namespace should I deploy to, and where are the model weights stored? -(e.g., S3 bucket, OCI registry, PVC) - -User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" - -Agent: Step 2 of 10: Determining runtime. - -I consulted supported-runtimes.md to understand runtime capabilities. - -Based on Llama 3.1 8B (open-source HuggingFace model): - -Recommended runtime: vLLM -Reason: Default runtime for open-source LLMs. Broad compatibility, - OpenAI-compatible API, no additional setup required. - -Proceed with vLLM? (yes / no / suggest alternative) - -User: "yes" - -Agent: Step 3 of 10: Looking up hardware profile. - -I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. - -| Requirement | Value | -|-------------|-------| -| GPUs | 1x A10G/L4/A100/H100 | -| VRAM | 16GB | -| Key Args | --max-model-len=4096 --tool-call-parser hermes | - -Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). -Ensure your cluster has at least 1 GPU with 16GB VRAM. - -Step 5 of 10: Skipped (NIM platform check not needed for vLLM). - -Step 6-7 of 10: Generating InferenceService YAML. - -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - name: llama-3-1-8b - namespace: ai-models - annotations: - serving.kserve.io/deploymentMode: Serverless -spec: - predictor: - model: - modelFormat: - name: vLLM - runtime: vllm-runtime - storageUri: s3://my-bucket/llama-3.1-8b - resources: - limits: - nvidia.com/gpu: "1" - requests: - cpu: "4" - memory: "16Gi" - args: - - --max-model-len=4096 - - --tool-call-parser=hermes - -| Setting | Value | -|---------|-------| -| Model | llama-3-1-8b | -| Runtime | vllm-runtime | -| GPUs | 1 | -| Namespace | ai-models | -| Mode | Serverless | - -Proceed with creating this InferenceService? (yes/no/modify) - -User: "yes" - -Agent: Step 8 of 10: Creating InferenceService... - [Creates via resources_create_or_update] - - InferenceService created. Monitoring rollout... - -Step 9 of 10: Monitoring rollout. - -| Stage | Status | -|-------|--------| -| Pod Scheduled | Complete | -| Image Pulled | Complete | -| Container Started | Complete | -| Model Loaded | Complete | -| Ready | Complete | - -Step 10 of 10: Deployment complete! - -Model Deployment Successful - -Model: llama-3-1-8b -Runtime: vLLM -Namespace: ai-models -GPUs: 1 - -Inference Endpoint: - URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com - API: OpenAI-compatible REST - -Quick Test: - curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' - -Next Steps: - Monitor performance: "/ai-observability llama-3-1-8b" -``` - -## Example 2: Deploy a Model with NIM (NIM Not Set Up) - -``` -User: "Deploy Llama 3.1 70B with NIM" - -Agent: [Invokes /model-deploy skill] - ... - Step 2: User requested NIM runtime. - - Step 5: Checking NIM platform... - -NIM platform is not set up in namespace ai-models. - -The NIM runtime requires a one-time platform setup. Would you like to: -1. Run /nim-setup now to configure the NIM platform -2. Switch to vLLM runtime instead (no setup required) -3. Abort deployment - -User: "1" - -Agent: [Invokes /nim-setup skill — walks through NGC credential setup] - ... - NIM Platform Setup Complete. - - Resuming model deployment... - [Continues from Step 6 with NIM runtime] -``` diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/examples/nim-setup.md b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/examples/nim-setup.md deleted file mode 100644 index ccfc1a5d..00000000 --- a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/examples/nim-setup.md +++ /dev/null @@ -1,115 +0,0 @@ -# /nim-setup Examples - -Example interactions demonstrating the nim-setup skill workflow. - -## Example 1: First-Time NIM Setup - -``` -User: "Set up NIM on my cluster" - -Agent: [Invokes /nim-setup skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand NIM platform requirements. - -Let me verify the prerequisites for NIM setup. - -Step 1 of 7: Checking GPU Operator and NFD Operator... - -GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) -NFD Operator: nfd.v4.19.0 (Succeeded) - -Both operators are installed and healthy. - -Step 2 of 7: I need your NGC credentials. - -To set up NIM, I need your NVIDIA NGC API key. -You can generate one at: https://ngc.nvidia.com/setup/api-key - -Please provide: -1. Your NGC API key -2. The target namespace for NIM resources - -User: "Here's my key: [user provides NGC API key], namespace: ai-models" - -Agent: Step 3 of 7: Creating NGC image pull secret... - -I'll create a docker-registry secret for pulling NIM images from nvcr.io: - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-image-pull-secret - namespace: ai-models - type: kubernetes.io/dockerconfigjson - data: - .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] - -Should I create this image pull secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - Image pull secret created successfully. - -Step 4 of 7: Creating NGC API key secret... - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-api-key - namespace: ai-models - type: Opaque - stringData: - NGC_API_KEY: "[REDACTED]" - -Should I create this API key secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - API key secret created successfully. - -Step 5 of 7: Creating NIM Account CR... - - apiVersion: nim.opendatahub.io/v1 - kind: Account - metadata: - name: nim-account - namespace: ai-models - spec: - apiKeySecret: - name: ngc-api-key - imagePullSecret: - name: ngc-image-pull-secret - -Should I create this NIM Account CR in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates Account CR via resources_create_or_update] - Account CR created successfully. - -Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) - -User: "no" - -Agent: Step 7 of 7: Validating NIM platform readiness... - -NIM Platform Setup Complete - -Account CR: `nim-account` (namespace: ai-models) - Status: Ready - -NGC Credentials: - Image Pull Secret: ngc-image-pull-secret [created] - API Key Secret: ngc-api-key [created] - -NIM ServingRuntimes Available: - - nim-llama-3-1-8b-instruct - - nim-llama-3-1-70b-instruct - - nim-mistral-7b-instruct - -Next Steps: - To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" -``` diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/live-doc-lookup.md b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/references/known-model-profiles.md b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/references/known-model-profiles.md deleted file mode 100644 index 40148981..00000000 --- a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/references/known-model-profiles.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Known Model Hardware Profiles -category: references -tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] -semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] -use_cases: [model-deploy, debug-inference, ai-observability] -last_updated: 2026-03-10 ---- - -# Known Model Hardware Profiles - -Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). - -**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. - -## Model Source Conventions - -Each model lists a recommended `storageUri` with its authentication requirements: -- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** -- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). -- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. - -When the user does not specify a model source, use the `hf://` URI listed in the profile below. - -## Llama 3.x (Meta) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | -| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | -| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | -| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | - -- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) -- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly - -## Granite 3.x (IBM/Red Hat) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | -| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | - -- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) -- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser granite --chat-template` -- Red Hat-supported model family on RHOAI - -## Mixtral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | -| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) -- Mixture-of-Experts architecture: only ~13B/45B parameters active per token - -## Mistral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | -| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) - -## When a Model Is Not Listed - -If the requested model is not in this file, the agent MUST use the live doc lookup protocol: - -1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure -2. Fetch hardware specs from the appropriate source -3. Report findings to the user before proceeding with deployment - -Common cases requiring live lookup: -- Newly released models (after this file's last update) -- Domain-specific fine-tuned models -- Models with custom quantization -- NIM-specific optimized profiles diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/references/supported-runtimes.md b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/references/supported-runtimes.md deleted file mode 100644 index 9d018651..00000000 --- a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/references/supported-runtimes.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: Supported Serving Runtimes -category: references -tags: [runtimes, vllm, nim, caikit, tgis, serving] -semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] -use_cases: [model-deploy, serving-runtime-config, nim-setup] -last_updated: 2026-02-26 ---- - -# Supported Serving Runtimes - -This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. - -## Runtime Comparison - -| Runtime | API Type | Model Format | GPU Support | License | Setup Required | -|---------|----------|-------------|-------------|---------|----------------| -| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | -| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | -| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | -| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | - -## vLLM - -**Default runtime for most open-source models.** - -- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) -- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized -- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) -- **Model source**: S3-compatible storage, OCI registry, PVC, URI -- **Key features**: - - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) - - PagedAttention for efficient memory management - - Tensor parallelism for multi-GPU inference - - Tool/function calling support (`--tool-call-parser`) - - Continuous batching for high throughput -- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name - -## NVIDIA NIM - -**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** - -- **API**: OpenAI-compatible REST -- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog -- **GPU support**: NVIDIA only (requires specific GPU models per model profile) -- **Model source**: NGC model catalog (pulled automatically via NGC credentials) -- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) -- **Key features**: - - TensorRT-LLM optimization for lower latency - - Pre-compiled model engines (no compilation on first load) - - Optimized and generic profiles per GPU type - - Automatic model download from NGC -- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. -- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM -- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. -- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state - -## Caikit+TGIS - -**Red Hat's Caikit format with Text Generation Inference Server.** - -- **API**: gRPC (not REST) -- **Model formats**: Caikit format (requires conversion from HuggingFace) -- **GPU support**: NVIDIA -- **Model source**: S3-compatible storage -- **Key features**: - - Red Hat-supported runtime - - gRPC API for streaming inference - - Integrated with RHOAI model serving platform -- **When to choose**: When the model is already in Caikit format or when gRPC API is required. -- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name - -## Custom Runtimes - -**User-provided ServingRuntime CRs for unsupported frameworks.** - -- **API**: Defined by the custom runtime -- **Model formats**: Defined by the custom runtime -- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. -- **How to create**: Use `/serving-runtime-config` skill -- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility - -## Runtime Selection Decision Tree - -``` -Is the user's preferred runtime explicitly stated? -├── Yes → Use that runtime -└── No → Continue - -Is the model available in the NGC NIM catalog? -├── Yes → Suggest NIM (with vLLM as fallback) -│ Note: Requires /nim-setup and NVIDIA GPUs -└── No → Continue - -Is the model in Caikit format? -├── Yes → Caikit+TGIS -└── No → Continue - -Is the model a standard open-source LLM (HuggingFace-compatible)? -├── Yes → vLLM (default) -└── No → Custom runtime via /serving-runtime-config -``` diff --git a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/skill-conventions.md b/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/without_skills/rh-ai-engineer__ai-observability/environment/docs/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/common-issues.md b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/examples/model-deploy.md b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/examples/model-deploy.md deleted file mode 100644 index dcd79e44..00000000 --- a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/examples/model-deploy.md +++ /dev/null @@ -1,160 +0,0 @@ -# /model-deploy Examples - -Example interactions demonstrating the model-deploy skill workflow. - -## Example 1: Deploy Llama 3.1 8B with vLLM - -``` -User: "Deploy Llama 3.1 8B on my cluster" - -Agent: [Invokes /model-deploy skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand runtime capabilities. - -Step 1 of 10: Gathering deployment information. - -| Setting | Value | Source | -|---------|-------|--------| -| Model | Llama 3.1 8B Instruct | user input | -| Runtime | TBD | auto-detect | -| Namespace | ? | needs input | -| Model Source | ? | needs input | -| Deployment Mode | Serverless | default | - -What namespace should I deploy to, and where are the model weights stored? -(e.g., S3 bucket, OCI registry, PVC) - -User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" - -Agent: Step 2 of 10: Determining runtime. - -I consulted supported-runtimes.md to understand runtime capabilities. - -Based on Llama 3.1 8B (open-source HuggingFace model): - -Recommended runtime: vLLM -Reason: Default runtime for open-source LLMs. Broad compatibility, - OpenAI-compatible API, no additional setup required. - -Proceed with vLLM? (yes / no / suggest alternative) - -User: "yes" - -Agent: Step 3 of 10: Looking up hardware profile. - -I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. - -| Requirement | Value | -|-------------|-------| -| GPUs | 1x A10G/L4/A100/H100 | -| VRAM | 16GB | -| Key Args | --max-model-len=4096 --tool-call-parser hermes | - -Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). -Ensure your cluster has at least 1 GPU with 16GB VRAM. - -Step 5 of 10: Skipped (NIM platform check not needed for vLLM). - -Step 6-7 of 10: Generating InferenceService YAML. - -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - name: llama-3-1-8b - namespace: ai-models - annotations: - serving.kserve.io/deploymentMode: Serverless -spec: - predictor: - model: - modelFormat: - name: vLLM - runtime: vllm-runtime - storageUri: s3://my-bucket/llama-3.1-8b - resources: - limits: - nvidia.com/gpu: "1" - requests: - cpu: "4" - memory: "16Gi" - args: - - --max-model-len=4096 - - --tool-call-parser=hermes - -| Setting | Value | -|---------|-------| -| Model | llama-3-1-8b | -| Runtime | vllm-runtime | -| GPUs | 1 | -| Namespace | ai-models | -| Mode | Serverless | - -Proceed with creating this InferenceService? (yes/no/modify) - -User: "yes" - -Agent: Step 8 of 10: Creating InferenceService... - [Creates via resources_create_or_update] - - InferenceService created. Monitoring rollout... - -Step 9 of 10: Monitoring rollout. - -| Stage | Status | -|-------|--------| -| Pod Scheduled | Complete | -| Image Pulled | Complete | -| Container Started | Complete | -| Model Loaded | Complete | -| Ready | Complete | - -Step 10 of 10: Deployment complete! - -Model Deployment Successful - -Model: llama-3-1-8b -Runtime: vLLM -Namespace: ai-models -GPUs: 1 - -Inference Endpoint: - URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com - API: OpenAI-compatible REST - -Quick Test: - curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' - -Next Steps: - Monitor performance: "/ai-observability llama-3-1-8b" -``` - -## Example 2: Deploy a Model with NIM (NIM Not Set Up) - -``` -User: "Deploy Llama 3.1 70B with NIM" - -Agent: [Invokes /model-deploy skill] - ... - Step 2: User requested NIM runtime. - - Step 5: Checking NIM platform... - -NIM platform is not set up in namespace ai-models. - -The NIM runtime requires a one-time platform setup. Would you like to: -1. Run /nim-setup now to configure the NIM platform -2. Switch to vLLM runtime instead (no setup required) -3. Abort deployment - -User: "1" - -Agent: [Invokes /nim-setup skill — walks through NGC credential setup] - ... - NIM Platform Setup Complete. - - Resuming model deployment... - [Continues from Step 6 with NIM runtime] -``` diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/examples/nim-setup.md b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/examples/nim-setup.md deleted file mode 100644 index ccfc1a5d..00000000 --- a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/examples/nim-setup.md +++ /dev/null @@ -1,115 +0,0 @@ -# /nim-setup Examples - -Example interactions demonstrating the nim-setup skill workflow. - -## Example 1: First-Time NIM Setup - -``` -User: "Set up NIM on my cluster" - -Agent: [Invokes /nim-setup skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand NIM platform requirements. - -Let me verify the prerequisites for NIM setup. - -Step 1 of 7: Checking GPU Operator and NFD Operator... - -GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) -NFD Operator: nfd.v4.19.0 (Succeeded) - -Both operators are installed and healthy. - -Step 2 of 7: I need your NGC credentials. - -To set up NIM, I need your NVIDIA NGC API key. -You can generate one at: https://ngc.nvidia.com/setup/api-key - -Please provide: -1. Your NGC API key -2. The target namespace for NIM resources - -User: "Here's my key: [user provides NGC API key], namespace: ai-models" - -Agent: Step 3 of 7: Creating NGC image pull secret... - -I'll create a docker-registry secret for pulling NIM images from nvcr.io: - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-image-pull-secret - namespace: ai-models - type: kubernetes.io/dockerconfigjson - data: - .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] - -Should I create this image pull secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - Image pull secret created successfully. - -Step 4 of 7: Creating NGC API key secret... - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-api-key - namespace: ai-models - type: Opaque - stringData: - NGC_API_KEY: "[REDACTED]" - -Should I create this API key secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - API key secret created successfully. - -Step 5 of 7: Creating NIM Account CR... - - apiVersion: nim.opendatahub.io/v1 - kind: Account - metadata: - name: nim-account - namespace: ai-models - spec: - apiKeySecret: - name: ngc-api-key - imagePullSecret: - name: ngc-image-pull-secret - -Should I create this NIM Account CR in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates Account CR via resources_create_or_update] - Account CR created successfully. - -Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) - -User: "no" - -Agent: Step 7 of 7: Validating NIM platform readiness... - -NIM Platform Setup Complete - -Account CR: `nim-account` (namespace: ai-models) - Status: Ready - -NGC Credentials: - Image Pull Secret: ngc-image-pull-secret [created] - API Key Secret: ngc-api-key [created] - -NIM ServingRuntimes Available: - - nim-llama-3-1-8b-instruct - - nim-llama-3-1-70b-instruct - - nim-mistral-7b-instruct - -Next Steps: - To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" -``` diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/live-doc-lookup.md b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/references/known-model-profiles.md b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/references/known-model-profiles.md deleted file mode 100644 index 40148981..00000000 --- a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/references/known-model-profiles.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Known Model Hardware Profiles -category: references -tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] -semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] -use_cases: [model-deploy, debug-inference, ai-observability] -last_updated: 2026-03-10 ---- - -# Known Model Hardware Profiles - -Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). - -**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. - -## Model Source Conventions - -Each model lists a recommended `storageUri` with its authentication requirements: -- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** -- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). -- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. - -When the user does not specify a model source, use the `hf://` URI listed in the profile below. - -## Llama 3.x (Meta) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | -| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | -| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | -| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | - -- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) -- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly - -## Granite 3.x (IBM/Red Hat) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | -| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | - -- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) -- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser granite --chat-template` -- Red Hat-supported model family on RHOAI - -## Mixtral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | -| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) -- Mixture-of-Experts architecture: only ~13B/45B parameters active per token - -## Mistral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | -| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) - -## When a Model Is Not Listed - -If the requested model is not in this file, the agent MUST use the live doc lookup protocol: - -1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure -2. Fetch hardware specs from the appropriate source -3. Report findings to the user before proceeding with deployment - -Common cases requiring live lookup: -- Newly released models (after this file's last update) -- Domain-specific fine-tuned models -- Models with custom quantization -- NIM-specific optimized profiles diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/references/supported-runtimes.md b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/references/supported-runtimes.md deleted file mode 100644 index 9d018651..00000000 --- a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/references/supported-runtimes.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: Supported Serving Runtimes -category: references -tags: [runtimes, vllm, nim, caikit, tgis, serving] -semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] -use_cases: [model-deploy, serving-runtime-config, nim-setup] -last_updated: 2026-02-26 ---- - -# Supported Serving Runtimes - -This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. - -## Runtime Comparison - -| Runtime | API Type | Model Format | GPU Support | License | Setup Required | -|---------|----------|-------------|-------------|---------|----------------| -| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | -| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | -| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | -| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | - -## vLLM - -**Default runtime for most open-source models.** - -- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) -- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized -- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) -- **Model source**: S3-compatible storage, OCI registry, PVC, URI -- **Key features**: - - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) - - PagedAttention for efficient memory management - - Tensor parallelism for multi-GPU inference - - Tool/function calling support (`--tool-call-parser`) - - Continuous batching for high throughput -- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name - -## NVIDIA NIM - -**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** - -- **API**: OpenAI-compatible REST -- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog -- **GPU support**: NVIDIA only (requires specific GPU models per model profile) -- **Model source**: NGC model catalog (pulled automatically via NGC credentials) -- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) -- **Key features**: - - TensorRT-LLM optimization for lower latency - - Pre-compiled model engines (no compilation on first load) - - Optimized and generic profiles per GPU type - - Automatic model download from NGC -- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. -- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM -- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. -- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state - -## Caikit+TGIS - -**Red Hat's Caikit format with Text Generation Inference Server.** - -- **API**: gRPC (not REST) -- **Model formats**: Caikit format (requires conversion from HuggingFace) -- **GPU support**: NVIDIA -- **Model source**: S3-compatible storage -- **Key features**: - - Red Hat-supported runtime - - gRPC API for streaming inference - - Integrated with RHOAI model serving platform -- **When to choose**: When the model is already in Caikit format or when gRPC API is required. -- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name - -## Custom Runtimes - -**User-provided ServingRuntime CRs for unsupported frameworks.** - -- **API**: Defined by the custom runtime -- **Model formats**: Defined by the custom runtime -- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. -- **How to create**: Use `/serving-runtime-config` skill -- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility - -## Runtime Selection Decision Tree - -``` -Is the user's preferred runtime explicitly stated? -├── Yes → Use that runtime -└── No → Continue - -Is the model available in the NGC NIM catalog? -├── Yes → Suggest NIM (with vLLM as fallback) -│ Note: Requires /nim-setup and NVIDIA GPUs -└── No → Continue - -Is the model in Caikit format? -├── Yes → Caikit+TGIS -└── No → Continue - -Is the model a standard open-source LLM (HuggingFace-compatible)? -├── Yes → vLLM (default) -└── No → Custom runtime via /serving-runtime-config -``` diff --git a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/skill-conventions.md b/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/without_skills/rh-ai-engineer__debug-inference/environment/docs/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/common-issues.md b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/model-deploy.md b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/model-deploy.md deleted file mode 100644 index dcd79e44..00000000 --- a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/model-deploy.md +++ /dev/null @@ -1,160 +0,0 @@ -# /model-deploy Examples - -Example interactions demonstrating the model-deploy skill workflow. - -## Example 1: Deploy Llama 3.1 8B with vLLM - -``` -User: "Deploy Llama 3.1 8B on my cluster" - -Agent: [Invokes /model-deploy skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand runtime capabilities. - -Step 1 of 10: Gathering deployment information. - -| Setting | Value | Source | -|---------|-------|--------| -| Model | Llama 3.1 8B Instruct | user input | -| Runtime | TBD | auto-detect | -| Namespace | ? | needs input | -| Model Source | ? | needs input | -| Deployment Mode | Serverless | default | - -What namespace should I deploy to, and where are the model weights stored? -(e.g., S3 bucket, OCI registry, PVC) - -User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" - -Agent: Step 2 of 10: Determining runtime. - -I consulted supported-runtimes.md to understand runtime capabilities. - -Based on Llama 3.1 8B (open-source HuggingFace model): - -Recommended runtime: vLLM -Reason: Default runtime for open-source LLMs. Broad compatibility, - OpenAI-compatible API, no additional setup required. - -Proceed with vLLM? (yes / no / suggest alternative) - -User: "yes" - -Agent: Step 3 of 10: Looking up hardware profile. - -I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. - -| Requirement | Value | -|-------------|-------| -| GPUs | 1x A10G/L4/A100/H100 | -| VRAM | 16GB | -| Key Args | --max-model-len=4096 --tool-call-parser hermes | - -Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). -Ensure your cluster has at least 1 GPU with 16GB VRAM. - -Step 5 of 10: Skipped (NIM platform check not needed for vLLM). - -Step 6-7 of 10: Generating InferenceService YAML. - -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - name: llama-3-1-8b - namespace: ai-models - annotations: - serving.kserve.io/deploymentMode: Serverless -spec: - predictor: - model: - modelFormat: - name: vLLM - runtime: vllm-runtime - storageUri: s3://my-bucket/llama-3.1-8b - resources: - limits: - nvidia.com/gpu: "1" - requests: - cpu: "4" - memory: "16Gi" - args: - - --max-model-len=4096 - - --tool-call-parser=hermes - -| Setting | Value | -|---------|-------| -| Model | llama-3-1-8b | -| Runtime | vllm-runtime | -| GPUs | 1 | -| Namespace | ai-models | -| Mode | Serverless | - -Proceed with creating this InferenceService? (yes/no/modify) - -User: "yes" - -Agent: Step 8 of 10: Creating InferenceService... - [Creates via resources_create_or_update] - - InferenceService created. Monitoring rollout... - -Step 9 of 10: Monitoring rollout. - -| Stage | Status | -|-------|--------| -| Pod Scheduled | Complete | -| Image Pulled | Complete | -| Container Started | Complete | -| Model Loaded | Complete | -| Ready | Complete | - -Step 10 of 10: Deployment complete! - -Model Deployment Successful - -Model: llama-3-1-8b -Runtime: vLLM -Namespace: ai-models -GPUs: 1 - -Inference Endpoint: - URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com - API: OpenAI-compatible REST - -Quick Test: - curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' - -Next Steps: - Monitor performance: "/ai-observability llama-3-1-8b" -``` - -## Example 2: Deploy a Model with NIM (NIM Not Set Up) - -``` -User: "Deploy Llama 3.1 70B with NIM" - -Agent: [Invokes /model-deploy skill] - ... - Step 2: User requested NIM runtime. - - Step 5: Checking NIM platform... - -NIM platform is not set up in namespace ai-models. - -The NIM runtime requires a one-time platform setup. Would you like to: -1. Run /nim-setup now to configure the NIM platform -2. Switch to vLLM runtime instead (no setup required) -3. Abort deployment - -User: "1" - -Agent: [Invokes /nim-setup skill — walks through NGC credential setup] - ... - NIM Platform Setup Complete. - - Resuming model deployment... - [Continues from Step 6 with NIM runtime] -``` diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/nim-setup.md b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/nim-setup.md deleted file mode 100644 index ccfc1a5d..00000000 --- a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/examples/nim-setup.md +++ /dev/null @@ -1,115 +0,0 @@ -# /nim-setup Examples - -Example interactions demonstrating the nim-setup skill workflow. - -## Example 1: First-Time NIM Setup - -``` -User: "Set up NIM on my cluster" - -Agent: [Invokes /nim-setup skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand NIM platform requirements. - -Let me verify the prerequisites for NIM setup. - -Step 1 of 7: Checking GPU Operator and NFD Operator... - -GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) -NFD Operator: nfd.v4.19.0 (Succeeded) - -Both operators are installed and healthy. - -Step 2 of 7: I need your NGC credentials. - -To set up NIM, I need your NVIDIA NGC API key. -You can generate one at: https://ngc.nvidia.com/setup/api-key - -Please provide: -1. Your NGC API key -2. The target namespace for NIM resources - -User: "Here's my key: [user provides NGC API key], namespace: ai-models" - -Agent: Step 3 of 7: Creating NGC image pull secret... - -I'll create a docker-registry secret for pulling NIM images from nvcr.io: - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-image-pull-secret - namespace: ai-models - type: kubernetes.io/dockerconfigjson - data: - .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] - -Should I create this image pull secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - Image pull secret created successfully. - -Step 4 of 7: Creating NGC API key secret... - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-api-key - namespace: ai-models - type: Opaque - stringData: - NGC_API_KEY: "[REDACTED]" - -Should I create this API key secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - API key secret created successfully. - -Step 5 of 7: Creating NIM Account CR... - - apiVersion: nim.opendatahub.io/v1 - kind: Account - metadata: - name: nim-account - namespace: ai-models - spec: - apiKeySecret: - name: ngc-api-key - imagePullSecret: - name: ngc-image-pull-secret - -Should I create this NIM Account CR in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates Account CR via resources_create_or_update] - Account CR created successfully. - -Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) - -User: "no" - -Agent: Step 7 of 7: Validating NIM platform readiness... - -NIM Platform Setup Complete - -Account CR: `nim-account` (namespace: ai-models) - Status: Ready - -NGC Credentials: - Image Pull Secret: ngc-image-pull-secret [created] - API Key Secret: ngc-api-key [created] - -NIM ServingRuntimes Available: - - nim-llama-3-1-8b-instruct - - nim-llama-3-1-70b-instruct - - nim-mistral-7b-instruct - -Next Steps: - To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" -``` diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/live-doc-lookup.md b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/known-model-profiles.md b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/known-model-profiles.md deleted file mode 100644 index 40148981..00000000 --- a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/known-model-profiles.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Known Model Hardware Profiles -category: references -tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] -semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] -use_cases: [model-deploy, debug-inference, ai-observability] -last_updated: 2026-03-10 ---- - -# Known Model Hardware Profiles - -Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). - -**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. - -## Model Source Conventions - -Each model lists a recommended `storageUri` with its authentication requirements: -- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** -- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). -- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. - -When the user does not specify a model source, use the `hf://` URI listed in the profile below. - -## Llama 3.x (Meta) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | -| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | -| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | -| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | - -- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) -- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly - -## Granite 3.x (IBM/Red Hat) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | -| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | - -- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) -- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser granite --chat-template` -- Red Hat-supported model family on RHOAI - -## Mixtral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | -| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) -- Mixture-of-Experts architecture: only ~13B/45B parameters active per token - -## Mistral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | -| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) - -## When a Model Is Not Listed - -If the requested model is not in this file, the agent MUST use the live doc lookup protocol: - -1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure -2. Fetch hardware specs from the appropriate source -3. Report findings to the user before proceeding with deployment - -Common cases requiring live lookup: -- Newly released models (after this file's last update) -- Domain-specific fine-tuned models -- Models with custom quantization -- NIM-specific optimized profiles diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/supported-runtimes.md b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/supported-runtimes.md deleted file mode 100644 index 9d018651..00000000 --- a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/references/supported-runtimes.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: Supported Serving Runtimes -category: references -tags: [runtimes, vllm, nim, caikit, tgis, serving] -semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] -use_cases: [model-deploy, serving-runtime-config, nim-setup] -last_updated: 2026-02-26 ---- - -# Supported Serving Runtimes - -This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. - -## Runtime Comparison - -| Runtime | API Type | Model Format | GPU Support | License | Setup Required | -|---------|----------|-------------|-------------|---------|----------------| -| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | -| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | -| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | -| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | - -## vLLM - -**Default runtime for most open-source models.** - -- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) -- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized -- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) -- **Model source**: S3-compatible storage, OCI registry, PVC, URI -- **Key features**: - - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) - - PagedAttention for efficient memory management - - Tensor parallelism for multi-GPU inference - - Tool/function calling support (`--tool-call-parser`) - - Continuous batching for high throughput -- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name - -## NVIDIA NIM - -**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** - -- **API**: OpenAI-compatible REST -- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog -- **GPU support**: NVIDIA only (requires specific GPU models per model profile) -- **Model source**: NGC model catalog (pulled automatically via NGC credentials) -- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) -- **Key features**: - - TensorRT-LLM optimization for lower latency - - Pre-compiled model engines (no compilation on first load) - - Optimized and generic profiles per GPU type - - Automatic model download from NGC -- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. -- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM -- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. -- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state - -## Caikit+TGIS - -**Red Hat's Caikit format with Text Generation Inference Server.** - -- **API**: gRPC (not REST) -- **Model formats**: Caikit format (requires conversion from HuggingFace) -- **GPU support**: NVIDIA -- **Model source**: S3-compatible storage -- **Key features**: - - Red Hat-supported runtime - - gRPC API for streaming inference - - Integrated with RHOAI model serving platform -- **When to choose**: When the model is already in Caikit format or when gRPC API is required. -- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name - -## Custom Runtimes - -**User-provided ServingRuntime CRs for unsupported frameworks.** - -- **API**: Defined by the custom runtime -- **Model formats**: Defined by the custom runtime -- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. -- **How to create**: Use `/serving-runtime-config` skill -- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility - -## Runtime Selection Decision Tree - -``` -Is the user's preferred runtime explicitly stated? -├── Yes → Use that runtime -└── No → Continue - -Is the model available in the NGC NIM catalog? -├── Yes → Suggest NIM (with vLLM as fallback) -│ Note: Requires /nim-setup and NVIDIA GPUs -└── No → Continue - -Is the model in Caikit format? -├── Yes → Caikit+TGIS -└── No → Continue - -Is the model a standard open-source LLM (HuggingFace-compatible)? -├── Yes → vLLM (default) -└── No → Custom runtime via /serving-runtime-config -``` diff --git a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/skill-conventions.md b/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/without_skills/rh-ai-engineer__ds-project-setup/environment/docs/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/common-issues.md b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/examples/model-deploy.md b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/examples/model-deploy.md deleted file mode 100644 index dcd79e44..00000000 --- a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/examples/model-deploy.md +++ /dev/null @@ -1,160 +0,0 @@ -# /model-deploy Examples - -Example interactions demonstrating the model-deploy skill workflow. - -## Example 1: Deploy Llama 3.1 8B with vLLM - -``` -User: "Deploy Llama 3.1 8B on my cluster" - -Agent: [Invokes /model-deploy skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand runtime capabilities. - -Step 1 of 10: Gathering deployment information. - -| Setting | Value | Source | -|---------|-------|--------| -| Model | Llama 3.1 8B Instruct | user input | -| Runtime | TBD | auto-detect | -| Namespace | ? | needs input | -| Model Source | ? | needs input | -| Deployment Mode | Serverless | default | - -What namespace should I deploy to, and where are the model weights stored? -(e.g., S3 bucket, OCI registry, PVC) - -User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" - -Agent: Step 2 of 10: Determining runtime. - -I consulted supported-runtimes.md to understand runtime capabilities. - -Based on Llama 3.1 8B (open-source HuggingFace model): - -Recommended runtime: vLLM -Reason: Default runtime for open-source LLMs. Broad compatibility, - OpenAI-compatible API, no additional setup required. - -Proceed with vLLM? (yes / no / suggest alternative) - -User: "yes" - -Agent: Step 3 of 10: Looking up hardware profile. - -I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. - -| Requirement | Value | -|-------------|-------| -| GPUs | 1x A10G/L4/A100/H100 | -| VRAM | 16GB | -| Key Args | --max-model-len=4096 --tool-call-parser hermes | - -Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). -Ensure your cluster has at least 1 GPU with 16GB VRAM. - -Step 5 of 10: Skipped (NIM platform check not needed for vLLM). - -Step 6-7 of 10: Generating InferenceService YAML. - -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - name: llama-3-1-8b - namespace: ai-models - annotations: - serving.kserve.io/deploymentMode: Serverless -spec: - predictor: - model: - modelFormat: - name: vLLM - runtime: vllm-runtime - storageUri: s3://my-bucket/llama-3.1-8b - resources: - limits: - nvidia.com/gpu: "1" - requests: - cpu: "4" - memory: "16Gi" - args: - - --max-model-len=4096 - - --tool-call-parser=hermes - -| Setting | Value | -|---------|-------| -| Model | llama-3-1-8b | -| Runtime | vllm-runtime | -| GPUs | 1 | -| Namespace | ai-models | -| Mode | Serverless | - -Proceed with creating this InferenceService? (yes/no/modify) - -User: "yes" - -Agent: Step 8 of 10: Creating InferenceService... - [Creates via resources_create_or_update] - - InferenceService created. Monitoring rollout... - -Step 9 of 10: Monitoring rollout. - -| Stage | Status | -|-------|--------| -| Pod Scheduled | Complete | -| Image Pulled | Complete | -| Container Started | Complete | -| Model Loaded | Complete | -| Ready | Complete | - -Step 10 of 10: Deployment complete! - -Model Deployment Successful - -Model: llama-3-1-8b -Runtime: vLLM -Namespace: ai-models -GPUs: 1 - -Inference Endpoint: - URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com - API: OpenAI-compatible REST - -Quick Test: - curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' - -Next Steps: - Monitor performance: "/ai-observability llama-3-1-8b" -``` - -## Example 2: Deploy a Model with NIM (NIM Not Set Up) - -``` -User: "Deploy Llama 3.1 70B with NIM" - -Agent: [Invokes /model-deploy skill] - ... - Step 2: User requested NIM runtime. - - Step 5: Checking NIM platform... - -NIM platform is not set up in namespace ai-models. - -The NIM runtime requires a one-time platform setup. Would you like to: -1. Run /nim-setup now to configure the NIM platform -2. Switch to vLLM runtime instead (no setup required) -3. Abort deployment - -User: "1" - -Agent: [Invokes /nim-setup skill — walks through NGC credential setup] - ... - NIM Platform Setup Complete. - - Resuming model deployment... - [Continues from Step 6 with NIM runtime] -``` diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/examples/nim-setup.md b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/examples/nim-setup.md deleted file mode 100644 index ccfc1a5d..00000000 --- a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/examples/nim-setup.md +++ /dev/null @@ -1,115 +0,0 @@ -# /nim-setup Examples - -Example interactions demonstrating the nim-setup skill workflow. - -## Example 1: First-Time NIM Setup - -``` -User: "Set up NIM on my cluster" - -Agent: [Invokes /nim-setup skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand NIM platform requirements. - -Let me verify the prerequisites for NIM setup. - -Step 1 of 7: Checking GPU Operator and NFD Operator... - -GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) -NFD Operator: nfd.v4.19.0 (Succeeded) - -Both operators are installed and healthy. - -Step 2 of 7: I need your NGC credentials. - -To set up NIM, I need your NVIDIA NGC API key. -You can generate one at: https://ngc.nvidia.com/setup/api-key - -Please provide: -1. Your NGC API key -2. The target namespace for NIM resources - -User: "Here's my key: [user provides NGC API key], namespace: ai-models" - -Agent: Step 3 of 7: Creating NGC image pull secret... - -I'll create a docker-registry secret for pulling NIM images from nvcr.io: - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-image-pull-secret - namespace: ai-models - type: kubernetes.io/dockerconfigjson - data: - .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] - -Should I create this image pull secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - Image pull secret created successfully. - -Step 4 of 7: Creating NGC API key secret... - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-api-key - namespace: ai-models - type: Opaque - stringData: - NGC_API_KEY: "[REDACTED]" - -Should I create this API key secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - API key secret created successfully. - -Step 5 of 7: Creating NIM Account CR... - - apiVersion: nim.opendatahub.io/v1 - kind: Account - metadata: - name: nim-account - namespace: ai-models - spec: - apiKeySecret: - name: ngc-api-key - imagePullSecret: - name: ngc-image-pull-secret - -Should I create this NIM Account CR in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates Account CR via resources_create_or_update] - Account CR created successfully. - -Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) - -User: "no" - -Agent: Step 7 of 7: Validating NIM platform readiness... - -NIM Platform Setup Complete - -Account CR: `nim-account` (namespace: ai-models) - Status: Ready - -NGC Credentials: - Image Pull Secret: ngc-image-pull-secret [created] - API Key Secret: ngc-api-key [created] - -NIM ServingRuntimes Available: - - nim-llama-3-1-8b-instruct - - nim-llama-3-1-70b-instruct - - nim-mistral-7b-instruct - -Next Steps: - To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" -``` diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/live-doc-lookup.md b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/references/known-model-profiles.md b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/references/known-model-profiles.md deleted file mode 100644 index 40148981..00000000 --- a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/references/known-model-profiles.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Known Model Hardware Profiles -category: references -tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] -semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] -use_cases: [model-deploy, debug-inference, ai-observability] -last_updated: 2026-03-10 ---- - -# Known Model Hardware Profiles - -Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). - -**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. - -## Model Source Conventions - -Each model lists a recommended `storageUri` with its authentication requirements: -- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** -- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). -- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. - -When the user does not specify a model source, use the `hf://` URI listed in the profile below. - -## Llama 3.x (Meta) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | -| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | -| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | -| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | - -- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) -- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly - -## Granite 3.x (IBM/Red Hat) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | -| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | - -- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) -- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser granite --chat-template` -- Red Hat-supported model family on RHOAI - -## Mixtral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | -| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) -- Mixture-of-Experts architecture: only ~13B/45B parameters active per token - -## Mistral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | -| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) - -## When a Model Is Not Listed - -If the requested model is not in this file, the agent MUST use the live doc lookup protocol: - -1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure -2. Fetch hardware specs from the appropriate source -3. Report findings to the user before proceeding with deployment - -Common cases requiring live lookup: -- Newly released models (after this file's last update) -- Domain-specific fine-tuned models -- Models with custom quantization -- NIM-specific optimized profiles diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/references/supported-runtimes.md b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/references/supported-runtimes.md deleted file mode 100644 index 9d018651..00000000 --- a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/references/supported-runtimes.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: Supported Serving Runtimes -category: references -tags: [runtimes, vllm, nim, caikit, tgis, serving] -semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] -use_cases: [model-deploy, serving-runtime-config, nim-setup] -last_updated: 2026-02-26 ---- - -# Supported Serving Runtimes - -This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. - -## Runtime Comparison - -| Runtime | API Type | Model Format | GPU Support | License | Setup Required | -|---------|----------|-------------|-------------|---------|----------------| -| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | -| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | -| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | -| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | - -## vLLM - -**Default runtime for most open-source models.** - -- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) -- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized -- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) -- **Model source**: S3-compatible storage, OCI registry, PVC, URI -- **Key features**: - - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) - - PagedAttention for efficient memory management - - Tensor parallelism for multi-GPU inference - - Tool/function calling support (`--tool-call-parser`) - - Continuous batching for high throughput -- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name - -## NVIDIA NIM - -**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** - -- **API**: OpenAI-compatible REST -- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog -- **GPU support**: NVIDIA only (requires specific GPU models per model profile) -- **Model source**: NGC model catalog (pulled automatically via NGC credentials) -- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) -- **Key features**: - - TensorRT-LLM optimization for lower latency - - Pre-compiled model engines (no compilation on first load) - - Optimized and generic profiles per GPU type - - Automatic model download from NGC -- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. -- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM -- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. -- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state - -## Caikit+TGIS - -**Red Hat's Caikit format with Text Generation Inference Server.** - -- **API**: gRPC (not REST) -- **Model formats**: Caikit format (requires conversion from HuggingFace) -- **GPU support**: NVIDIA -- **Model source**: S3-compatible storage -- **Key features**: - - Red Hat-supported runtime - - gRPC API for streaming inference - - Integrated with RHOAI model serving platform -- **When to choose**: When the model is already in Caikit format or when gRPC API is required. -- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name - -## Custom Runtimes - -**User-provided ServingRuntime CRs for unsupported frameworks.** - -- **API**: Defined by the custom runtime -- **Model formats**: Defined by the custom runtime -- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. -- **How to create**: Use `/serving-runtime-config` skill -- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility - -## Runtime Selection Decision Tree - -``` -Is the user's preferred runtime explicitly stated? -├── Yes → Use that runtime -└── No → Continue - -Is the model available in the NGC NIM catalog? -├── Yes → Suggest NIM (with vLLM as fallback) -│ Note: Requires /nim-setup and NVIDIA GPUs -└── No → Continue - -Is the model in Caikit format? -├── Yes → Caikit+TGIS -└── No → Continue - -Is the model a standard open-source LLM (HuggingFace-compatible)? -├── Yes → vLLM (default) -└── No → Custom runtime via /serving-runtime-config -``` diff --git a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/skill-conventions.md b/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/without_skills/rh-ai-engineer__model-deploy/environment/docs/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/common-issues.md b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/examples/model-deploy.md b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/examples/model-deploy.md deleted file mode 100644 index dcd79e44..00000000 --- a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/examples/model-deploy.md +++ /dev/null @@ -1,160 +0,0 @@ -# /model-deploy Examples - -Example interactions demonstrating the model-deploy skill workflow. - -## Example 1: Deploy Llama 3.1 8B with vLLM - -``` -User: "Deploy Llama 3.1 8B on my cluster" - -Agent: [Invokes /model-deploy skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand runtime capabilities. - -Step 1 of 10: Gathering deployment information. - -| Setting | Value | Source | -|---------|-------|--------| -| Model | Llama 3.1 8B Instruct | user input | -| Runtime | TBD | auto-detect | -| Namespace | ? | needs input | -| Model Source | ? | needs input | -| Deployment Mode | Serverless | default | - -What namespace should I deploy to, and where are the model weights stored? -(e.g., S3 bucket, OCI registry, PVC) - -User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" - -Agent: Step 2 of 10: Determining runtime. - -I consulted supported-runtimes.md to understand runtime capabilities. - -Based on Llama 3.1 8B (open-source HuggingFace model): - -Recommended runtime: vLLM -Reason: Default runtime for open-source LLMs. Broad compatibility, - OpenAI-compatible API, no additional setup required. - -Proceed with vLLM? (yes / no / suggest alternative) - -User: "yes" - -Agent: Step 3 of 10: Looking up hardware profile. - -I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. - -| Requirement | Value | -|-------------|-------| -| GPUs | 1x A10G/L4/A100/H100 | -| VRAM | 16GB | -| Key Args | --max-model-len=4096 --tool-call-parser hermes | - -Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). -Ensure your cluster has at least 1 GPU with 16GB VRAM. - -Step 5 of 10: Skipped (NIM platform check not needed for vLLM). - -Step 6-7 of 10: Generating InferenceService YAML. - -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - name: llama-3-1-8b - namespace: ai-models - annotations: - serving.kserve.io/deploymentMode: Serverless -spec: - predictor: - model: - modelFormat: - name: vLLM - runtime: vllm-runtime - storageUri: s3://my-bucket/llama-3.1-8b - resources: - limits: - nvidia.com/gpu: "1" - requests: - cpu: "4" - memory: "16Gi" - args: - - --max-model-len=4096 - - --tool-call-parser=hermes - -| Setting | Value | -|---------|-------| -| Model | llama-3-1-8b | -| Runtime | vllm-runtime | -| GPUs | 1 | -| Namespace | ai-models | -| Mode | Serverless | - -Proceed with creating this InferenceService? (yes/no/modify) - -User: "yes" - -Agent: Step 8 of 10: Creating InferenceService... - [Creates via resources_create_or_update] - - InferenceService created. Monitoring rollout... - -Step 9 of 10: Monitoring rollout. - -| Stage | Status | -|-------|--------| -| Pod Scheduled | Complete | -| Image Pulled | Complete | -| Container Started | Complete | -| Model Loaded | Complete | -| Ready | Complete | - -Step 10 of 10: Deployment complete! - -Model Deployment Successful - -Model: llama-3-1-8b -Runtime: vLLM -Namespace: ai-models -GPUs: 1 - -Inference Endpoint: - URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com - API: OpenAI-compatible REST - -Quick Test: - curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' - -Next Steps: - Monitor performance: "/ai-observability llama-3-1-8b" -``` - -## Example 2: Deploy a Model with NIM (NIM Not Set Up) - -``` -User: "Deploy Llama 3.1 70B with NIM" - -Agent: [Invokes /model-deploy skill] - ... - Step 2: User requested NIM runtime. - - Step 5: Checking NIM platform... - -NIM platform is not set up in namespace ai-models. - -The NIM runtime requires a one-time platform setup. Would you like to: -1. Run /nim-setup now to configure the NIM platform -2. Switch to vLLM runtime instead (no setup required) -3. Abort deployment - -User: "1" - -Agent: [Invokes /nim-setup skill — walks through NGC credential setup] - ... - NIM Platform Setup Complete. - - Resuming model deployment... - [Continues from Step 6 with NIM runtime] -``` diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/examples/nim-setup.md b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/examples/nim-setup.md deleted file mode 100644 index ccfc1a5d..00000000 --- a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/examples/nim-setup.md +++ /dev/null @@ -1,115 +0,0 @@ -# /nim-setup Examples - -Example interactions demonstrating the nim-setup skill workflow. - -## Example 1: First-Time NIM Setup - -``` -User: "Set up NIM on my cluster" - -Agent: [Invokes /nim-setup skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand NIM platform requirements. - -Let me verify the prerequisites for NIM setup. - -Step 1 of 7: Checking GPU Operator and NFD Operator... - -GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) -NFD Operator: nfd.v4.19.0 (Succeeded) - -Both operators are installed and healthy. - -Step 2 of 7: I need your NGC credentials. - -To set up NIM, I need your NVIDIA NGC API key. -You can generate one at: https://ngc.nvidia.com/setup/api-key - -Please provide: -1. Your NGC API key -2. The target namespace for NIM resources - -User: "Here's my key: [user provides NGC API key], namespace: ai-models" - -Agent: Step 3 of 7: Creating NGC image pull secret... - -I'll create a docker-registry secret for pulling NIM images from nvcr.io: - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-image-pull-secret - namespace: ai-models - type: kubernetes.io/dockerconfigjson - data: - .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] - -Should I create this image pull secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - Image pull secret created successfully. - -Step 4 of 7: Creating NGC API key secret... - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-api-key - namespace: ai-models - type: Opaque - stringData: - NGC_API_KEY: "[REDACTED]" - -Should I create this API key secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - API key secret created successfully. - -Step 5 of 7: Creating NIM Account CR... - - apiVersion: nim.opendatahub.io/v1 - kind: Account - metadata: - name: nim-account - namespace: ai-models - spec: - apiKeySecret: - name: ngc-api-key - imagePullSecret: - name: ngc-image-pull-secret - -Should I create this NIM Account CR in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates Account CR via resources_create_or_update] - Account CR created successfully. - -Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) - -User: "no" - -Agent: Step 7 of 7: Validating NIM platform readiness... - -NIM Platform Setup Complete - -Account CR: `nim-account` (namespace: ai-models) - Status: Ready - -NGC Credentials: - Image Pull Secret: ngc-image-pull-secret [created] - API Key Secret: ngc-api-key [created] - -NIM ServingRuntimes Available: - - nim-llama-3-1-8b-instruct - - nim-llama-3-1-70b-instruct - - nim-mistral-7b-instruct - -Next Steps: - To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" -``` diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/live-doc-lookup.md b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/references/known-model-profiles.md b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/references/known-model-profiles.md deleted file mode 100644 index 40148981..00000000 --- a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/references/known-model-profiles.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Known Model Hardware Profiles -category: references -tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] -semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] -use_cases: [model-deploy, debug-inference, ai-observability] -last_updated: 2026-03-10 ---- - -# Known Model Hardware Profiles - -Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). - -**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. - -## Model Source Conventions - -Each model lists a recommended `storageUri` with its authentication requirements: -- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** -- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). -- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. - -When the user does not specify a model source, use the `hf://` URI listed in the profile below. - -## Llama 3.x (Meta) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | -| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | -| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | -| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | - -- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) -- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly - -## Granite 3.x (IBM/Red Hat) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | -| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | - -- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) -- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser granite --chat-template` -- Red Hat-supported model family on RHOAI - -## Mixtral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | -| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) -- Mixture-of-Experts architecture: only ~13B/45B parameters active per token - -## Mistral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | -| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) - -## When a Model Is Not Listed - -If the requested model is not in this file, the agent MUST use the live doc lookup protocol: - -1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure -2. Fetch hardware specs from the appropriate source -3. Report findings to the user before proceeding with deployment - -Common cases requiring live lookup: -- Newly released models (after this file's last update) -- Domain-specific fine-tuned models -- Models with custom quantization -- NIM-specific optimized profiles diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/references/supported-runtimes.md b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/references/supported-runtimes.md deleted file mode 100644 index 9d018651..00000000 --- a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/references/supported-runtimes.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: Supported Serving Runtimes -category: references -tags: [runtimes, vllm, nim, caikit, tgis, serving] -semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] -use_cases: [model-deploy, serving-runtime-config, nim-setup] -last_updated: 2026-02-26 ---- - -# Supported Serving Runtimes - -This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. - -## Runtime Comparison - -| Runtime | API Type | Model Format | GPU Support | License | Setup Required | -|---------|----------|-------------|-------------|---------|----------------| -| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | -| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | -| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | -| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | - -## vLLM - -**Default runtime for most open-source models.** - -- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) -- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized -- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) -- **Model source**: S3-compatible storage, OCI registry, PVC, URI -- **Key features**: - - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) - - PagedAttention for efficient memory management - - Tensor parallelism for multi-GPU inference - - Tool/function calling support (`--tool-call-parser`) - - Continuous batching for high throughput -- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name - -## NVIDIA NIM - -**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** - -- **API**: OpenAI-compatible REST -- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog -- **GPU support**: NVIDIA only (requires specific GPU models per model profile) -- **Model source**: NGC model catalog (pulled automatically via NGC credentials) -- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) -- **Key features**: - - TensorRT-LLM optimization for lower latency - - Pre-compiled model engines (no compilation on first load) - - Optimized and generic profiles per GPU type - - Automatic model download from NGC -- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. -- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM -- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. -- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state - -## Caikit+TGIS - -**Red Hat's Caikit format with Text Generation Inference Server.** - -- **API**: gRPC (not REST) -- **Model formats**: Caikit format (requires conversion from HuggingFace) -- **GPU support**: NVIDIA -- **Model source**: S3-compatible storage -- **Key features**: - - Red Hat-supported runtime - - gRPC API for streaming inference - - Integrated with RHOAI model serving platform -- **When to choose**: When the model is already in Caikit format or when gRPC API is required. -- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name - -## Custom Runtimes - -**User-provided ServingRuntime CRs for unsupported frameworks.** - -- **API**: Defined by the custom runtime -- **Model formats**: Defined by the custom runtime -- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. -- **How to create**: Use `/serving-runtime-config` skill -- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility - -## Runtime Selection Decision Tree - -``` -Is the user's preferred runtime explicitly stated? -├── Yes → Use that runtime -└── No → Continue - -Is the model available in the NGC NIM catalog? -├── Yes → Suggest NIM (with vLLM as fallback) -│ Note: Requires /nim-setup and NVIDIA GPUs -└── No → Continue - -Is the model in Caikit format? -├── Yes → Caikit+TGIS -└── No → Continue - -Is the model a standard open-source LLM (HuggingFace-compatible)? -├── Yes → vLLM (default) -└── No → Custom runtime via /serving-runtime-config -``` diff --git a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/skill-conventions.md b/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/without_skills/rh-ai-engineer__nim-setup/environment/docs/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/common-issues.md b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/model-deploy.md b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/model-deploy.md deleted file mode 100644 index dcd79e44..00000000 --- a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/model-deploy.md +++ /dev/null @@ -1,160 +0,0 @@ -# /model-deploy Examples - -Example interactions demonstrating the model-deploy skill workflow. - -## Example 1: Deploy Llama 3.1 8B with vLLM - -``` -User: "Deploy Llama 3.1 8B on my cluster" - -Agent: [Invokes /model-deploy skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand runtime capabilities. - -Step 1 of 10: Gathering deployment information. - -| Setting | Value | Source | -|---------|-------|--------| -| Model | Llama 3.1 8B Instruct | user input | -| Runtime | TBD | auto-detect | -| Namespace | ? | needs input | -| Model Source | ? | needs input | -| Deployment Mode | Serverless | default | - -What namespace should I deploy to, and where are the model weights stored? -(e.g., S3 bucket, OCI registry, PVC) - -User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" - -Agent: Step 2 of 10: Determining runtime. - -I consulted supported-runtimes.md to understand runtime capabilities. - -Based on Llama 3.1 8B (open-source HuggingFace model): - -Recommended runtime: vLLM -Reason: Default runtime for open-source LLMs. Broad compatibility, - OpenAI-compatible API, no additional setup required. - -Proceed with vLLM? (yes / no / suggest alternative) - -User: "yes" - -Agent: Step 3 of 10: Looking up hardware profile. - -I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. - -| Requirement | Value | -|-------------|-------| -| GPUs | 1x A10G/L4/A100/H100 | -| VRAM | 16GB | -| Key Args | --max-model-len=4096 --tool-call-parser hermes | - -Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). -Ensure your cluster has at least 1 GPU with 16GB VRAM. - -Step 5 of 10: Skipped (NIM platform check not needed for vLLM). - -Step 6-7 of 10: Generating InferenceService YAML. - -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - name: llama-3-1-8b - namespace: ai-models - annotations: - serving.kserve.io/deploymentMode: Serverless -spec: - predictor: - model: - modelFormat: - name: vLLM - runtime: vllm-runtime - storageUri: s3://my-bucket/llama-3.1-8b - resources: - limits: - nvidia.com/gpu: "1" - requests: - cpu: "4" - memory: "16Gi" - args: - - --max-model-len=4096 - - --tool-call-parser=hermes - -| Setting | Value | -|---------|-------| -| Model | llama-3-1-8b | -| Runtime | vllm-runtime | -| GPUs | 1 | -| Namespace | ai-models | -| Mode | Serverless | - -Proceed with creating this InferenceService? (yes/no/modify) - -User: "yes" - -Agent: Step 8 of 10: Creating InferenceService... - [Creates via resources_create_or_update] - - InferenceService created. Monitoring rollout... - -Step 9 of 10: Monitoring rollout. - -| Stage | Status | -|-------|--------| -| Pod Scheduled | Complete | -| Image Pulled | Complete | -| Container Started | Complete | -| Model Loaded | Complete | -| Ready | Complete | - -Step 10 of 10: Deployment complete! - -Model Deployment Successful - -Model: llama-3-1-8b -Runtime: vLLM -Namespace: ai-models -GPUs: 1 - -Inference Endpoint: - URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com - API: OpenAI-compatible REST - -Quick Test: - curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' - -Next Steps: - Monitor performance: "/ai-observability llama-3-1-8b" -``` - -## Example 2: Deploy a Model with NIM (NIM Not Set Up) - -``` -User: "Deploy Llama 3.1 70B with NIM" - -Agent: [Invokes /model-deploy skill] - ... - Step 2: User requested NIM runtime. - - Step 5: Checking NIM platform... - -NIM platform is not set up in namespace ai-models. - -The NIM runtime requires a one-time platform setup. Would you like to: -1. Run /nim-setup now to configure the NIM platform -2. Switch to vLLM runtime instead (no setup required) -3. Abort deployment - -User: "1" - -Agent: [Invokes /nim-setup skill — walks through NGC credential setup] - ... - NIM Platform Setup Complete. - - Resuming model deployment... - [Continues from Step 6 with NIM runtime] -``` diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/nim-setup.md b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/nim-setup.md deleted file mode 100644 index ccfc1a5d..00000000 --- a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/examples/nim-setup.md +++ /dev/null @@ -1,115 +0,0 @@ -# /nim-setup Examples - -Example interactions demonstrating the nim-setup skill workflow. - -## Example 1: First-Time NIM Setup - -``` -User: "Set up NIM on my cluster" - -Agent: [Invokes /nim-setup skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand NIM platform requirements. - -Let me verify the prerequisites for NIM setup. - -Step 1 of 7: Checking GPU Operator and NFD Operator... - -GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) -NFD Operator: nfd.v4.19.0 (Succeeded) - -Both operators are installed and healthy. - -Step 2 of 7: I need your NGC credentials. - -To set up NIM, I need your NVIDIA NGC API key. -You can generate one at: https://ngc.nvidia.com/setup/api-key - -Please provide: -1. Your NGC API key -2. The target namespace for NIM resources - -User: "Here's my key: [user provides NGC API key], namespace: ai-models" - -Agent: Step 3 of 7: Creating NGC image pull secret... - -I'll create a docker-registry secret for pulling NIM images from nvcr.io: - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-image-pull-secret - namespace: ai-models - type: kubernetes.io/dockerconfigjson - data: - .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] - -Should I create this image pull secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - Image pull secret created successfully. - -Step 4 of 7: Creating NGC API key secret... - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-api-key - namespace: ai-models - type: Opaque - stringData: - NGC_API_KEY: "[REDACTED]" - -Should I create this API key secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - API key secret created successfully. - -Step 5 of 7: Creating NIM Account CR... - - apiVersion: nim.opendatahub.io/v1 - kind: Account - metadata: - name: nim-account - namespace: ai-models - spec: - apiKeySecret: - name: ngc-api-key - imagePullSecret: - name: ngc-image-pull-secret - -Should I create this NIM Account CR in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates Account CR via resources_create_or_update] - Account CR created successfully. - -Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) - -User: "no" - -Agent: Step 7 of 7: Validating NIM platform readiness... - -NIM Platform Setup Complete - -Account CR: `nim-account` (namespace: ai-models) - Status: Ready - -NGC Credentials: - Image Pull Secret: ngc-image-pull-secret [created] - API Key Secret: ngc-api-key [created] - -NIM ServingRuntimes Available: - - nim-llama-3-1-8b-instruct - - nim-llama-3-1-70b-instruct - - nim-mistral-7b-instruct - -Next Steps: - To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" -``` diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/live-doc-lookup.md b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/known-model-profiles.md b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/known-model-profiles.md deleted file mode 100644 index 40148981..00000000 --- a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/known-model-profiles.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Known Model Hardware Profiles -category: references -tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] -semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] -use_cases: [model-deploy, debug-inference, ai-observability] -last_updated: 2026-03-10 ---- - -# Known Model Hardware Profiles - -Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). - -**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. - -## Model Source Conventions - -Each model lists a recommended `storageUri` with its authentication requirements: -- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** -- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). -- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. - -When the user does not specify a model source, use the `hf://` URI listed in the profile below. - -## Llama 3.x (Meta) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | -| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | -| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | -| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | - -- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) -- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly - -## Granite 3.x (IBM/Red Hat) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | -| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | - -- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) -- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser granite --chat-template` -- Red Hat-supported model family on RHOAI - -## Mixtral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | -| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) -- Mixture-of-Experts architecture: only ~13B/45B parameters active per token - -## Mistral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | -| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) - -## When a Model Is Not Listed - -If the requested model is not in this file, the agent MUST use the live doc lookup protocol: - -1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure -2. Fetch hardware specs from the appropriate source -3. Report findings to the user before proceeding with deployment - -Common cases requiring live lookup: -- Newly released models (after this file's last update) -- Domain-specific fine-tuned models -- Models with custom quantization -- NIM-specific optimized profiles diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/supported-runtimes.md b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/supported-runtimes.md deleted file mode 100644 index 9d018651..00000000 --- a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/references/supported-runtimes.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: Supported Serving Runtimes -category: references -tags: [runtimes, vllm, nim, caikit, tgis, serving] -semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] -use_cases: [model-deploy, serving-runtime-config, nim-setup] -last_updated: 2026-02-26 ---- - -# Supported Serving Runtimes - -This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. - -## Runtime Comparison - -| Runtime | API Type | Model Format | GPU Support | License | Setup Required | -|---------|----------|-------------|-------------|---------|----------------| -| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | -| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | -| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | -| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | - -## vLLM - -**Default runtime for most open-source models.** - -- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) -- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized -- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) -- **Model source**: S3-compatible storage, OCI registry, PVC, URI -- **Key features**: - - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) - - PagedAttention for efficient memory management - - Tensor parallelism for multi-GPU inference - - Tool/function calling support (`--tool-call-parser`) - - Continuous batching for high throughput -- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name - -## NVIDIA NIM - -**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** - -- **API**: OpenAI-compatible REST -- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog -- **GPU support**: NVIDIA only (requires specific GPU models per model profile) -- **Model source**: NGC model catalog (pulled automatically via NGC credentials) -- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) -- **Key features**: - - TensorRT-LLM optimization for lower latency - - Pre-compiled model engines (no compilation on first load) - - Optimized and generic profiles per GPU type - - Automatic model download from NGC -- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. -- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM -- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. -- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state - -## Caikit+TGIS - -**Red Hat's Caikit format with Text Generation Inference Server.** - -- **API**: gRPC (not REST) -- **Model formats**: Caikit format (requires conversion from HuggingFace) -- **GPU support**: NVIDIA -- **Model source**: S3-compatible storage -- **Key features**: - - Red Hat-supported runtime - - gRPC API for streaming inference - - Integrated with RHOAI model serving platform -- **When to choose**: When the model is already in Caikit format or when gRPC API is required. -- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name - -## Custom Runtimes - -**User-provided ServingRuntime CRs for unsupported frameworks.** - -- **API**: Defined by the custom runtime -- **Model formats**: Defined by the custom runtime -- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. -- **How to create**: Use `/serving-runtime-config` skill -- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility - -## Runtime Selection Decision Tree - -``` -Is the user's preferred runtime explicitly stated? -├── Yes → Use that runtime -└── No → Continue - -Is the model available in the NGC NIM catalog? -├── Yes → Suggest NIM (with vLLM as fallback) -│ Note: Requires /nim-setup and NVIDIA GPUs -└── No → Continue - -Is the model in Caikit format? -├── Yes → Caikit+TGIS -└── No → Continue - -Is the model a standard open-source LLM (HuggingFace-compatible)? -├── Yes → vLLM (default) -└── No → Custom runtime via /serving-runtime-config -``` diff --git a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/skill-conventions.md b/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/without_skills/rh-ai-engineer__serving-runtime-config/environment/docs/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/common-issues.md b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/common-issues.md deleted file mode 100644 index dbefbc81..00000000 --- a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/common-issues.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Common Issues Across Skills -category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 ---- - -# Common Issues Across Skills - -Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. Individual skills reference this document and add skill-specific issues inline. - -## GPU Scheduling Failure - -**Applies to**: `/model-deploy`, `/debug-inference`, `/workbench-manage` - -**Error**: Pod stuck in Pending with events showing "Insufficient nvidia.com/gpu" - -**Cause**: Cluster does not have enough available GPUs of the required type. - -**Solution:** -1. Check GPU availability: `get_gpu_info` from ai-observability (if available) or inspect node resources via `resources_get` -2. Reduce GPU request or use a quantized model variant -3. Check if other workloads are consuming GPU resources -4. Verify GPU Operator and NFD Operator are healthy -5. Consider using fewer GPUs with `--tensor-parallel-size` reduction and quantization - -## OOMKilled During Model or Workbench Loading - -**Applies to**: `/model-deploy`, `/debug-inference` - -**Error**: Pod terminated with OOMKilled exit code, often during initial model weight loading - -**Cause**: Model requires more memory than allocated in resource requests/limits. Common with large models or when `--max-model-len` is set too high. - -**Solution:** -1. Increase memory limits in the InferenceService or workbench spec -2. Reduce `--max-model-len` to lower KV cache memory usage -3. Use a quantized model variant (AWQ/GPTQ/FP8) to reduce memory footprint -4. Verify GPU VRAM is sufficient using `get_gpu_info` -5. Consult [known-model-profiles.md](known-model-profiles.md) for correct resource sizing - -## Image Pull Error from nvcr.io (NIM) - -**Applies to**: `/model-deploy`, `/nim-setup` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` for NIM container images referencing `nvcr.io` - -**Cause**: NGC image pull secret is missing, expired, or not in the correct namespace. - -**Solution:** -1. Verify NGC pull secret exists in the target namespace: `resources_get` for the secret -2. Check that the secret contains valid docker credentials for `nvcr.io` -3. Re-run `/nim-setup` to recreate credentials with a fresh NGC API key -4. Ensure the secret is referenced by the ServiceAccount or Account CR - -## Image Pull Error from OCI Registries - -**Applies to**: `/model-deploy`, `/serving-runtime-config` - -**Error**: Pod fails with `ErrImagePull` or `ImagePullBackOff` with `unauthorized` message for `registry.redhat.io/rhelai1/*` or custom container images - -**Cause**: OCI model images or custom container images require authentication credentials not available in the namespace. - -**Solution:** -1. For `registry.redhat.io/rhelai1/*` models: switch to HuggingFace source (`hf://`) which requires no authentication -- this is the recommended default for public models -2. If OCI source is required: verify entitlements are included in the pull secret -3. For custom images: create an image pull secret and link it to the default ServiceAccount (`oc secrets link default --for=pull`) -4. Verify the image URI and tag are correct - -## RBAC / Permission Errors - -**Applies to**: All skills that create or modify Kubernetes resources - -**Error**: API call returns 403 Forbidden or "insufficient permissions" message - -**Cause**: The service account or user credentials in KUBECONFIG lack the required RBAC roles for the target resource type and namespace. - -**Solution:** -1. Report the specific permission error to the user -2. Identify the required role: which API group, resource, and verb is needed -3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding -4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/model-deploy.md b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/model-deploy.md deleted file mode 100644 index dcd79e44..00000000 --- a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/model-deploy.md +++ /dev/null @@ -1,160 +0,0 @@ -# /model-deploy Examples - -Example interactions demonstrating the model-deploy skill workflow. - -## Example 1: Deploy Llama 3.1 8B with vLLM - -``` -User: "Deploy Llama 3.1 8B on my cluster" - -Agent: [Invokes /model-deploy skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand runtime capabilities. - -Step 1 of 10: Gathering deployment information. - -| Setting | Value | Source | -|---------|-------|--------| -| Model | Llama 3.1 8B Instruct | user input | -| Runtime | TBD | auto-detect | -| Namespace | ? | needs input | -| Model Source | ? | needs input | -| Deployment Mode | Serverless | default | - -What namespace should I deploy to, and where are the model weights stored? -(e.g., S3 bucket, OCI registry, PVC) - -User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" - -Agent: Step 2 of 10: Determining runtime. - -I consulted supported-runtimes.md to understand runtime capabilities. - -Based on Llama 3.1 8B (open-source HuggingFace model): - -Recommended runtime: vLLM -Reason: Default runtime for open-source LLMs. Broad compatibility, - OpenAI-compatible API, no additional setup required. - -Proceed with vLLM? (yes / no / suggest alternative) - -User: "yes" - -Agent: Step 3 of 10: Looking up hardware profile. - -I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. - -| Requirement | Value | -|-------------|-------| -| GPUs | 1x A10G/L4/A100/H100 | -| VRAM | 16GB | -| Key Args | --max-model-len=4096 --tool-call-parser hermes | - -Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). -Ensure your cluster has at least 1 GPU with 16GB VRAM. - -Step 5 of 10: Skipped (NIM platform check not needed for vLLM). - -Step 6-7 of 10: Generating InferenceService YAML. - -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - name: llama-3-1-8b - namespace: ai-models - annotations: - serving.kserve.io/deploymentMode: Serverless -spec: - predictor: - model: - modelFormat: - name: vLLM - runtime: vllm-runtime - storageUri: s3://my-bucket/llama-3.1-8b - resources: - limits: - nvidia.com/gpu: "1" - requests: - cpu: "4" - memory: "16Gi" - args: - - --max-model-len=4096 - - --tool-call-parser=hermes - -| Setting | Value | -|---------|-------| -| Model | llama-3-1-8b | -| Runtime | vllm-runtime | -| GPUs | 1 | -| Namespace | ai-models | -| Mode | Serverless | - -Proceed with creating this InferenceService? (yes/no/modify) - -User: "yes" - -Agent: Step 8 of 10: Creating InferenceService... - [Creates via resources_create_or_update] - - InferenceService created. Monitoring rollout... - -Step 9 of 10: Monitoring rollout. - -| Stage | Status | -|-------|--------| -| Pod Scheduled | Complete | -| Image Pulled | Complete | -| Container Started | Complete | -| Model Loaded | Complete | -| Ready | Complete | - -Step 10 of 10: Deployment complete! - -Model Deployment Successful - -Model: llama-3-1-8b -Runtime: vLLM -Namespace: ai-models -GPUs: 1 - -Inference Endpoint: - URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com - API: OpenAI-compatible REST - -Quick Test: - curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' - -Next Steps: - Monitor performance: "/ai-observability llama-3-1-8b" -``` - -## Example 2: Deploy a Model with NIM (NIM Not Set Up) - -``` -User: "Deploy Llama 3.1 70B with NIM" - -Agent: [Invokes /model-deploy skill] - ... - Step 2: User requested NIM runtime. - - Step 5: Checking NIM platform... - -NIM platform is not set up in namespace ai-models. - -The NIM runtime requires a one-time platform setup. Would you like to: -1. Run /nim-setup now to configure the NIM platform -2. Switch to vLLM runtime instead (no setup required) -3. Abort deployment - -User: "1" - -Agent: [Invokes /nim-setup skill — walks through NGC credential setup] - ... - NIM Platform Setup Complete. - - Resuming model deployment... - [Continues from Step 6 with NIM runtime] -``` diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/nim-setup.md b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/nim-setup.md deleted file mode 100644 index ccfc1a5d..00000000 --- a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/examples/nim-setup.md +++ /dev/null @@ -1,115 +0,0 @@ -# /nim-setup Examples - -Example interactions demonstrating the nim-setup skill workflow. - -## Example 1: First-Time NIM Setup - -``` -User: "Set up NIM on my cluster" - -Agent: [Invokes /nim-setup skill] - [Reads supported-runtimes.md] - -I consulted supported-runtimes.md to understand NIM platform requirements. - -Let me verify the prerequisites for NIM setup. - -Step 1 of 7: Checking GPU Operator and NFD Operator... - -GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) -NFD Operator: nfd.v4.19.0 (Succeeded) - -Both operators are installed and healthy. - -Step 2 of 7: I need your NGC credentials. - -To set up NIM, I need your NVIDIA NGC API key. -You can generate one at: https://ngc.nvidia.com/setup/api-key - -Please provide: -1. Your NGC API key -2. The target namespace for NIM resources - -User: "Here's my key: [user provides NGC API key], namespace: ai-models" - -Agent: Step 3 of 7: Creating NGC image pull secret... - -I'll create a docker-registry secret for pulling NIM images from nvcr.io: - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-image-pull-secret - namespace: ai-models - type: kubernetes.io/dockerconfigjson - data: - .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] - -Should I create this image pull secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - Image pull secret created successfully. - -Step 4 of 7: Creating NGC API key secret... - - apiVersion: v1 - kind: Secret - metadata: - name: ngc-api-key - namespace: ai-models - type: Opaque - stringData: - NGC_API_KEY: "[REDACTED]" - -Should I create this API key secret in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates secret via resources_create_or_update] - API key secret created successfully. - -Step 5 of 7: Creating NIM Account CR... - - apiVersion: nim.opendatahub.io/v1 - kind: Account - metadata: - name: nim-account - namespace: ai-models - spec: - apiKeySecret: - name: ngc-api-key - imagePullSecret: - name: ngc-image-pull-secret - -Should I create this NIM Account CR in namespace `ai-models`? (yes/no) - -User: "yes" - -Agent: [Creates Account CR via resources_create_or_update] - Account CR created successfully. - -Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) - -User: "no" - -Agent: Step 7 of 7: Validating NIM platform readiness... - -NIM Platform Setup Complete - -Account CR: `nim-account` (namespace: ai-models) - Status: Ready - -NGC Credentials: - Image Pull Secret: ngc-image-pull-secret [created] - API Key Secret: ngc-api-key [created] - -NIM ServingRuntimes Available: - - nim-llama-3-1-8b-instruct - - nim-llama-3-1-70b-instruct - - nim-mistral-7b-instruct - -Next Steps: - To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" -``` diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/live-doc-lookup.md b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/live-doc-lookup.md deleted file mode 100644 index f09979aa..00000000 --- a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/live-doc-lookup.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Live Documentation Lookup Protocol -category: references -tags: [live-lookup, webfetch, documentation, models, runtimes] -semantic_keywords: [live doc lookup, external documentation fetch, model discovery, runtime discovery] -use_cases: [model-deploy, nim-setup, serving-runtime-config, debug-inference] -last_updated: 2026-02-26 ---- - -# Live Documentation Lookup Protocol - -This document defines the protocol for fetching external documentation at runtime when the agent encounters an unfamiliar model, runtime, or error. Because OpenShift AI and the NVIDIA NIM ecosystem evolve rapidly, the agent's training data may be stale. This protocol ensures accurate, up-to-date information. - -## When to Trigger Live Lookup - -The agent MUST trigger a live doc lookup when ANY of these conditions are true: - -1. **Unknown model**: The user requests deployment of a model not listed in [known-model-profiles.md](known-model-profiles.md) -2. **Uncertain hardware requirements**: The agent is not confident about GPU type, VRAM, or GPU count for a model -3. **Unfamiliar runtime or feature**: The user mentions a serving runtime configuration the agent is uncertain about -4. **Unrecognized deployment error**: A deployment fails with an error message the agent cannot diagnose from its training data - -## Lookup Targets - -Query these sources in order of relevance. Stop once sufficient information is found. - -### 1. Red Hat OpenShift AI Documentation - -**URL**: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` - -**When to use**: For RHOAI-specific features, supported runtimes, CRD schemas, model catalog entries, and deployment guides. - -**What to extract**: -- Supported serving runtime versions and configurations -- InferenceService CRD schema updates -- Model catalog entries with deployment parameters -- Known issues and workarounds - -### 2. NVIDIA NIM Model Catalog - -**URL**: `https://build.nvidia.com/models` - -**When to use**: For NIM-specific models — hardware profiles, API specs, deployment parameters, GPU compatibility. - -**What to extract**: -- GPU type and count requirements (e.g., "2x A100 80GB") -- Model-specific deployment parameters -- API specifications and endpoints -- Available model profiles (optimized vs. generic) - -### 3. NVIDIA NIM Supported Models Matrix - -**URL**: `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` - -**When to use**: For the definitive list of NIM-supported models with GPU compatibility matrix. - -**What to extract**: -- GPU compatibility per model (which GPU types are supported) -- Model profiles: optimized (TensorRT-LLM) vs. generic (vLLM-based) -- Minimum GPU memory requirements -- Tensor parallelism configuration - -## Lookup Procedure - -### Step 1: Determine the lookup target - -Based on the trigger condition, select the most relevant URL: -- Model hardware requirements → Start with NVIDIA NIM catalog (#2), then RHOAI docs (#1) -- Runtime configuration → RHOAI docs (#1) -- NIM GPU compatibility → NIM supported models matrix (#3) -- General deployment issues → RHOAI docs (#1) - -### Step 2: Fetch the page - -Use the **WebFetch** tool to retrieve the relevant page content. - -### Step 3: Extract relevant information - -Parse the fetched content for: -- GPU type and count requirements -- Model-specific serving parameters (max sequence length, quantization, tensor parallelism) -- Compatible runtimes and their versions -- Known issues or special configuration notes - -### Step 4: Report to user - -**REQUIRED** (Document Consultation Transparency - Design Principle #1): - -Always report what was looked up and from where: - -``` -"I looked up [model-name] on [source-name] to confirm its hardware requirements: -- GPU: [count]x [type] ([VRAM]) -- Key parameters: [list] -- Compatible runtimes: [list]" -``` - -### Step 5: Proceed with deployment - -Use the fetched specs to configure the deployment. The looked-up information takes precedence over any cached or training data. - -## Security Considerations - -- Live lookup URLs are read-only documentation pages -- No credentials are sent to external URLs -- Fetched content is used only for parameter extraction, not executed diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/references/known-model-profiles.md b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/references/known-model-profiles.md deleted file mode 100644 index 40148981..00000000 --- a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/references/known-model-profiles.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Known Model Hardware Profiles -category: references -tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] -semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] -use_cases: [model-deploy, debug-inference, ai-observability] -last_updated: 2026-03-10 ---- - -# Known Model Hardware Profiles - -Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). - -**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. - -## Model Source Conventions - -Each model lists a recommended `storageUri` with its authentication requirements: -- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** -- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). -- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. - -When the user does not specify a model source, use the `hf://` URI listed in the profile below. - -## Llama 3.x (Meta) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | -| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | -| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | -| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | - -- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) -- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly - -## Granite 3.x (IBM/Red Hat) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | -| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | - -- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) -- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) -- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) -- Tool calling: `--tool-call-parser granite --chat-template` -- Red Hat-supported model family on RHOAI - -## Mixtral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | -| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) -- Mixture-of-Experts architecture: only ~13B/45B parameters active per token - -## Mistral (Mistral AI) - -| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | -|---------|-----------|------|----------|------|---------------| -| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | -| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | - -- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) - -## When a Model Is Not Listed - -If the requested model is not in this file, the agent MUST use the live doc lookup protocol: - -1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure -2. Fetch hardware specs from the appropriate source -3. Report findings to the user before proceeding with deployment - -Common cases requiring live lookup: -- Newly released models (after this file's last update) -- Domain-specific fine-tuned models -- Models with custom quantization -- NIM-specific optimized profiles diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/references/supported-runtimes.md b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/references/supported-runtimes.md deleted file mode 100644 index 9d018651..00000000 --- a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/references/supported-runtimes.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: Supported Serving Runtimes -category: references -tags: [runtimes, vllm, nim, caikit, tgis, serving] -semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] -use_cases: [model-deploy, serving-runtime-config, nim-setup] -last_updated: 2026-02-26 ---- - -# Supported Serving Runtimes - -This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. - -## Runtime Comparison - -| Runtime | API Type | Model Format | GPU Support | License | Setup Required | -|---------|----------|-------------|-------------|---------|----------------| -| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | -| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | -| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | -| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | - -## vLLM - -**Default runtime for most open-source models.** - -- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) -- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized -- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) -- **Model source**: S3-compatible storage, OCI registry, PVC, URI -- **Key features**: - - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) - - PagedAttention for efficient memory management - - Tensor parallelism for multi-GPU inference - - Tool/function calling support (`--tool-call-parser`) - - Continuous batching for high throughput -- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name - -## NVIDIA NIM - -**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** - -- **API**: OpenAI-compatible REST -- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog -- **GPU support**: NVIDIA only (requires specific GPU models per model profile) -- **Model source**: NGC model catalog (pulled automatically via NGC credentials) -- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) -- **Key features**: - - TensorRT-LLM optimization for lower latency - - Pre-compiled model engines (no compilation on first load) - - Optimized and generic profiles per GPU type - - Automatic model download from NGC -- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. -- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM -- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. -- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state - -## Caikit+TGIS - -**Red Hat's Caikit format with Text Generation Inference Server.** - -- **API**: gRPC (not REST) -- **Model formats**: Caikit format (requires conversion from HuggingFace) -- **GPU support**: NVIDIA -- **Model source**: S3-compatible storage -- **Key features**: - - Red Hat-supported runtime - - gRPC API for streaming inference - - Integrated with RHOAI model serving platform -- **When to choose**: When the model is already in Caikit format or when gRPC API is required. -- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API -- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name - -## Custom Runtimes - -**User-provided ServingRuntime CRs for unsupported frameworks.** - -- **API**: Defined by the custom runtime -- **Model formats**: Defined by the custom runtime -- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. -- **How to create**: Use `/serving-runtime-config` skill -- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility - -## Runtime Selection Decision Tree - -``` -Is the user's preferred runtime explicitly stated? -├── Yes → Use that runtime -└── No → Continue - -Is the model available in the NGC NIM catalog? -├── Yes → Suggest NIM (with vLLM as fallback) -│ Note: Requires /nim-setup and NVIDIA GPUs -└── No → Continue - -Is the model in Caikit format? -├── Yes → Caikit+TGIS -└── No → Continue - -Is the model a standard open-source LLM (HuggingFace-compatible)? -├── Yes → vLLM (default) -└── No → Custom runtime via /serving-runtime-config -``` diff --git a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/skill-conventions.md b/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/skill-conventions.md deleted file mode 100644 index ada90ecc..00000000 --- a/evaluation/without_skills/rh-ai-engineer__workbench-manage/environment/docs/skill-conventions.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Skill Conventions -category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability] -last_updated: 2026-03-01 ---- - -# rh-ai-engineer Skill Conventions - -Shared conventions for all skills in the rh-ai-engineer agentic collection. - -## Prerequisite Verification Protocol - -Before executing any skill, verify MCP server availability: - -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) - -**When prerequisites fail:** - -1. Stop execution immediately -2. Report the specific missing prerequisite: - ``` - Cannot execute [skill-name]: [specific prerequisite] is not available - - Setup Instructions: - 1. [Server-specific setup steps] - 2. Set required environment variables - 3. Restart Claude Code to reload MCP servers - - Documentation: [link to server docs] - ``` -3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) -4. WAIT for user decision -- never proceed automatically - -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: -- Source: https://github.com/openshift/openshift-mcp-server -- Required env var: `KUBECONFIG` -- Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code - -## Common Prerequisites - -All rh-ai-engineer skills share these baseline prerequisites. Individual skills reference this section instead of repeating them. - -**Required Environment Variables**: -- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access - -**Required Cluster Setup**: -- OpenShift cluster with Red Hat OpenShift AI operator installed -- For model serving skills (`/model-deploy`, `/serving-runtime-config`, `/debug-inference`): KServe model serving platform configured, model serving enabled on the target namespace (label: `opendatahub.io/dashboard: "true"`) -- For NIM runtime: NVIDIA GPU Operator and Node Feature Discovery (NFD) Operator installed - -## Human-in-the-Loop Requirements - -All rh-ai-engineer skills that create or modify Kubernetes resources MUST: - -1. **Display the resource manifest** (with credentials REDACTED) before creation -2. **Ask for explicit confirmation** -- "yes/no" or "yes/no/modify" -3. **WAIT for user response** -- never auto-execute -4. **On failure, present diagnostic options** -- never auto-delete or auto-retry - -**Never:** -- Create resources without user reviewing the manifest -- Display actual credential values (API keys, passwords, tokens) -- Skip confirmation for any resource creation -- Assume approval -- always wait for explicit user confirmation - -**Why This Matters:** -- GPU resources are expensive and may have associated costs -- Deployments may affect other workloads competing for cluster resources -- Credentials grant access to external services (NGC, model registries) - -## Security Conventions - -- **Credentials**: Never display actual values; only report presence/absence -- **Secrets**: Use proper Kubernetes Secret types (`dockerconfigjson`, `Opaque`) -- **KUBECONFIG**: Path and contents never exposed in output -- **Namespace isolation**: All resources created in user-specified namespace only -- **RBAC**: Check for sufficient permissions before attempting resource creation -- **Credential lifecycle**: Advise users to rotate API keys periodically diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/without_skills/rh-developer__containerize-deploy/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__debug-build/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/without_skills/rh-developer__debug-build/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__debug-build/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/without_skills/rh-developer__debug-build/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__debug-build/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/without_skills/rh-developer__debug-build/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__debug-build/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/without_skills/rh-developer__debug-build/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__debug-build/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/without_skills/rh-developer__debug-build/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__debug-build/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/without_skills/rh-developer__debug-build/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__debug-build/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/without_skills/rh-developer__debug-build/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__debug-build/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/without_skills/rh-developer__debug-build/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/without_skills/rh-developer__debug-build/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__debug-build/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/without_skills/rh-developer__debug-build/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__debug-container/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/without_skills/rh-developer__debug-container/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__debug-container/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/without_skills/rh-developer__debug-container/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__debug-container/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/without_skills/rh-developer__debug-container/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__debug-container/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/without_skills/rh-developer__debug-container/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__debug-container/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/without_skills/rh-developer__debug-container/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__debug-container/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/without_skills/rh-developer__debug-container/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__debug-container/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/without_skills/rh-developer__debug-container/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__debug-container/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/without_skills/rh-developer__debug-container/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/without_skills/rh-developer__debug-container/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__debug-container/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/without_skills/rh-developer__debug-container/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__debug-network/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/without_skills/rh-developer__debug-network/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__debug-network/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/without_skills/rh-developer__debug-network/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__debug-network/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/without_skills/rh-developer__debug-network/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__debug-network/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/without_skills/rh-developer__debug-network/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__debug-network/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/without_skills/rh-developer__debug-network/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__debug-network/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/without_skills/rh-developer__debug-network/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__debug-network/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/without_skills/rh-developer__debug-network/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__debug-network/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/without_skills/rh-developer__debug-network/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/without_skills/rh-developer__debug-network/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__debug-network/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/without_skills/rh-developer__debug-network/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/without_skills/rh-developer__debug-pipeline/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__debug-pod/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/without_skills/rh-developer__debug-pod/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/without_skills/rh-developer__debug-rhel/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__deploy/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__deploy/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/without_skills/rh-developer__deploy/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__deploy/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__deploy/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/without_skills/rh-developer__deploy/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/without_skills/rh-developer__deploy/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__deploy/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/without_skills/rh-developer__deploy/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/without_skills/rh-developer__deploy/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__deploy/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/without_skills/rh-developer__deploy/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__deploy/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__deploy/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/without_skills/rh-developer__deploy/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/without_skills/rh-developer__deploy/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__deploy/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/without_skills/rh-developer__deploy/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__deploy/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__deploy/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/without_skills/rh-developer__deploy/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__deploy/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__deploy/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/without_skills/rh-developer__deploy/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/without_skills/rh-developer__deploy/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__deploy/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/without_skills/rh-developer__deploy/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__detect-project/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/without_skills/rh-developer__detect-project/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__detect-project/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/without_skills/rh-developer__detect-project/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__detect-project/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/without_skills/rh-developer__detect-project/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__detect-project/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/without_skills/rh-developer__detect-project/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__detect-project/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/without_skills/rh-developer__detect-project/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__detect-project/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/without_skills/rh-developer__detect-project/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__detect-project/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/without_skills/rh-developer__detect-project/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__detect-project/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/without_skills/rh-developer__detect-project/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/without_skills/rh-developer__detect-project/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__detect-project/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/without_skills/rh-developer__detect-project/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/without_skills/rh-developer__helm-deploy/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__recommend-image/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/without_skills/rh-developer__recommend-image/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/without_skills/rh-developer__rhel-deploy/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__s2i-build/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/without_skills/rh-developer__s2i-build/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/builder-images.md b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/builder-images.md deleted file mode 100644 index 6561c5ca..00000000 --- a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/builder-images.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: S2I Builder Image Reference -category: containers -sources: - - title: Red Hat Container Catalog - url: https://catalog.redhat.com/software/containers/search - sections: UBI images, S2I builders - date_accessed: 2026-02-08 - - title: OpenShift Source-to-Image (S2I) - url: https://docs.openshift.com/container-platform/latest/openshift_images/using_images/using-s21-images.html - sections: S2I builder images, Language detection - date_accessed: 2026-02-08 - - title: Red Hat Universal Base Images - url: https://developers.redhat.com/products/rhel/ubi - sections: UBI9 images, Language runtimes - date_accessed: 2026-02-08 ---- - -# S2I Builder Image Reference - -Use this reference when recommending S2I builder images to users. - -> **Note:** Versions marked "Recommended" may change. Always verify with `skopeo inspect` before use. Prefer matching the project's version requirements over these defaults. - -For use-case-aware image selection, use the `/recommend-image` skill. - ---- - -## Dynamic Lookup and Verification - -**This reference may be outdated.** Always verify image availability before recommending. - -### Verify with Skopeo (Recommended) - -```bash -# Check if an image exists and get metadata -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Get specific fields -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# List all available tags -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 -``` - -**If skopeo is not installed**, prompt the user: -``` -Install with: sudo dnf install skopeo (Fedora/RHEL) - sudo apt install skopeo (Ubuntu/Debian) - brew install skopeo (macOS) -``` - -### Check Security Status (Red Hat Security Data API) - -Query CVE information (no authentication required): - -```bash -# Check for critical CVEs affecting UBI9 -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' - -# Get CVE details -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' -``` - -### Verify with Red Hat Catalog API (Alternative) - -```bash -# Search for available Node.js images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/nodejs" | jq '.data[].repository' - -# Search for available Python images -curl -s "https://catalog.redhat.com/api/containers/v1/repositories?filter=repository=like=ubi9/python" | jq '.data[].repository' -``` - ---- - -## Project Detection and Version Mapping - -### Extract Version from Project Files - -Before recommending an image, check the project's version requirements: - -| Project File | How to Extract Version | -|--------------|------------------------| -| `package.json` | `.engines.node` field | -| `requirements.txt` | `python_requires` or comments | -| `pyproject.toml` | `[project].requires-python` | -| `pom.xml` | `` or `` | -| `go.mod` | `go` directive (e.g., `go 1.21`) | -| `*.csproj` | `` (e.g., `net8.0`) | - -### Detect Language from Files - -| Indicator File(s) | Language | Framework | Version Source | -|-------------------|----------|-----------|----------------| -| `package.json` | Node.js | - | `.engines.node` | -| `package.json` + `next.config.js` | Node.js | Next.js | `.engines.node` | -| `package.json` + `angular.json` | Node.js | Angular | `.engines.node` | -| `pom.xml` | Java | Maven | `` or `` | -| `pom.xml` + quarkus dep | Java | Quarkus | `` (prefer 21+) | -| `pom.xml` + spring-boot dep | Java | Spring Boot | `` | -| `build.gradle` / `build.gradle.kts` | Java | Gradle | `sourceCompatibility` or `java.toolchain` | -| `requirements.txt` | Python | - | `python_requires` or shebang | -| `Pipfile` | Python | Pipenv | `[requires].python_version` | -| `pyproject.toml` | Python | Poetry/Modern | `[project].requires-python` | -| `go.mod` | Go | - | `go` directive line | -| `Gemfile` | Ruby | - | `ruby` directive or `.ruby-version` | -| `*.csproj` / `*.sln` | .NET | - | `` (e.g., net8.0 → 80) | -| `composer.json` | PHP | - | `require.php` field | -| `Cargo.toml` | Rust | - | Custom (no official S2I) | - -### Map Version to Image - -**Quick lookup pattern:** `ubi9/{language}-{version}` (e.g., `ubi9/nodejs-20`, `ubi9/python-311`) - -| Language | Version Mapping | Image Pattern | -|----------|-----------------|---------------| -| Node.js | 18.x → 18, 20.x → 20, 22.x → 22 | `ubi9/nodejs-{major}` | -| Python | 3.9 → 39, 3.11 → 311, 3.12 → 312 | `ubi9/python-{majmin}` | -| Java | 11, 17, 21 (use nearest LTS) | `ubi9/openjdk-{version}` | -| Go | 1.21 → 1.21, 1.22 → 1.22 | `ubi9/go-toolset:{version}` | -| Ruby | 3.1 → 31, 3.3 → 33 | `ubi9/ruby-{majmin}` | -| .NET | net6.0 → 60, net8.0 → 80 | `ubi9/dotnet-{version}` | -| PHP | 8.0 → 80, 8.1 → 81 | `ubi9/php-{majmin}` | - -### Verify and Fallback - -1. **Verify image exists**: `skopeo inspect docker://registry.access.redhat.com/ubi9/{image}` -2. **If version not found**: Use nearest available LTS version -3. **If no version in project**: Use current LTS (check catalog API) - ---- - -## Red Hat UBI-based Images - -### Node.js - -| Version | Full Image | Minimal Image | Use Case | -|---------|------------|---------------|----------| -| 18 LTS | `registry.access.redhat.com/ubi9/nodejs-18` | `registry.access.redhat.com/ubi9/nodejs-18-minimal` | Long-term support | -| 20 LTS | `registry.access.redhat.com/ubi9/nodejs-20` | `registry.access.redhat.com/ubi9/nodejs-20-minimal` | **Recommended** | -| 22 | `registry.access.redhat.com/ubi9/nodejs-22` | `registry.access.redhat.com/ubi9/nodejs-22-minimal` | Current | - -**Choose minimal for:** Production, security-focused, smaller image size -**Choose full for:** Development, native module compilation - -### Python - -| Version | Image | Notes | -|---------|-------|-------| -| 3.9 | `registry.access.redhat.com/ubi9/python-39` | | -| 3.11 | `registry.access.redhat.com/ubi9/python-311` | **Recommended** | -| 3.12 | `registry.access.redhat.com/ubi9/python-312` | Latest | - -### Java / OpenJDK - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 11 LTS | `registry.access.redhat.com/ubi8/openjdk-11` | `registry.access.redhat.com/ubi8/openjdk-11-runtime` | LTS | -| 17 LTS | `registry.access.redhat.com/ubi9/openjdk-17` | `registry.access.redhat.com/ubi9/openjdk-17-runtime` | **Recommended** | -| 21 LTS | `registry.access.redhat.com/ubi9/openjdk-21` | `registry.access.redhat.com/ubi9/openjdk-21-runtime` | Latest LTS | - -**Choose runtime for:** Production with pre-built JARs, smallest footprint -**Choose build for:** S2I builds, Maven/Gradle compilation needed - -### Go - -| Version | Image | Notes | -|---------|-------|-------| -| 1.20 | `registry.access.redhat.com/ubi9/go-toolset:1.20` | | -| 1.21 | `registry.access.redhat.com/ubi9/go-toolset:1.21` | **Recommended** | - -### Ruby - -| Version | Image | Notes | -|---------|-------|-------| -| 3.1 | `registry.access.redhat.com/ubi9/ruby-31` | | -| 3.3 | `registry.access.redhat.com/ubi9/ruby-33` | **Recommended** | - -### .NET - -| Version | Build Image | Runtime Image | Notes | -|---------|-------------|---------------|-------| -| 6.0 LTS | `registry.access.redhat.com/ubi8/dotnet-60` | `registry.access.redhat.com/ubi8/dotnet-60-runtime` | LTS | -| 7.0 | `registry.access.redhat.com/ubi8/dotnet-70` | `registry.access.redhat.com/ubi8/dotnet-70-runtime` | | -| 8.0 LTS | `registry.access.redhat.com/ubi9/dotnet-80` | `registry.access.redhat.com/ubi9/dotnet-80-runtime` | **Recommended** | - -**Choose runtime for:** Production with pre-built assemblies -**Choose build for:** S2I builds, dotnet build/publish needed - -### PHP - -| Version | Image | Notes | -|---------|-------|-------| -| 8.0 | `registry.access.redhat.com/ubi9/php-80` | | -| 8.1 | `registry.access.redhat.com/ubi9/php-81` | **Recommended** | - -### Perl - -| Version | Image | Notes | -|---------|-------|-------| -| 5.32 | `registry.access.redhat.com/ubi9/perl-532` | | - ---- - -## Image Variants and Use-Case Selection - -### Quick Use-Case Matrix - -| Use Case | Variant | Priority | Example | -|----------|---------|----------|---------| -| Production | Minimal/Runtime | Security, Size | `nodejs-20-minimal` | -| Development | Full | Tools, Debug | `nodejs-20` | -| Serverless | Minimal | Startup Time | `openjdk-21-runtime` | -| Edge/IoT | Minimal | Size | `nodejs-20-minimal` | - -### Image Variants - -| Variant | Description | Has Build Tools | Size | -|---------|-------------|-----------------|------| -| Full | Complete development environment | Yes | Largest | -| Minimal | Essential packages only | Limited | Medium | -| Runtime | Runtime only, no build tools | No | Smallest | - -**Availability by language:** - -| Language | Full | Minimal | Runtime | -|----------|------|---------|---------| -| Node.js | `nodejs-{ver}` | `nodejs-{ver}-minimal` | - | -| Python | `python-{ver}` | - | - | -| Java | `openjdk-{ver}` | - | `openjdk-{ver}-runtime` | -| Go | `go-toolset:{ver}` | - | (produces static binary) | -| .NET | `dotnet-{ver}` | - | `dotnet-{ver}-runtime` | -| Ruby | `ruby-{ver}` | - | - | -| PHP | `php-{ver}` | - | - | - -### When to Recommend Each Variant - -**Full variant:** -- User needs to compile native extensions -- Development/debugging environment -- CI/CD build stages - -**Minimal variant:** -- Production deployments -- Security-focused environments -- When size matters but some tools needed - -**Runtime variant:** -- Pre-compiled applications (JARs, .NET assemblies) -- Maximum security posture -- Smallest possible footprint - ---- - -## OpenShift Built-in ImageStreams - -These are often pre-configured in OpenShift clusters under the `openshift` namespace: - -| ImageStream | Usage | -|-------------|-------| -| `nodejs:20-ubi9` | Node.js 20 on UBI 9 | -| `python:3.11-ubi9` | Python 3.11 on UBI 9 | -| `openjdk-17-ubi8` | Java 17 on UBI 8 | -| `ruby:3.1-ubi9` | Ruby 3.1 on UBI 9 | -| `php:8.0-ubi9` | PHP 8.0 on UBI 9 | - -When using OpenShift ImageStreams, reference them as: -```yaml -from: - kind: ImageStreamTag - namespace: openshift - name: nodejs:20-ubi9 -``` - ---- - -## Framework-Specific Recommendations - -### Quarkus (Java) -- **Native build**: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- **JVM build**: `registry.access.redhat.com/ubi9/openjdk-21` - -### Spring Boot (Java) -- Use: `registry.access.redhat.com/ubi9/openjdk-17` or `openjdk-21` -- Ensure `spring-boot-maven-plugin` is configured for packaging - -### Next.js / React (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-20` -- Ensure build outputs to `build/` or `.next/` - -### Django / Flask (Python) -- Use: `registry.access.redhat.com/ubi9/python-311` -- Ensure `requirements.txt` or `Pipfile` exists at root - -### Express.js (Node.js) -- Use: `registry.access.redhat.com/ubi9/nodejs-18` or higher -- Ensure `npm start` script is defined in `package.json` - ---- - -## Python S2I Entry Point Requirements - -**Quick reference:** -- Default entry point: `app.py` (works without configuration) -- Custom entry points require: `gunicorn` + `APP_MODULE` environment variable -- Format: `APP_MODULE=module:variable` (e.g., `main:app`) diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/debugging-patterns.md b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/debugging-patterns.md deleted file mode 100644 index 2863d559..00000000 --- a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/debugging-patterns.md +++ /dev/null @@ -1,478 +0,0 @@ ---- -title: Debugging Patterns -category: references -sources: - - title: Kubernetes Debugging Pods - url: https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/ - sections: Debugging Pods, Common Errors - date_accessed: 2026-02-16 - - title: OpenShift Troubleshooting Guide - url: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html - sections: Pod issues, Build issues - date_accessed: 2026-02-16 - - title: OpenShift Pipelines Troubleshooting - url: https://docs.openshift.com/pipelines/latest/about/about-openshift-pipelines.html - sections: Troubleshooting, PipelineRun status, TaskRun status - date_accessed: 2026-02-25 - - title: Podman Troubleshooting - url: https://github.com/containers/podman/blob/main/troubleshooting.md - sections: Common Issues - date_accessed: 2026-02-16 ---- - -# Debugging Patterns - -This document provides common error patterns, exit codes, and troubleshooting decision trees for the debugging skills. - -## Exit Code Reference - -### Container/Process Exit Codes - -| Exit Code | Signal | Meaning | Common Cause | -|-----------|--------|---------|--------------| -| 0 | - | Success | Normal termination | -| 1 | - | General error | Application error, unhandled exception | -| 2 | - | Misuse of shell | Invalid arguments, syntax error | -| 126 | - | Permission denied | Cannot execute command | -| 127 | - | Command not found | Binary/script missing in PATH | -| 128 | - | Invalid exit argument | Exit called with non-integer | -| 128+N | Signal N | Killed by signal | See signal table below | -| 137 | SIGKILL (9) | Force killed | OOM kill, manual kill, timeout | -| 139 | SIGSEGV (11) | Segmentation fault | Memory corruption, null pointer | -| 143 | SIGTERM (15) | Terminated | Graceful shutdown request | - -### Signal Reference (128+N) - -| Signal | Number | Exit Code | Typical Cause | -|--------|--------|-----------|---------------| -| SIGHUP | 1 | 129 | Terminal closed | -| SIGINT | 2 | 130 | Ctrl+C | -| SIGQUIT | 3 | 131 | Ctrl+\ | -| SIGKILL | 9 | 137 | OOM, forced termination | -| SIGSEGV | 11 | 139 | Segmentation fault | -| SIGTERM | 15 | 143 | Graceful stop request | - -## Pod Failure Patterns - -### CrashLoopBackOff - -**Symptom:** Pod repeatedly crashes and restarts - -**Diagnosis Flow:** -``` -CrashLoopBackOff -├─ Check exit code -│ ├─ 0 → Application exits normally (missing loop/server?) -│ ├─ 1 → Application error (check logs) -│ ├─ 127 → Command not found (check entrypoint) -│ └─ 137 → OOM killed (check memory limits) -├─ Check logs (current + previous) -│ ├─ Import errors → Missing dependencies -│ ├─ Connection errors → External service down -│ └─ Config errors → Missing env vars/secrets -└─ Check events - └─ FailedMount → Missing secrets/configmaps -``` - -**Common Causes:** -1. Application crashes on startup (dependency errors) -2. Memory limit too low (OOMKilled) -3. Missing environment variables or secrets -4. Database/service connection failures -5. Health probe failing immediately - -### ImagePullBackOff - -**Symptom:** Cannot pull container image - -**Diagnosis Flow:** -``` -ImagePullBackOff -├─ Check event message -│ ├─ "unauthorized" → Registry authentication -│ │ └─ Check imagePullSecrets -│ ├─ "not found" → Wrong image name/tag -│ │ └─ Verify image exists in registry -│ ├─ "timeout" → Network/registry issue -│ │ └─ Check cluster network egress -│ └─ "manifest unknown" → Tag doesn't exist -│ └─ Verify tag in registry -└─ Check image reference - ├─ Missing registry prefix? - ├─ Typo in image name? - └─ Tag exists? -``` - -**Common Causes:** -1. Private registry without imagePullSecret -2. Image tag doesn't exist -3. Registry URL typo -4. Network policy blocking egress -5. Registry rate limiting - -### Pending Pod - -**Symptom:** Pod stuck in Pending state - -**Diagnosis Flow:** -``` -Pending -├─ Check events -│ ├─ "FailedScheduling" -│ │ ├─ "Insufficient cpu/memory" → Scale cluster or reduce requests -│ │ ├─ "node selector" → No matching nodes -│ │ ├─ "taints" → Need tolerations -│ │ └─ "PVC not bound" → Storage issue -│ └─ No events → Check resourceQuota -└─ Check node status - └─ All nodes NotReady? → Node issue -``` - -**Common Causes:** -1. Insufficient cluster resources -2. Node selector doesn't match any nodes -3. PersistentVolumeClaim not bound -4. Resource quota exceeded -5. Affinity/anti-affinity rules too strict - -### OOMKilled - -**Symptom:** Container terminated with exit code 137 - -**Diagnosis Flow:** -``` -OOMKilled (exit 137) -├─ Check container state -│ └─ OOMKilled: true → Memory exhaustion confirmed -├─ Compare memory usage vs limit -│ ├─ Limit too low → Increase memory limit -│ └─ Memory leak → Profile application -└─ Check for: - ├─ Java → Heap size (-Xmx) exceeds limit - ├─ Node.js → --max-old-space-size too high - └─ Python → Large data structures in memory -``` - -**Common Causes:** -1. Memory limit set too low for application -2. Memory leak in application -3. Java heap size exceeds container limit -4. Processing large files/datasets in memory - -## Build Failure Patterns - -### S2I Build Phases - -| Phase | What Happens | Common Failures | -|-------|--------------|-----------------| -| **fetch-source** | Clone git repository | Auth failure, repo not found | -| **pull-builder** | Pull S2I builder image | Image not found, auth | -| **assemble** | Run S2I assemble script | Dependency install, build errors | -| **commit** | Create image layer | Disk space | -| **push** | Push to internal registry | Auth, quota | - -### Assemble Phase Failures - -**Node.js:** -``` -npm ERR! 404 Not Found -└─ Package doesn't exist in registry - → Check package.json for typos - -npm ERR! code ERESOLVE -└─ Dependency conflict - → Run npm install --legacy-peer-deps - -npm ERR! code ENOENT -└─ File not found - → Check paths in package.json -``` - -**Python:** -``` -ERROR: Could not find a version that satisfies the requirement -└─ Package not found - → Check requirements.txt spelling - -ModuleNotFoundError: No module named 'X' -└─ APP_MODULE misconfigured - → See docs/python-s2i-entrypoints.md - -gunicorn: command not found -└─ gunicorn not in requirements - → Add gunicorn to requirements.txt -``` - -**Java:** -``` -[ERROR] Failed to execute goal -└─ Maven/Gradle build failure - → Check pom.xml or build.gradle - -java.lang.OutOfMemoryError: Java heap space -└─ Build needs more memory - → Add MAVEN_OPTS=-Xmx512m -``` - -## Pipeline/Tekton Failure Patterns - -### PipelineRun Failure Decision Tree - -``` -PipelineRun Failed -├─ Check PipelineRun status conditions -│ ├─ "PipelineRunTimeout" → Increase spec.timeouts.pipeline -│ ├─ "CouldntGetPipeline" → Pipeline reference invalid, check name/namespace -│ ├─ "PipelineRunCancelled" → Check if timeout or manual cancellation -│ └─ "Failed" → Check which TaskRun failed (see below) -├─ Check failed TaskRun -│ ├─ Step failure (non-zero exit) -│ │ ├─ git-clone step → Auth/URL issue (check SA secrets) -│ │ ├─ build step → Compilation/dependency error -│ │ ├─ push step → Registry auth (check SA dockerconfigjson secret) -│ │ └─ test step → Test failures -│ ├─ Pod scheduling failure → Resource constraints (FailedScheduling event) -│ ├─ Workspace issue → PVC not bound or permission denied -│ └─ Step image pull failure → ImagePullBackOff on step container -└─ Pipeline stuck (Running too long) - ├─ TaskRun pending → Pod can't be scheduled - ├─ Step running indefinitely → Check logs for hang/deadlock - └─ Custom task waiting → Check custom task controller -``` - -### TaskRun Failure Analysis - -``` -TaskRun Failed -├─ Pod not created → Check ServiceAccount exists, resource quotas -├─ Pod pending → Scheduling issue (see Pod Failure Patterns) -├─ Pod terminated → Check step statuses -│ ├─ Exit 1 → Script/application error (check step logs) -│ ├─ Exit 125-127 → Entrypoint/command issue in step image -│ └─ Exit 137 → OOM killed (increase step resources) -└─ Workspace binding failure - ├─ PVC not found → Create PVC or fix workspace binding - ├─ RWO blocks parallel tasks → Use RWX or separate workspaces - └─ Permission denied → Check fsGroup, runAsUser in pod security context -``` - -### Common Tekton Error Messages - -| Error Message | Fix | -|--------------|-----| -| `task "X" not found` | Verify Task name, kind (Task vs ClusterTask), namespace | -| `could not read Username for...` | Add git-credentials secret (annotated with `tekton.dev/git-0`) to ServiceAccount | -| `unauthorized: access denied` (push) | Add dockerconfigjson secret (annotated with `tekton.dev/docker-0`) to ServiceAccount | -| `persistentvolumeclaim "X" not found` | Create PVC or change workspace binding to emptyDir | -| `exceeded timeout` | Increase timeouts in PipelineRun spec (`spec.timeouts.pipeline` / `spec.timeouts.tasks`) | -| `missing required parameter "X"` | Add parameter value to PipelineRun spec | -| `couldn't find remote ref` | Fix git `revision` parameter (branch/tag name) | -| `unable to open Containerfile/Dockerfile` | Fix `DOCKERFILE` param path relative to workspace root | - -## Network Troubleshooting - -### Service Has No Endpoints - -**Diagnosis Flow:** -``` -No endpoints -├─ Check service selector -│ └─ Compare with pod labels -│ ├─ Labels don't match → Fix selector or pod labels -│ └─ Labels match → Check pod readiness -├─ Check pod status -│ ├─ Pods not running → Debug pods first -│ └─ Pods running but not ready → Check readiness probe -└─ Check readiness probe - ├─ HTTP probe failing → Application not listening - └─ TCP probe failing → Wrong port -``` - -### Route Returning 503 - -**Diagnosis Flow:** -``` -503 Service Unavailable -├─ Check endpoints -│ └─ No endpoints → Pods not ready -├─ Check backend pods -│ ├─ All pods failing readiness → Application issue -│ └─ Some pods ready → Load balancer issue -└─ Check route configuration - └─ Wrong service or port → Fix route spec -``` - -### Connection Refused - -**Diagnosis Flow:** -``` -Connection refused -├─ Is service created? → oc get svc -├─ Does service have endpoints? → oc get endpoints -├─ Is pod running? → oc get pods -├─ Is application listening? → Check container port -└─ Is port correct? → Compare service port vs container port -``` - -## RHEL System Patterns - -### systemd Service Failures - -| Exit Code | Meaning | Common Fix | -|-----------|---------|------------| -| 1 | General error | Check application logs | -| 126 | Permission | Check ExecStart permissions | -| 127 | Not found | Check binary path in ExecStart | -| 203 | EXEC | Wrong architecture or format | -| 217 | USER | Service user doesn't exist | - -### SELinux Denial Patterns - -| Denial Type | Example | Typical Fix | -|-------------|---------|-------------| -| Port binding | `httpd_t` bind `port_t` | `semanage port -a -t http_port_t -p tcp [port]` | -| File read | `httpd_t` read `user_home_t` | `semanage fcontext` + `restorecon` | -| Network connect | `httpd_t` connect | `setsebool -P httpd_can_network_connect on` | -| Container | `container_t` manage | `setsebool -P container_manage_cgroup on` | - -See [selinux-troubleshooting.md](selinux-troubleshooting.md) for detailed SELinux guidance. - -## Troubleshooting Decision Tree - -### Application Not Accessible - -``` -Cannot access application -├─ Internal (from cluster)? -│ ├─ Yes, works internally → Route/Ingress issue -│ │ ├─ Check route admitted -│ │ ├─ Check route host/path -│ │ └─ Check TLS configuration -│ └─ No, fails internally too → Service/Pod issue -│ ├─ Check service endpoints -│ ├─ Check pod status -│ └─ Check pod readiness -└─ Neither works? - └─ Debug pod first (/debug-pod) -``` - -### Build Keeps Failing - -``` -Build failures -├─ Which phase? -│ ├─ fetch-source → Git access issue -│ │ ├─ Check source secret -│ │ └─ Verify git URL -│ ├─ pull-builder → Builder image issue -│ │ ├─ Check image reference -│ │ └─ Import ImageStream -│ ├─ assemble → Build script issue -│ │ ├─ Check dependencies -│ │ └─ Check language-specific config -│ └─ push → Registry issue -│ └─ Check push secret -└─ Same failure pattern? - └─ Compare with last successful build -``` - -### Pipeline Keeps Failing - -``` -Pipeline failures -├─ Same task always fails? -│ ├─ git-clone → Check ServiceAccount secrets, git URL, revision -│ ├─ build step → Check source code, Containerfile path, build context -│ └─ push step → Check ServiceAccount imagePullSecrets, registry URL -├─ Different tasks fail? -│ ├─ Resource exhaustion → Reduce parallel tasks or increase quotas -│ └─ Workspace contention → Use RWX PVC or separate workspaces -├─ Pipeline hangs? -│ ├─ TaskRun pending → Pod can't be scheduled -│ └─ Step running indefinitely → Check step logs -└─ Pipeline never triggers? - ├─ EventListener pod not running → Check EL deployment/logs - ├─ Webhook misconfigured → Verify webhook URL and secret - └─ TriggerBinding wrong → Check CEL expression param extraction -``` - -## Quick Reference Commands - -### OpenShift Debugging - -```bash -# Pod status and events -oc describe pod [pod-name] - -# Pod logs (current) -oc logs [pod-name] - -# Pod logs (previous container) -oc logs [pod-name] --previous - -# All events in namespace -oc get events --sort-by='.lastTimestamp' - -# Check endpoints -oc get endpoints [service-name] - -# Build logs -oc logs build/[build-name] -``` - -### Pipeline/Tekton Debugging - -```bash -# List PipelineRuns (oldest first) -oc get pipelinerun --sort-by='.metadata.creationTimestamp' - -# Get PipelineRun details -oc get pipelinerun [name] -o yaml - -# List TaskRuns for a PipelineRun -oc get taskrun -l tekton.dev/pipelineRun=[pipelinerun-name] - -# Get TaskRun pod logs for a specific step -oc logs [taskrun-name]-pod -c step-[step-name] - -# Get events for pipeline resources -oc get events --field-selector involvedObject.kind=PipelineRun - -# Describe EventListener -oc get eventlistener [name] -o yaml -``` - -### RHEL Debugging - -```bash -# Service status -systemctl status [service] - -# Journal logs -journalctl -u [service] -n 100 - -# SELinux denials -ausearch -m AVC -ts recent - -# Firewall rules -firewall-cmd --list-all - -# SELinux context -ls -lZ [path] -``` - -### Container Debugging - -```bash -# List all containers -podman ps -a - -# Container inspect -podman inspect [container] - -# Container logs -podman logs [container] - -# Run interactively for debugging -podman run -it --entrypoint /bin/sh [image] -``` diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/dynamic-validation.md b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/dynamic-validation.md deleted file mode 100644 index a027f0ce..00000000 --- a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/dynamic-validation.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -title: Dynamic Image Validation Reference -category: containers -sources: - - title: Skopeo Documentation - url: https://github.com/containers/skopeo - sections: Inspecting images, Copying images - date_accessed: 2026-02-08 - - title: Red Hat Security Data API - url: https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0 - sections: CVE queries, Product filtering - date_accessed: 2026-02-08 ---- - -# Dynamic Image Validation Reference - -This document provides detailed patterns for validating container images using Skopeo and the Red Hat Security Data API. - -## Skopeo Commands - -Skopeo inspects container images without downloading them, providing real-time metadata. - -### Prerequisites - -**Check if skopeo is installed:** -```bash -which skopeo -# or -skopeo --version -``` - -**Installation:** -| OS | Command | -|----|---------| -| Fedora/RHEL/CentOS | `sudo dnf install skopeo` | -| Ubuntu/Debian | `sudo apt install skopeo` | -| macOS (Homebrew) | `brew install skopeo` | - -### Basic Inspection - -```bash -# Inspect an image (full JSON output) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# The docker:// transport is OCI-standard and works with all registries -# (Docker Hub, Red Hat, Quay, Podman registries, etc.) -``` - -### Extracting Specific Fields - -```bash -# Get creation date -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Created}}' - -# Get architecture -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Architecture}}' - -# Get all labels -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{.Labels}}' - -# Get specific label (e.g., version) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{index .Labels "version"}}' - -# Get layer count -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 --format '{{len .Layers}}' -``` - -### Listing Available Tags - -```bash -# List all tags for an image -skopeo list-tags docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Output includes all available versions/tags -``` - -### Image Transport Options - -```bash -# Remote registry (most common) -skopeo inspect docker://registry.access.redhat.com/ubi9/nodejs-20 - -# Local Podman storage -skopeo inspect containers-storage:localhost/myimage:latest - -# OCI layout directory -skopeo inspect oci:/path/to/oci-layout:tag - -# Docker archive -skopeo inspect docker-archive:/path/to/image.tar -``` - -### Useful Metadata Fields - -| Field | Description | Use Case | -|-------|-------------|----------| -| `Created` | Image build timestamp | Freshness indicator | -| `Architecture` | CPU architecture | Verify ARM64/x86_64 support | -| `Os` | Operating system | Should be "linux" for UBI | -| `Labels` | Image labels (version, maintainer, etc.) | Verify language version | -| `Layers` | Layer digests | Calculate approximate size | -| `Digest` | Immutable image hash | Pin exact version | - -### Error Handling - -**Image not found:** -``` -Error: Error reading manifest: ... 404 Not Found -``` -→ Image does not exist at specified tag - -**Authentication required:** -``` -Error: Error reading manifest: unauthorized -``` -→ Private registry, need `skopeo login` first - -**Network error:** -``` -Error: Error initializing source: pinging container registry -``` -→ Network connectivity issue - ---- - -## Red Hat Security Data API - -The Security Data API provides CVE information without authentication. - -### Base Endpoint - -``` -https://access.redhat.com/hydra/rest/securitydata/ -``` - -### Query CVEs - -```bash -# Get all CVEs for UBI 9 (may return many results) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209" - -# Filter by severity (critical, important, moderate, low) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" - -# Filter by date (CVEs after a specific date) -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&after=2025-01-01" - -# Count critical CVEs -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length' -``` - -### Product Names for Queries - -| Image Base | Product Name (URL-encoded) | -|------------|---------------------------| -| UBI 9 | `Red%20Hat%20Universal%20Base%20Image%209` | -| UBI 8 | `Red%20Hat%20Universal%20Base%20Image%208` | -| RHEL 9 | `Red%20Hat%20Enterprise%20Linux%209` | -| RHEL 8 | `Red%20Hat%20Enterprise%20Linux%208` | - -### Response Fields - -Each CVE object contains: - -| Field | Description | -|-------|-------------| -| `CVE` | CVE identifier (e.g., CVE-2024-1234) | -| `severity` | critical, important, moderate, low | -| `public_date` | When CVE was disclosed | -| `advisories` | Related Red Hat advisories | -| `bugzilla` | Bugzilla tracking URL | -| `affected_packages` | Packages affected by CVE | - -### Parsing Examples - -```bash -# Get CVE IDs and severities -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[] | {cve: .CVE, severity: .severity}' - -# Get most recent CVE date -curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq '.[0].public_date' - -# Check if any critical CVEs exist -CRITICAL_COUNT=$(curl -s "https://access.redhat.com/hydra/rest/securitydata/cve.json?product=Red%20Hat%20Universal%20Base%20Image%209&severity=critical" | jq 'length') -if [ "$CRITICAL_COUNT" -gt 0 ]; then - echo "Warning: $CRITICAL_COUNT critical CVEs found" -fi -``` - ---- - -## Validation Workflow - -### Complete Validation Sequence - -``` -1. Check if skopeo is installed - ├── Yes → Continue to step 2 - └── No → Prompt user to install, offer to continue with static data - -2. For each candidate image: - a. Run: skopeo inspect docker://registry.access.redhat.com/ubi9/[image] - b. If fails → Remove from candidates, try next - c. If succeeds → Extract: Created, Architecture, Labels - -3. Query Security Data API for UBI base: - a. Run: curl CVE query for critical severity - b. Parse count of critical CVEs - c. If count > 0 → Add warning to recommendation - -4. Compile results: - - Image metadata (from skopeo) - - Security status (from API) - - Static scoring data (from reference tables) - -5. Present recommendation with sources indicated -``` - -### Fallback Behavior - -| Scenario | Action | -|----------|--------| -| Skopeo not installed | Prompt installation, offer static-only mode | -| Skopeo command fails | Note "unable to verify", use static data | -| Security API unavailable | Note "security not verified", proceed | -| Image not found | Remove from candidates, suggest alternatives | -| Network offline | Use static data only, note limitations | - ---- - -## Integration with Recommendation Output - -### When Dynamic Data Available - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | 147 MB | Skopeo | -| Built | 2026-01-28 | Skopeo | -| Architecture | amd64, arm64 | Skopeo | - -**Security Status:** No critical CVEs -- Last checked: 2026-02-03 -- Source: Red Hat Security Data API -``` - -### When Dynamic Data Unavailable - -```markdown -| Property | Value | Source | -|----------|-------|--------| -| Size | ~150 MB (estimate) | Static | -| Built | Unknown | - | -| Architecture | Assumed amd64 | Static | - -**Security Status:** Not verified (warning) -- Skopeo not installed - install for accurate metadata -- Run: `sudo dnf install skopeo` -``` diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/human-in-the-loop.md b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/human-in-the-loop.md deleted file mode 100644 index 696fccf1..00000000 --- a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/human-in-the-loop.md +++ /dev/null @@ -1,98 +0,0 @@ -# Human-in-the-Loop Requirements - -This document defines mandatory checkpoint behavior for all rh-developer skills. - -## Critical Requirements - -**IMPORTANT:** All skills require explicit user confirmation at each step. You MUST: - -1. **Wait for user confirmation** before executing any actions -2. **Do NOT proceed** to the next step until the user explicitly approves -3. **Present options clearly** (yes/no/modify) and wait for response -4. **Never auto-execute** resource creation, builds, or deployments -5. **Never skip configuration questions** even if user seems to know what they want - -If the user says "no" or wants modifications, address their concerns before proceeding. - -## Anti-Patterns to Avoid - -**CRITICAL - DO NOT DO THIS:** - -| Anti-Pattern | Why It's Wrong | -|--------------|----------------| -| User says "yes do X to namespace Y" → Skip config questions | Strategy ≠ Configuration. User chose WHAT, not HOW | -| User seems experienced → Assume they've considered all options | Even experts benefit from checklists | -| User provides multiple answers at once → Skip individual confirmations | Each checkpoint exists for a reason | -| User is in a hurry → Rush through phases | Speed causes mistakes in production | - -## When User Provides Multiple Answers - -If user says: "yes do helm deployment to test-app namespace" - -**DO NOT** skip phases. Instead: - -1. Acknowledge: "Great, you've chosen Helm strategy and test-app namespace." -2. Continue: "Let me confirm the configuration details..." -3. Still ask: Environment type, config approach, resources, etc. -4. Get explicit confirmation for each phase - -**The user specifying WHAT to deploy does not mean they've decided HOW to configure it.** - -## Standard Checkpoint Language - -Use this exact pattern after EVERY step/phase: - -```markdown -**WAIT for user confirmation before proceeding.** Do NOT continue to the next phase until user explicitly confirms. - -- If user says "yes" → Proceed to next phase -- If user says "no" → Ask what they would like to change -- If user says "modify" → Update configuration and show again for confirmation -- If user gives multiple answers at once → Still confirm each remaining checkpoint individually -``` - -## Mandatory Configuration Questions - -Before ANY resource creation, these questions should be asked: - -| Question | Why It Matters | -|----------|----------------| -| Environment type (dev/staging/prod) | Affects image tags, resources, replicas | -| Runtime vs build-time config | Affects flexibility and rebuild frequency | -| Resource limits | Prevents OOM, ensures fair scheduling | -| Replicas | Affects availability and cost | - -## Include in Your Skill - -Add this section after Prerequisites in your SKILL.md: - -```markdown -## Critical: Human-in-the-Loop Requirements - -See [Human-in-the-Loop Requirements](../docs/human-in-the-loop.md) for mandatory checkpoint behavior. - -**Key Rules:** -1. WAIT for explicit user confirmation at each phase -2. Never skip configuration questions, even if user specifies strategy upfront -3. Strategy choice ≠ Configuration approval -``` - -## Phase Execution Rules - -**MANDATORY:** Execute phases in order. Each phase MUST: - -1. Display the phase information to the user -2. Ask the specific question for that phase -3. Wait for user response -4. Only then proceed to next phase - -**Even if user provides information for multiple phases at once:** -- Acknowledge what they said -- But still display each phase's confirmation prompt -- Get explicit "yes" for each phase before executing - -Example: -- User: "yes do helm to test-app namespace" -- AI: "Great, you've chosen Helm strategy and test-app namespace. Let me confirm the configuration details..." -- [Still show Configuration Review phase] -- [Still ask environment type, config approach, etc.] diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/image-selection-criteria.md b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/image-selection-criteria.md deleted file mode 100644 index 184b7f5e..00000000 --- a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/image-selection-criteria.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -title: Image Selection Criteria Reference -category: containers -sources: - - title: Red Hat Container Best Practices - url: https://developers.redhat.com/articles/2023/02/14/best-practices-building-images-pass-red-hat-container-certification - sections: Image sizing, Security considerations - date_accessed: 2026-02-08 - - title: OpenShift Image Guidelines - url: https://docs.openshift.com/container-platform/latest/openshift_images/create-images.html - sections: Image creation, Optimization - date_accessed: 2026-02-08 ---- - -# Image Selection Criteria Reference - -This document provides detailed criteria for selecting the optimal container image based on use case requirements. - -## Scoring Matrix - -Use this matrix to score image options based on user requirements. - -### Criteria Weights by Environment - -| Criteria | Production | Development | Edge/IoT | Serverless | -|----------|------------|-------------|----------|------------| -| Image Size | 3 | 1 | 5 | 4 | -| Security Posture | 5 | 2 | 4 | 3 | -| Build Tools | 1 | 5 | 1 | 1 | -| Startup Time | 3 | 1 | 3 | 5 | -| LTS Status | 5 | 2 | 4 | 3 | -| Debug Tools | 1 | 5 | 1 | 1 | - -**Scale:** 1 (low importance) to 5 (high importance) - -### Image Variant Scores - -| Variant | Size | Security | Build Tools | Startup | Debug | -|---------|------|----------|-------------|---------|-------| -| Full | 2 | 2 | 5 | 2 | 5 | -| Minimal | 4 | 4 | 2 | 4 | 2 | -| Runtime | 5 | 5 | 1 | 5 | 1 | - -**Scale:** 1 (poor) to 5 (excellent) - -## Image Size Reference - -Approximate compressed image sizes: - -### Node.js -| Image | Size | -|-------|------| -| `ubi9/nodejs-20` | ~250MB | -| `ubi9/nodejs-20-minimal` | ~150MB | - -### Python -| Image | Size | -|-------|------| -| `ubi9/python-311` | ~280MB | - -### Java -| Image | Size | -|-------|------| -| `ubi9/openjdk-17` | ~400MB | -| `ubi9/openjdk-17-runtime` | ~200MB | - -### Go -| Image | Size | -|-------|------| -| `ubi9/go-toolset:1.21` | ~500MB | -| Final binary | ~10-50MB | - -### .NET -| Image | Size | -|-------|------| -| `ubi9/dotnet-80` | ~350MB | -| `ubi9/dotnet-80-runtime` | ~150MB | - -## LTS Support Timeline - -### Node.js -| Version | Status | End of Life | -|---------|--------|-------------| -| 18 LTS | Active | April 2025 | -| 20 LTS | Active | April 2026 | -| 22 LTS | Active | April 2027 | - -### Python -| Version | Status | End of Life | -|---------|--------|-------------| -| 3.9 | Security | October 2025 | -| 3.11 | Active | October 2027 | -| 3.12 | Active | October 2028 | - -### Java (OpenJDK) -| Version | Status | Extended Support | -|---------|--------|------------------| -| 11 LTS | Active | Red Hat until 2027 | -| 17 LTS | Active | Red Hat until 2029 | -| 21 LTS | Active | Red Hat until 2031 | - -### .NET -| Version | Status | End of Life | -|---------|--------|-------------| -| 6.0 LTS | Active | November 2024 | -| 8.0 LTS | Active | November 2026 | - -## Security Considerations - -### Minimal Images - When to Use -- Fewer installed packages = smaller attack surface -- Recommended for production workloads -- May lack debugging tools when issues occur - -### Full Images - When to Use -- Include development tools (gcc, make, etc.) -- Needed for native extensions (Python C extensions, Node native modules) -- Better for development and debugging - -### Runtime Images - When to Use -- No build tools at all -- Smallest possible footprint -- Requires pre-compiled application (JAR, static binary) - -## Framework-Specific Considerations - -### Quarkus (Java) -**For JVM mode:** -- Use `ubi9/openjdk-21` for build -- Use `ubi9/openjdk-21-runtime` for production - -**For Native mode:** -- Build: `quay.io/quarkus/ubi-quarkus-mandrel-builder-image:jdk-21` -- Run: `quay.io/quarkus/quarkus-micro-image:2.0` -- Dramatically faster startup (~50ms vs ~2s) - -### Spring Boot (Java) -**Standard:** -- Build and run: `ubi9/openjdk-17` - -**Optimized production:** -- Build with layered JAR: `spring-boot-maven-plugin` with layers -- Run on: `ubi9/openjdk-17-runtime` - -### Next.js (Node.js) -**Development:** -- Use `ubi9/nodejs-20` - -**Production (multi-stage recommended):** -1. Build stage: `ubi9/nodejs-20` -2. Run stage: `ubi9/nodejs-20-minimal` with `.next` output - -### Django/Flask (Python) -- Always use full image (may need compilation for dependencies) -- `ubi9/python-311` recommended -- Consider `gunicorn` for production - -## Decision Tree - -``` -START - | - v -Is this production? - | - +-- YES --> Need native compilation? - | | - | +-- YES --> Use FULL variant - | | - | +-- NO --> Is app pre-compiled? - | | - | +-- YES --> Use RUNTIME variant - | | - | +-- NO --> Use MINIMAL variant - | - +-- NO (Development) --> Use FULL variant -``` - -## Multi-Stage Build Recommendations - -For optimal production images, consider multi-stage builds: - -### Node.js Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/nodejs-20 AS builder -COPY . . -RUN npm ci && npm run build - -# Production stage -FROM registry.access.redhat.com/ubi9/nodejs-20-minimal -COPY --from=builder /app/dist /app -CMD ["node", "/app/index.js"] -``` - -### Java Example -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder -COPY . . -RUN mvn package -DskipTests - -# Production stage -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime -COPY --from=builder /app/target/*.jar /app/app.jar -CMD ["java", "-jar", "/app/app.jar"] -``` - -### Go Example -Go produces static binaries, so minimal base is ideal: -```dockerfile -# Build stage -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 AS builder -COPY . . -RUN go build -o /app/server - -# Production stage -FROM registry.access.redhat.com/ubi9/ubi-micro -COPY --from=builder /app/server /server -CMD ["/server"] -``` diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/prerequisites.md b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/prerequisites.md deleted file mode 100644 index d81a9b5c..00000000 --- a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/prerequisites.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Prerequisites -category: setup -sources: - - title: OpenShift CLI (oc) Installation - url: https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html - sections: Installing the CLI, Logging in - date_accessed: 2026-02-08 - - title: Helm Installation Guide - url: https://helm.sh/docs/intro/install/ - sections: From script, From package managers - date_accessed: 2026-02-08 - - title: Podman Installation - url: https://podman.io/docs/installation - sections: Linux, macOS, Windows - date_accessed: 2026-02-08 - - title: Skopeo Installation - url: https://github.com/containers/skopeo/blob/main/install.md - sections: Distribution packages, Building from source - date_accessed: 2026-02-08 ---- - -# Prerequisites - -This document lists all tools required by the rh-developer agentic collection. - -## Required Tools by Skill - -| Skill | Required Tools | Optional Tools | -|-------|----------------|----------------| -| `/detect-project` | `git` | - | -| `/s2i-build` | `oc` | `git` | -| `/deploy` | `oc` | - | -| `/helm-deploy` | `oc`, `helm` | - | -| `/containerize-deploy` | `oc` | `git`, `helm` | -| `/rhel-deploy` | `ssh`, `podman` or `docker` | `git`, `dnf` | -| `/recommend-image` | - | `skopeo`, `curl`, `jq` | -| `/debug-pod` | `oc` | - | -| `/debug-build` | `oc` | - | -| `/debug-network` | `oc` | - | -| `/debug-rhel` | `ssh` | `ausearch`, `journalctl` | -| `/debug-container` | `podman` or `docker` | - | - -## Tool Reference - -### OpenShift CLI (oc) - -**Required for:** Cluster operations, S2I builds, deployments - -```bash -# Check installation -oc version - -# Installation -# Download from: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ -# Or via package manager: -sudo dnf install openshift-clients # Fedora/RHEL -brew install openshift-cli # macOS -``` - -### Helm - -**Required for:** Helm chart deployments - -```bash -# Check installation -helm version - -# Installation -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -# Or via package manager: -sudo dnf install helm # Fedora/RHEL -brew install helm # macOS -``` - -### Podman - -**Required for:** Container builds, RHEL container deployments - -```bash -# Check installation -podman --version - -# Installation -sudo dnf install podman # Fedora/RHEL/CentOS -sudo apt install podman # Ubuntu/Debian -brew install podman # macOS -``` - -### Docker (alternative to Podman) - -**Required for:** Container builds (if Podman not available) - -```bash -# Check installation -docker --version - -# Installation -# See: https://docs.docker.com/engine/install/ -``` - -### Skopeo - -**Required for:** Image inspection, tag verification - -```bash -# Check installation -skopeo --version - -# Installation -sudo dnf install skopeo # Fedora/RHEL/CentOS -sudo apt install skopeo # Ubuntu/Debian -brew install skopeo # macOS -``` - -### Git - -**Required for:** Repository cloning - -```bash -# Check installation -git --version - -# Installation -sudo dnf install git # Fedora/RHEL/CentOS -sudo apt install git # Ubuntu/Debian -brew install git # macOS (or Xcode Command Line Tools) -``` - -### SSH - -**Required for:** RHEL remote deployments - -```bash -# Check installation -ssh -V - -# Usually pre-installed on Linux/macOS -# Windows: Use OpenSSH or WSL -``` - -### curl and jq - -**Required for:** API calls and JSON parsing - -```bash -# Check installation -curl --version -jq --version - -# Installation -sudo dnf install curl jq # Fedora/RHEL/CentOS -sudo apt install curl jq # Ubuntu/Debian -brew install curl jq # macOS -``` - -## Cluster Requirements - -### OpenShift Cluster Access - -For S2I builds and deployments, you need: - -1. **Logged in to cluster:** - ```bash - oc login - # or - oc login --token= --server= - ``` - -2. **Namespace with edit permissions:** - ```bash - # Verify access - oc auth can-i create deployments - oc auth can-i create buildconfigs - ``` - -3. **Image registry accessible:** - ```bash - # Verify internal registry - oc get route -n openshift-image-registry - ``` - -### RHEL/Fedora Host Access - -For RHEL deployments, you need: - -1. **SSH access to target host:** - ```bash - ssh user@target-host - ``` - -2. **sudo privileges on target** (for systemd services) - -3. **Firewall ports open** (for application access) - -## Quick Validation - -Run these commands to check your environment: - -```bash -# Core tools -which oc helm podman git ssh curl jq skopeo - -# Cluster connection (if using OpenShift) -oc whoami -oc project - -# Container runtime -podman info || docker info -``` - -Use the `/validate-environment` skill for automated checking. diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/python-s2i-entrypoints.md b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/python-s2i-entrypoints.md deleted file mode 100644 index bb29398e..00000000 --- a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/python-s2i-entrypoints.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Python S2I Entry Point Requirements -category: containers -sources: - - title: UBI Python S2I Builder - url: https://github.com/sclorg/s2i-python-container - sections: Run script logic, APP_MODULE configuration - date_accessed: 2026-02-08 - - title: Red Hat Python S2I Documentation - url: https://catalog.redhat.com/software/containers/ubi9/python-311 - sections: Environment variables, Startup behavior - date_accessed: 2026-02-08 ---- - -# Python S2I Entry Point Requirements - -The UBI Python S2I builder has specific startup logic that must be understood to avoid deployment failures. - -## How the S2I Python Run Script Works - -The S2I Python builder uses this startup logic (in order): - -1. If `app.sh` exists → Execute it directly -2. If `gunicorn` is installed AND `APP_MODULE` is set → Start with gunicorn -3. If `app.py` exists → Run with Python directly -4. Otherwise → **ERROR: No start command found** - -## Entry Point Configuration Matrix - -| Entry Point File | gunicorn in requirements | Configuration Needed | Result | -|------------------|--------------------------|----------------------|--------| -| `app.py` | No | None | Works (Python direct) | -| `app.py` | Yes | None (optional APP_MODULE) | Works | -| `main.py` | **No** | - | **FAILS** | -| `main.py` | Yes | `APP_MODULE=main:app` | Works | -| `wsgi.py` | Yes | `APP_MODULE=wsgi` or `APP_MODULE=wsgi:application` | Works | -| Custom file | Yes | `APP_MODULE=[module]:[variable]` | Works | - -## APP_MODULE Format - -- **Format:** `[python_module]:[flask_app_variable]` -- **Example:** `main:app` → imports `app` from `main.py` -- **Requires:** `gunicorn` in `requirements.txt` - -### Common Patterns - -| File | Typical APP_MODULE | -|------|-------------------| -| `main.py` with `app = Flask(__name__)` | `main:app` | -| `main.py` with `application = Flask(__name__)` | `main:application` | -| `wsgi.py` with `application` | `wsgi:application` or just `wsgi` | -| `src/app.py` with `app` | `src.app:app` | - -## Alternative: APP_FILE - -- Set `APP_FILE=main.py` to run with Python directly (development mode) -- **Not recommended for production** (no WSGI server, no worker management) -- Use only if gunicorn is not an option - -## Critical Warning - -**If the entry point is NOT `app.py` and `gunicorn` is NOT installed:** -- The S2I build will succeed (dependencies install) -- The container will **fail to start** with "No start command found" -- This is a **runtime failure**, not a build failure - -**Always verify:** -1. Entry point file name -2. `gunicorn` in requirements.txt -3. `APP_MODULE` environment variable in BuildConfig diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/rhel-deployment.md b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/rhel-deployment.md deleted file mode 100644 index 06eda277..00000000 --- a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/rhel-deployment.md +++ /dev/null @@ -1,580 +0,0 @@ ---- -title: RHEL Deployment Reference -category: deployment -sources: - - title: RHEL System Administrator's Guide - systemd - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/managing-system-services-with-systemctl_configuring-basic-system-settings - sections: Managing services, Unit files - date_accessed: 2026-02-08 - - title: RHEL SELinux Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/using_selinux - sections: Contexts, Port labeling - date_accessed: 2026-02-08 - - title: RHEL Firewall Configuration - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_firewalls_and_packet_filters - sections: firewalld, Opening ports - date_accessed: 2026-02-08 ---- - -# RHEL Deployment Reference - -Reference material for deploying applications to standalone RHEL systems. - -## Table of Contents - -1. [RHEL Version Compatibility](#rhel-version-compatibility) -2. [Systemd Unit Templates](#systemd-unit-templates) -3. [SELinux Configuration](#selinux-configuration) -4. [Firewall Commands](#firewall-commands) -5. [SSH Connection Patterns](#ssh-connection-patterns) -6. [Runtime Package Mapping](#runtime-package-mapping) - ---- - -## RHEL Version Compatibility - -| Distribution | Version | Podman | Recommended | -|--------------|---------|--------|-------------| -| RHEL | 8.x | 4.0+ | Production ready | -| RHEL | 9.x | 4.4+ | **Recommended** | -| CentOS Stream | 8 | 4.0+ | Development | -| CentOS Stream | 9 | 4.4+ | Development | -| Rocky Linux | 8.x | 4.0+ | Production ready | -| Rocky Linux | 9.x | 4.4+ | Production ready | -| AlmaLinux | 8.x | 4.0+ | Production ready | -| AlmaLinux | 9.x | 4.4+ | Production ready | -| Fedora | 38+ | 4.6+ | Latest features | - -### Version Detection Commands - -```bash -# Get RHEL/CentOS version -cat /etc/redhat-release - -# Get detailed OS info -cat /etc/os-release - -# Check architecture -uname -m - -# Check kernel version -uname -r -``` - ---- - -## Systemd Unit Templates - -### Podman Container Service (Rootful) - -```ini -[Unit] -Description=${APP_NAME} Container -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -# Pre-start: ensure clean state -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} - -# Main container run -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} - -# Stop container gracefully -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Podman Container Service (Rootless) - -```ini -[Unit] -Description=${APP_NAME} Container (Rootless) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 -TimeoutStartSec=300 -TimeoutStopSec=70 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=default.target -``` - -**Rootless setup commands:** -```bash -# Create user systemd directory -mkdir -p ~/.config/systemd/user - -# Place unit file -cp ${APP_NAME}.service ~/.config/systemd/user/ - -# Reload and enable -systemctl --user daemon-reload -systemctl --user enable --now ${APP_NAME} - -# Keep services running after logout -loginctl enable-linger ${USER} -``` - -### Podman Container with Volumes - -```ini -[Unit] -Description=${APP_NAME} Container with Persistent Data -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -Restart=always -RestartSec=5 - -ExecStartPre=-/usr/bin/podman stop -t 10 ${APP_NAME} -ExecStartPre=-/usr/bin/podman rm ${APP_NAME} -ExecStart=/usr/bin/podman run \ - --name ${APP_NAME} \ - -p ${HOST_PORT}:${CONTAINER_PORT} \ - -v /var/lib/${APP_NAME}/data:/app/data:z \ - -e DATABASE_URL=${DATABASE_URL} \ - --rm \ - ${IMAGE} -ExecStop=/usr/bin/podman stop -t 10 ${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Node.js Application - -```ini -[Unit] -Description=${APP_NAME} Node.js Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=NODE_ENV=production -Environment=PORT=${PORT} -ExecStart=/usr/bin/node /opt/${APP_NAME}/server.js -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Python Application - -```ini -[Unit] -Description=${APP_NAME} Python Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PYTHONUNBUFFERED=1 -Environment=PORT=${PORT} -ExecStart=/usr/bin/python3 /opt/${APP_NAME}/app.py -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Java Application - -```ini -[Unit] -Description=${APP_NAME} Java Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=JAVA_OPTS=-Xmx512m -ExecStart=/usr/bin/java -jar /opt/${APP_NAME}/app.jar --server.port=${PORT} -Restart=always -RestartSec=5 -SuccessExitStatus=143 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -ReadWritePaths=/opt/${APP_NAME} - -[Install] -WantedBy=multi-user.target -``` - -### Native Go Application - -```ini -[Unit] -Description=${APP_NAME} Go Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=${SERVICE_USER} -WorkingDirectory=/opt/${APP_NAME} -Environment=PORT=${PORT} -ExecStart=/opt/${APP_NAME}/${BINARY_NAME} -Restart=always -RestartSec=5 - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - ---- - -## SELinux Configuration - -### Common SELinux Contexts - -| Context Type | Use Case | -|--------------|----------| -| `container_t` | Standard Podman container processes | -| `container_file_t` | Container data files | -| `bin_t` | Executable binaries | -| `httpd_sys_content_t` | Web application content (read-only) | -| `httpd_sys_rw_content_t` | Web application content (read-write) | -| `var_lib_t` | Application data in /var/lib | - -### Volume Label Options for Podman - -| Option | Description | Use Case | -|--------|-------------|----------| -| `:z` | Shared volume label | Volume accessed by multiple containers | -| `:Z` | Private volume label | Volume accessed by single container only | - -Example: -```bash -podman run -v /data/shared:/app/shared:z myimage # Shared -podman run -v /data/private:/app/data:Z myimage # Private -``` - -### SELinux Commands - -```bash -# Check current SELinux mode -getenforce - -# View file context -ls -Z /path/to/file - -# Set context for application directory -sudo semanage fcontext -a -t bin_t "/opt/myapp(/.*)?" -sudo restorecon -Rv /opt/myapp - -# Set context for web content -sudo semanage fcontext -a -t httpd_sys_content_t "/opt/myapp/public(/.*)?" -sudo restorecon -Rv /opt/myapp/public - -# Allow non-standard port for HTTP -sudo semanage port -a -t http_port_t -p tcp 8080 - -# View port contexts -sudo semanage port -l | grep http - -# Check for SELinux denials -sudo ausearch -m AVC -ts recent - -# Generate policy from denials (troubleshooting) -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy -sudo semodule -i mypolicy.pp - -# Temporarily set permissive (for debugging only) -sudo setenforce 0 -``` - -### Common SELinux Booleans - -```bash -# Allow HTTP to connect to network (for proxy/API calls) -sudo setsebool -P httpd_can_network_connect 1 - -# Allow HTTP to connect to databases -sudo setsebool -P httpd_can_network_connect_db 1 - -# List all HTTP-related booleans -getsebool -a | grep httpd -``` - ---- - -## Firewall Commands - -### Basic Port Management - -```bash -# Check firewall status -sudo firewall-cmd --state - -# List all open ports -sudo firewall-cmd --list-ports - -# List all services -sudo firewall-cmd --list-services - -# Open port permanently -sudo firewall-cmd --permanent --add-port=8080/tcp - -# Open port temporarily (until reload) -sudo firewall-cmd --add-port=8080/tcp - -# Reload firewall to apply permanent changes -sudo firewall-cmd --reload - -# Remove port -sudo firewall-cmd --permanent --remove-port=8080/tcp -sudo firewall-cmd --reload -``` - -### Service-Based Management - -```bash -# Add HTTP service -sudo firewall-cmd --permanent --add-service=http - -# Add HTTPS service -sudo firewall-cmd --permanent --add-service=https - -# Remove service -sudo firewall-cmd --permanent --remove-service=http - -# Apply changes -sudo firewall-cmd --reload -``` - -### Zone Management - -```bash -# List zones -sudo firewall-cmd --get-zones - -# Get active zone -sudo firewall-cmd --get-active-zones - -# Add port to specific zone -sudo firewall-cmd --zone=public --permanent --add-port=8080/tcp - -# Set default zone -sudo firewall-cmd --set-default-zone=public -``` - -### Rich Rules (Advanced) - -```bash -# Allow specific IP to access port -sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port protocol="tcp" port="8080" accept' - -# Rate limiting -sudo firewall-cmd --permanent --add-rich-rule='rule service name="http" limit value="10/m" accept' - -# Apply changes -sudo firewall-cmd --reload -``` - ---- - -## SSH Connection Patterns - -### Test Connection - -```bash -# Basic connection test -ssh -o BatchMode=yes -o ConnectTimeout=10 user@host "echo 'OK'" - -# Verbose output for debugging -ssh -v user@host - -# Test with specific key -ssh -i ~/.ssh/mykey user@host "echo 'OK'" -``` - -### Execute Remote Commands - -```bash -# Single command -ssh user@host "command" - -# Multiple commands -ssh user@host "cmd1 && cmd2 && cmd3" - -# With sudo -ssh user@host "sudo command" - -# Preserve environment -ssh user@host 'bash -l -c "command"' -``` - -### File Transfer - -```bash -# Copy file to remote -scp local_file user@host:/remote/path/ - -# Copy directory recursively -scp -r local_dir user@host:/remote/path/ - -# Using rsync (preferred for large transfers) -rsync -avz --progress local_dir/ user@host:/remote/path/ - -# Exclude patterns -rsync -avz --exclude 'node_modules' --exclude '.git' ./ user@host:/remote/path/ -``` - -### SSH Config for Convenience - -``` -# ~/.ssh/config -Host myrhel - HostName 192.168.1.100 - User deploy - Port 22 - IdentityFile ~/.ssh/id_rsa - StrictHostKeyChecking accept-new -``` - -Usage: `ssh myrhel "command"` - ---- - -## Runtime Package Mapping - -### Node.js - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 18 | `dnf module enable nodejs:18 && dnf install -y nodejs npm` | `dnf install -y nodejs npm` | -| 20 | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | `dnf module enable nodejs:20 && dnf install -y nodejs npm` | - -### Python - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.8 | `dnf install -y python38 python38-pip` | N/A | -| 3.9 | `dnf install -y python39 python39-pip` | `dnf install -y python3 python3-pip` | -| 3.11 | N/A | `dnf install -y python3.11 python3.11-pip` | -| 3.12 | N/A | `dnf install -y python3.12 python3.12-pip` | - -### Java - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 11 | `dnf install -y java-11-openjdk java-11-openjdk-devel` | `dnf install -y java-11-openjdk java-11-openjdk-devel` | -| 17 | `dnf install -y java-17-openjdk java-17-openjdk-devel` | `dnf install -y java-17-openjdk java-17-openjdk-devel` | -| 21 | N/A | `dnf install -y java-21-openjdk java-21-openjdk-devel` | - -### Go - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 1.20+ | `dnf install -y go-toolset` | `dnf install -y golang` | - -### Ruby - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 3.0 | `dnf module enable ruby:3.0 && dnf install -y ruby ruby-devel` | `dnf install -y ruby ruby-devel` | -| 3.1 | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | `dnf module enable ruby:3.1 && dnf install -y ruby ruby-devel` | - -### PHP - -| Version | RHEL 8 | RHEL 9 | -|---------|--------|--------| -| 7.4 | `dnf module enable php:7.4 && dnf install -y php php-cli php-fpm` | N/A | -| 8.0 | `dnf module enable php:8.0 && dnf install -y php php-cli php-fpm` | `dnf install -y php php-cli php-fpm` | -| 8.1 | N/A | `dnf module enable php:8.1 && dnf install -y php php-cli php-fpm` | - -### Module Stream Commands - -```bash -# List available streams for a module -dnf module list nodejs - -# Enable specific stream -sudo dnf module enable nodejs:20 - -# Reset module (to switch streams) -sudo dnf module reset nodejs - -# Install from enabled stream -sudo dnf install -y nodejs npm -``` - ---- - -## Service User Creation - -For running applications as non-root: - -```bash -# Create system user for the application -sudo useradd -r -s /sbin/nologin -d /opt/myapp myapp - -# Set ownership -sudo chown -R myapp:myapp /opt/myapp - -# Allow user to bind to privileged port (if needed) -sudo setcap 'cap_net_bind_service=+ep' /opt/myapp/binary -``` diff --git a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/selinux-troubleshooting.md b/evaluation/without_skills/rh-developer__validate-environment/environment/docs/selinux-troubleshooting.md deleted file mode 100644 index 9942375c..00000000 --- a/evaluation/without_skills/rh-developer__validate-environment/environment/docs/selinux-troubleshooting.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: SELinux Troubleshooting -category: references -sources: - - title: Red Hat SELinux User's and Administrator's Guide - url: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/using_selinux/index - sections: Troubleshooting, Managing confined services - date_accessed: 2026-02-16 - - title: SELinux Project Wiki - url: https://selinuxproject.org/page/Main_Page - sections: Troubleshooting - date_accessed: 2026-02-16 - - title: Fedora SELinux Guide - url: https://docs.fedoraproject.org/en-US/quick-docs/selinux-getting-started/ - sections: Troubleshooting - date_accessed: 2026-02-16 ---- - -# SELinux Troubleshooting - -This document provides guidance for diagnosing and resolving SELinux access denials on RHEL/Fedora/CentOS systems. - -## Understanding SELinux - -### SELinux Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| **Enforcing** | SELinux policy is enforced, denials are blocked and logged | Production | -| **Permissive** | SELinux policy is not enforced, denials are logged only | Debugging | -| **Disabled** | SELinux is completely disabled | Not recommended | - -```bash -# Check current mode -getenforce - -# Temporarily switch to permissive (until reboot) -sudo setenforce 0 - -# Switch back to enforcing -sudo setenforce 1 -``` - -### SELinux Contexts - -Every file, process, and port has an SELinux context: - -``` -user:role:type:level -``` - -Example: `system_u:object_r:httpd_sys_content_t:s0` - -- **user**: SELinux user (system_u, user_u, etc.) -- **role**: Role (object_r for files) -- **type**: Type label (most important for troubleshooting) -- **level**: MLS/MCS level (usually s0) - -```bash -# View file context -ls -lZ /path/to/file - -# View process context -ps auxZ | grep [process] - -# View port context -semanage port -l | grep [port] -``` - -## Finding SELinux Denials - -### Using ausearch - -```bash -# Recent denials (last 10 minutes) -sudo ausearch -m AVC -ts recent - -# Denials from today -sudo ausearch -m AVC -ts today - -# Denials for specific process -sudo ausearch -m AVC -c [command-name] - -# Denials involving specific file -sudo ausearch -m AVC -f /path/to/file -``` - -### Using journalctl - -```bash -# SELinux messages in journal -sudo journalctl -t setroubleshoot - -# AVC messages -sudo journalctl | grep "avc: denied" -``` - -### Using sealert - -```bash -# Install setroubleshoot (if not installed) -sudo dnf install setroubleshoot-server - -# Analyze all denials -sudo sealert -a /var/log/audit/audit.log - -# Interactive analysis -sudo sealert -b -``` - -## Reading AVC Denials - -Example AVC denial: - -``` -type=AVC msg=audit(1234567890.123:456): avc: denied { bind } for pid=1234 comm="httpd" src=8080 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket permissive=0 -``` - -**Breakdown:** -| Field | Value | Meaning | -|-------|-------|---------| -| `denied { bind }` | bind | Denied action (bind to socket) | -| `pid=1234` | 1234 | Process ID | -| `comm="httpd"` | httpd | Command name | -| `src=8080` | 8080 | Port number | -| `scontext=...httpd_t...` | httpd_t | Source type (process) | -| `tcontext=...unreserved_port_t...` | unreserved_port_t | Target type (port) | -| `tclass=tcp_socket` | tcp_socket | Object class | - -**Translation:** Process `httpd` (type `httpd_t`) was denied permission to `bind` to port `8080` (type `unreserved_port_t`). - -## Common Denial Types and Fixes - -### Port Binding Denials - -**Symptom:** Application cannot bind to non-standard port - -**Example denial:** -``` -avc: denied { name_bind } for comm="nginx" src=8080 scontext=httpd_t tcontext=unreserved_port_t -``` - -**Fix:** -```bash -# Add port to allowed type -sudo semanage port -a -t http_port_t -p tcp 8080 - -# Verify -sudo semanage port -l | grep 8080 -``` - -**Common port types:** -| Port Type | Typical Ports | Used By | -|-----------|---------------|---------| -| `http_port_t` | 80, 443, 8080 | Web servers | -| `postgresql_port_t` | 5432 | PostgreSQL | -| `mysqld_port_t` | 3306 | MySQL/MariaDB | -| `redis_port_t` | 6379 | Redis | -| `mongod_port_t` | 27017 | MongoDB | - -### File Access Denials - -**Symptom:** Application cannot read/write files - -**Example denial:** -``` -avc: denied { read } for comm="httpd" name="config.yaml" scontext=httpd_t tcontext=user_home_t -``` - -**Fix - Change file context:** -```bash -# Set file context pattern -sudo semanage fcontext -a -t httpd_sys_content_t "/srv/myapp(/.*)?" - -# Apply the context -sudo restorecon -Rv /srv/myapp - -# Verify -ls -lZ /srv/myapp -``` - -**Common file types:** -| File Type | Access | Use Case | -|-----------|--------|----------| -| `httpd_sys_content_t` | Read | Web content | -| `httpd_sys_rw_content_t` | Read/Write | Web app data | -| `container_file_t` | Container access | Podman volumes | -| `var_log_t` | Log files | Application logs | - -### Network Connection Denials - -**Symptom:** Application cannot connect to external services - -**Example denial:** -``` -avc: denied { name_connect } for comm="httpd" dest=5432 scontext=httpd_t tcontext=postgresql_port_t -``` - -**Fix - Enable boolean:** -```bash -# Allow httpd to connect to network -sudo setsebool -P httpd_can_network_connect on - -# Or specifically to databases -sudo setsebool -P httpd_can_network_connect_db on - -# List all httpd booleans -sudo getsebool -a | grep httpd -``` - -**Common booleans:** -| Boolean | Purpose | -|---------|---------| -| `httpd_can_network_connect` | Allow outbound network connections | -| `httpd_can_network_connect_db` | Allow database connections | -| `httpd_can_sendmail` | Allow sending email | -| `httpd_use_nfs` | Allow NFS access | -| `container_manage_cgroup` | Allow container cgroup management | - -## Container-Specific Issues - -### Podman Volume Mounts - -When mounting host directories into containers, SELinux may block access. - -**Solutions:** - -1. **Shared label (:z)** - Multiple containers can access - ```bash - podman run -v /host/path:/container/path:z [image] - ``` - -2. **Private label (:Z)** - Only this container can access - ```bash - podman run -v /host/path:/container/path:Z [image] - ``` - -3. **Manual relabeling:** - ```bash - sudo semanage fcontext -a -t container_file_t "/data(/.*)?" - sudo restorecon -Rv /data - ``` - -### Container Booleans - -```bash -# Enable container to manage cgroups (for systemd in container) -sudo setsebool -P container_manage_cgroup on - -# Allow containers to connect to any port -sudo setsebool -P container_connect_any on - -# List all container booleans -sudo getsebool -a | grep container -``` - -## Troubleshooting Workflow - -### Step 1: Confirm SELinux is the Issue - -```bash -# Temporarily disable SELinux -sudo setenforce 0 - -# Test if application works -[test application] - -# Re-enable SELinux -sudo setenforce 1 -``` - -If application works with SELinux permissive, SELinux is blocking. - -### Step 2: Find the Denial - -```bash -# Get recent denials -sudo ausearch -m AVC -ts recent - -# Or use sealert for analysis -sudo sealert -a /var/log/audit/audit.log -``` - -### Step 3: Determine Fix Type - -| Denial Type | Fix Approach | -|-------------|--------------| -| Port binding | `semanage port` | -| File access | `semanage fcontext` + `restorecon` | -| Network connection | `setsebool` | -| Process capability | Custom policy or boolean | - -### Step 4: Apply Fix - -```bash -# For port: -sudo semanage port -a -t [type] -p [tcp/udp] [port] - -# For file: -sudo semanage fcontext -a -t [type] "[path](/.*)?" -sudo restorecon -Rv [path] - -# For boolean: -sudo setsebool -P [boolean] on -``` - -### Step 5: Verify - -```bash -# Test application -[restart and test] - -# Check for new denials -sudo ausearch -m AVC -ts recent -``` - -## Generating Custom Policies - -If no existing type or boolean works, generate a custom policy: - -```bash -# Generate policy from recent denials -sudo ausearch -m AVC -ts recent | audit2allow -M mypolicy - -# Review the policy -cat mypolicy.te - -# Install the policy -sudo semodule -i mypolicy.pp -``` - -**Warning:** Custom policies should be reviewed carefully. They grant permanent permissions. - -## Quick Reference - -### Common Commands - -```bash -# SELinux status -getenforce -sestatus - -# File context -ls -lZ [path] -restorecon -Rv [path] - -# Process context -ps auxZ | grep [process] - -# Port context -semanage port -l | grep [port] -semanage port -a -t [type] -p tcp [port] - -# Booleans -getsebool -a | grep [keyword] -setsebool -P [boolean] on - -# File context rules -semanage fcontext -l | grep [path] -semanage fcontext -a -t [type] "[path](/.*)?" - -# Audit logs -ausearch -m AVC -ts recent -sealert -a /var/log/audit/audit.log -``` - -### Common Types for Web Applications - -| Resource | Type | -|----------|------| -| Web content (read-only) | `httpd_sys_content_t` | -| Web content (read-write) | `httpd_sys_rw_content_t` | -| Web scripts | `httpd_sys_script_exec_t` | -| Application logs | `httpd_log_t` | -| HTTP ports | `http_port_t` | -| Container files | `container_file_t` | - -### Common Booleans for Applications - -| Application | Boolean | Purpose | -|-------------|---------|---------| -| Web server | `httpd_can_network_connect` | Outbound connections | -| Web server | `httpd_can_network_connect_db` | Database connections | -| Web server | `httpd_unified` | Unified handling | -| Container | `container_manage_cgroup` | cgroup management | -| Container | `container_connect_any` | Connect to any port | -| NFS | `use_nfs_home_dirs` | NFS home directories | diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__cve-impact/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 6c122770..00000000 --- a/evaluation/without_skills/rh-sre__cve-impact/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- Pass Rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__cve-validation/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 6c122770..00000000 --- a/evaluation/without_skills/rh-sre__cve-validation/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- Pass Rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__execution-summary/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 6c122770..00000000 --- a/evaluation/without_skills/rh-sre__execution-summary/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- Pass Rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 6c122770..00000000 --- a/evaluation/without_skills/rh-sre__fleet-inventory/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- Pass Rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 6c122770..00000000 --- a/evaluation/without_skills/rh-sre__job-template-creator/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- Pass Rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 6c122770..00000000 --- a/evaluation/without_skills/rh-sre__job-template-remediation-validator/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- Pass Rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 6c122770..00000000 --- a/evaluation/without_skills/rh-sre__mcp-aap-validator/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- Pass Rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 6c122770..00000000 --- a/evaluation/without_skills/rh-sre__mcp-lightspeed-validator/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- Pass Rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 6c122770..00000000 --- a/evaluation/without_skills/rh-sre__playbook-executor/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- Pass Rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 6c122770..00000000 --- a/evaluation/without_skills/rh-sre__playbook-generator/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- Pass Rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 6c122770..00000000 --- a/evaluation/without_skills/rh-sre__remediation-verifier/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- Pass Rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/without_skills/rh-sre__remediation/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/without_skills/rh-sre__remediation/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/without_skills/rh-sre__remediation/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/without_skills/rh-sre__remediation/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/without_skills/rh-sre__remediation/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/without_skills/rh-sre__remediation/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/without_skills/rh-sre__remediation/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/without_skills/rh-sre__remediation/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/without_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/without_skills/rh-sre__remediation/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/without_skills/rh-sre__remediation/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/without_skills/rh-sre__remediation/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/without_skills/rh-sre__remediation/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__remediation/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__remediation/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 6c122770..00000000 --- a/evaluation/without_skills/rh-sre__remediation/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- Pass Rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/cross-reference-graph.json b/evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/cross-reference-graph.json deleted file mode 100644 index 20837038..00000000 --- a/evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/cross-reference-graph.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Document relationship graph for intelligent follow-up doc discovery. Enables 'if you read X, you probably also need Y' inference.", - "graph": { - "ansible/cve-remediation-templates.md": { - "complements": [ - { - "doc": "rhel/package-management.md", - "reason": "Templates reference DNF/YUM patterns and reboot detection methods", - "confidence": 0.95, - "use_when": "Any package update template (1, 4, 6)" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Templates should include error handling and rollback strategies", - "confidence": 0.85, - "use_when": "Batch remediation or complex workflows" - } - ], - "prerequisites": [ - { - "doc": "ansible/README.md", - "reason": "Overview of Ansible documentation structure", - "confidence": 0.50, - "use_when": "First-time users or orientation needed" - } - ], - "specializations": [ - { - "doc": "rhel/selinux-context.md", - "condition": "if selinux_cve", - "reason": "SELinux CVEs need context restoration patterns", - "confidence": 0.90, - "use_when": "Template 5 (SELinux update)" - } - ] - }, - "rhel/package-management.md": { - "complements": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Playbook templates implement these package management patterns", - "confidence": 0.95, - "use_when": "Creating or modifying remediation playbooks" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "Multi-version environments need compatibility guidance", - "confidence": 0.75, - "use_when": "Managing RHEL 7, 8, and 9 simultaneously" - }, - { - "doc": "rhel/systemd-services.md", - "reason": "Package updates often require service restarts", - "confidence": 0.70, - "use_when": "Service restart needed after package update" - } - ], - "prerequisites": [], - "specializations": [] - }, - "ansible/README.md": { - "leads_to": [ - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "Primary Ansible documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to detailed templates" - }, - { - "doc": "ansible/error-handling.md", - "reason": "Error handling patterns for production playbooks", - "confidence": 0.70, - "use_when": "Need advanced error handling beyond templates" - } - ] - }, - "rhel/README.md": { - "leads_to": [ - { - "doc": "rhel/package-management.md", - "reason": "Primary RHEL documentation for CVE remediation", - "confidence": 0.95, - "use_when": "User navigating from overview to package management" - }, - { - "doc": "rhel/version-compatibility.md", - "reason": "RHEL version differences and compatibility", - "confidence": 0.75, - "use_when": "Multi-version environment" - } - ] - }, - "insights/vulnerability-logic.md": { - "complements": [ - { - "doc": "references/cvss-scoring.md", - "reason": "CVSS scoring complements Red Hat vulnerability assessment", - "confidence": 0.90, - "use_when": "Performing risk assessment" - }, - { - "doc": "ansible/cve-remediation-templates.md", - "reason": "After assessment, create remediation playbook", - "confidence": 0.85, - "use_when": "Proceeding from assessment to remediation" - } - ], - "prerequisites": [], - "specializations": [] - }, - "references/cvss-scoring.md": { - "complements": [ - { - "doc": "insights/vulnerability-logic.md", - "reason": "Red Hat-specific risk assessment methodology", - "confidence": 0.90, - "use_when": "Understanding Red Hat severity ratings" - }, - { - "doc": "references/compliance-frameworks.md", - "reason": "Compliance requirements often tied to CVSS scores", - "confidence": 0.70, - "use_when": "Compliance-driven remediation" - } - ], - "prerequisites": [], - "specializations": [] - } - }, - "relationship_types": { - "complements": "Documents that enhance each other when read together. High confidence indicates strong recommendation.", - "prerequisites": "Documents that should be read before the current document. Lower confidence = optional background.", - "specializations": "Documents needed only under specific conditions. Check condition field before loading.", - "leads_to": "Natural navigation path from overview to detailed content." - }, - "usage_instructions": { - "for_agents": "After reading a document, check its graph entry for related docs. Load 'complements' with confidence > 0.80. Check 'specializations' conditions against current task context.", - "confidence_thresholds": { - "high": ">= 0.90 - Always load if relevant", - "medium": "0.70-0.89 - Load if task complexity warrants it", - "low": "< 0.70 - Optional, load only if explicitly needed" - }, - "condition_evaluation": "Evaluate specializations conditions using task context. Common conditions: kubernetes_system, selinux_cve, package_update, kernel_update, batch_remediation." - } -} diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/semantic-index.json b/evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/semantic-index.json deleted file mode 100644 index 9cfee28f..00000000 --- a/evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/semantic-index.json +++ /dev/null @@ -1,297 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "documents": [ - { - "path": "ansible/cve-remediation-templates.md", - "title": "CVE Remediation Playbook Templates", - "category": "ansible", - "tags": ["cve", "remediation", "playbooks", "templates", "ansible"], - "semantic_keywords": [ - "package update", - "kernel update", - "service restart", - "configuration change", - "SELinux context", - "batch remediation", - "reboot handling", - "rollback strategy", - "idempotent playbook", - "CVE patch", - "Ansible playbook", - "error handling", - "audit logging" - ], - "use_cases": [ - "package_update_cve", - "kernel_cve", - "service_restart_cve", - "config_file_cve", - "selinux_cve", - "batch_remediation" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "6 production-ready Ansible playbook templates for CVE remediation with error handling, rollback, and audit logging. Includes package updates, kernel updates, service restarts, config changes, SELinux fixes, and batch operations.", - "related_docs": [ - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "token_estimate": 2500 - }, - { - "path": "rhel/package-management.md", - "title": "RHEL Package Management for CVE Remediation", - "category": "rhel", - "tags": ["dnf", "yum", "package-management", "rhel", "updates", "reboot-detection"], - "semantic_keywords": [ - "DNF package manager", - "YUM package manager", - "package update", - "repository management", - "reboot detection", - "systemd service management", - "needs-restarting", - "subscription manager", - "RHEL compatibility", - "service restart" - ], - "use_cases": [ - "package_update_cve", - "rhel_version_compatibility", - "reboot_detection", - "service_restart_after_update" - ], - "rhel_versions": ["rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Comprehensive RHEL 7/8/9 package management patterns including DNF/YUM workflows, reboot detection with needs-restarting, service restart logic, repository management, and subscription manager integration.", - "related_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/version-compatibility.md", - "rhel/systemd-services.md" - ], - "token_estimate": 1600 - }, - { - "path": "insights/vulnerability-logic.md", - "title": "Red Hat Lightspeed Vulnerability Assessment Logic", - "category": "insights", - "tags": ["insights", "vulnerability", "cve", "risk-assessment", "remediation"], - "semantic_keywords": [ - "Red Hat Lightspeed", - "vulnerability assessment", - "CVE risk scoring", - "vulnerable vs affected", - "security rules", - "severity rating", - "remediation priority", - "CVSS scoring", - "threat intelligence", - "patch prioritization", - "Red Hat severity", - "exploit availability", - "priority matrix" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "vulnerability_analysis", - "remediation_planning", - "compliance_reporting" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "Explains how Red Hat Lightspeed assesses CVE vulnerabilities using vulnerable vs affected classification, Security Rules designation, Red Hat severity ratings, and remediation prioritization methodology. Includes priority decision matrix and integration guidance.", - "related_docs": [ - "references/cvss-scoring.md", - "ansible/cve-remediation-templates.md", - "references/compliance-frameworks.md" - ], - "token_estimate": 900 - }, - { - "path": "references/cvss-scoring.md", - "title": "CVSS Scoring and Red Hat Severity Mappings", - "category": "references", - "tags": ["cvss", "severity", "scoring", "risk-assessment", "priority"], - "semantic_keywords": [ - "CVSS score", - "severity rating", - "Red Hat severity", - "vulnerability scoring", - "risk assessment", - "priority matrix", - "CVSS v3.1", - "attack vector", - "exploitability", - "impact metrics", - "CVSS calculator", - "compliance requirements", - "PCI-DSS", - "SOC 2", - "NIST 800-53" - ], - "use_cases": [ - "risk_assessment", - "cve_prioritization", - "compliance_reporting", - "stakeholder_communication", - "severity_interpretation" - ], - "rhel_versions": ["rhel6", "rhel7", "rhel8", "rhel9"], - "applies_to_systems": ["bare_metal", "vm", "kubernetes", "openshift"], - "content_summary": "CVSS v3.1 interpretation guide with all 8 metrics explained, Red Hat severity mappings, priority decision matrix, real-world CVE examples, and compliance framework requirements (PCI-DSS, SOC 2, NIST).", - "related_docs": [ - "insights/vulnerability-logic.md", - "references/compliance-frameworks.md", - "ansible/cve-remediation-templates.md" - ], - "token_estimate": 1000 - } - ], - "task_mappings": { - "package_update_rhel": { - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "Standard package update CVE on bare metal or VM RHEL systems" - }, - "service_restart_cve": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt"], - "estimated_tokens": 4100, - "description": "CVE requiring service configuration changes and restart" - }, - "selinux_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": ["templates", "selinux-context"], - "estimated_tokens": 2500, - "description": "CVE affecting SELinux file contexts or policies" - }, - "batch_remediation": { - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": ["templates", "package-mgmt", "error-handling"], - "estimated_tokens": 4100, - "description": "Multiple CVEs across fleet of systems" - }, - "risk_assessment": { - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": ["vulnerability-logic", "cvss-scoring"], - "estimated_tokens": 1900, - "description": "Analyze CVE impact without creating remediation, including Red Hat Lightspeed assessment and CVSS interpretation" - } - }, - "inference_hints": { - "cve_type_detection": { - "kernel": { - "keywords": ["kernel", "vmlinuz", "grub", "reboot required", "kernel-", "linux kernel"], - "confidence": 0.95, - "implies_reboot": true, - "recommended_templates": ["template_4_kernel_update"] - }, - "package": { - "keywords": ["dnf", "yum", "rpm", "package update", "httpd", "openssl", "glibc", "python"], - "confidence": 0.85, - "implies_reboot": false, - "recommended_templates": ["template_1_package_update"] - }, - "service": { - "keywords": ["systemd", "service restart", "daemon", "sshd", "nginx", "apache"], - "confidence": 0.80, - "implies_reboot": false, - "recommended_templates": ["template_2_service_restart"] - }, - "configuration": { - "keywords": ["config file", "sshd_config", "httpd.conf", "sysctl", "kernel parameter"], - "confidence": 0.75, - "implies_reboot": false, - "recommended_templates": ["template_3_config_update"] - }, - "selinux": { - "keywords": ["SELinux", "restorecon", "semanage", "context", "selinux policy"], - "confidence": 0.90, - "implies_reboot": false, - "recommended_templates": ["template_5_selinux"] - } - }, - "system_type_detection": { - "kubernetes": { - "keywords": ["pod", "deployment", "namespace", "k8s", "kubectl", "container"], - "confidence": 0.90, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "openshift": { - "keywords": ["OpenShift", "OCP", "oc", "route", "project", "openshift"], - "confidence": 0.95, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "bare_metal": { - "keywords": ["physical", "hardware", "IPMI", "bare metal", "bmc"], - "confidence": 0.70, - "requires_pod_eviction": false, - "additional_docs": [] - }, - "vm": { - "keywords": ["virtual", "VM", "hypervisor", "guest", "virtual machine", "vmware", "kvm"], - "confidence": 0.75, - "requires_pod_eviction": false, - "additional_docs": [] - } - }, - "rhel_version_detection": { - "rhel7": { - "keywords": ["rhel7", "rhel 7", "centos 7", "yum", "python 2.7"], - "package_manager": "yum", - "systemd_version": "219", - "needs_restarting_available": false - }, - "rhel8": { - "keywords": ["rhel8", "rhel 8", "centos 8", "dnf", "python 3.6"], - "package_manager": "dnf", - "systemd_version": "239", - "needs_restarting_available": true - }, - "rhel9": { - "keywords": ["rhel9", "rhel 9", "centos 9", "dnf", "python 3.9"], - "package_manager": "dnf", - "systemd_version": "252", - "needs_restarting_available": true - } - } - }, - "usage_instructions": { - "for_agents": "Read this semantic index first (~200 tokens) to intelligently discover relevant documentation. Use task_mappings for common workflows or semantic_keywords for custom queries. Follow workflow_order for optimal context loading.", - "query_based_discovery": "Match user query terms against semantic_keywords across all documents. Filter by rhel_versions and applies_to_systems based on context. Use inference_hints to detect CVE type and system type automatically.", - "token_optimization": "Load only required_docs first. Add optional_docs only if needed based on task complexity. Check estimated_tokens to manage context budget.", - "maintenance": "Regenerate this index when docs are added/updated using docs/.ai-index/generate-index.py. Version is semantic (MAJOR.MINOR for breaking changes)." - } -} diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/task-to-docs-mapping.json b/evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/task-to-docs-mapping.json deleted file mode 100644 index 19eff879..00000000 --- a/evaluation/without_skills/rh-sre__system-context/environment/docs/.ai-index/task-to-docs-mapping.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "version": "1.0", - "generated": "2026-01-20T00:00:00Z", - "description": "Pre-computed document mappings for common CVE remediation workflows. Eliminates search overhead by providing direct doc access based on task type.", - "task_mappings": { - "kernel_cve_bare_metal": { - "description": "Kernel CVE on bare metal or VM RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 4: Kernel Update with Reboot", - "rhel/package-management.md": "Pattern 3: Kernel Package Updates, Reboot Detection Patterns" - }, - "prerequisites": [ - "System backup created", - "Maintenance window scheduled" - ] - }, - "package_update_rhel": { - "description": "Standard package update CVE on RHEL systems (non-Kubernetes)", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update", - "rhel/package-management.md": "Pattern 1: Single Package Update, Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "RHEL subscription active", - "Repository access verified" - ] - }, - "service_restart_cve": { - "description": "CVE requiring service configuration changes and restart", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 2: Service Restart", - "rhel/package-management.md": "Service Restart After Package Updates" - }, - "prerequisites": [ - "Service configuration backup", - "Config syntax validation available" - ] - }, - "config_file_cve": { - "description": "CVE requiring system configuration file updates", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 3: Configuration File Update" - }, - "prerequisites": [ - "Configuration backup created", - "Changes reviewed and approved" - ] - }, - "selinux_remediation": { - "description": "CVE affecting SELinux file contexts or policies", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "rhel/selinux-context.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/selinux-context.md" - ], - "estimated_tokens": 2500, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 5: SELinux Context Update" - }, - "prerequisites": [ - "SELinux enabled and enforcing", - "AVC denials reviewed" - ] - }, - "batch_remediation": { - "description": "Multiple CVEs across fleet of RHEL systems", - "required_docs": [ - "ansible/cve-remediation-templates.md" - ], - "optional_docs": [ - "ansible/error-handling.md", - "rhel/version-compatibility.md", - "rhel/package-management.md" - ], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md", - "ansible/error-handling.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 6: Batch Remediation", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages" - }, - "prerequisites": [ - "Inventory grouped by environment", - "Batch size determined", - "Rollback plan documented" - ] - }, - "risk_assessment": { - "description": "Analyze CVE impact without creating remediation", - "required_docs": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "optional_docs": [ - "references/compliance-frameworks.md" - ], - "workflow_order": [ - "insights/vulnerability-logic.md", - "references/cvss-scoring.md" - ], - "estimated_tokens": 1000, - "critical_sections": { - "insights/vulnerability-logic.md": "CVE Risk Assessment Methodology", - "references/cvss-scoring.md": "Red Hat Severity Mappings" - }, - "prerequisites": [ - "CVE ID known", - "Red Hat Lightspeed access" - ] - }, - "httpd_cve": { - "description": "Apache httpd package CVE remediation", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (httpd example)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (httpd + httpd-tools)" - }, - "prerequisites": [ - "httpd service can be restarted", - "Web traffic can tolerate brief interruption" - ] - }, - "openssl_cve": { - "description": "OpenSSL library CVE requiring service restarts", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (service restart patterns)", - "rhel/package-management.md": "Pattern 2: Multiple Related Packages (openssl + openssl-libs), Service Restart After Package Updates" - }, - "prerequisites": [ - "All services using openssl identified", - "Service restart order planned (dependencies)" - ] - }, - "glibc_cve": { - "description": "glibc CVE requiring reboot", - "required_docs": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "optional_docs": [], - "workflow_order": [ - "ansible/cve-remediation-templates.md", - "rhel/package-management.md" - ], - "estimated_tokens": 4100, - "critical_sections": { - "ansible/cve-remediation-templates.md": "Template 1: Package Update (reboot handling)", - "rhel/package-management.md": "Reboot Detection Patterns (needs-restarting)" - }, - "prerequisites": [ - "Reboot can be scheduled", - "Maintenance window available" - ] - } - }, - "usage_guide": { - "how_to_use": "Agents should check this mapping first for common task patterns. If exact match found, load docs in workflow_order. If no match, fall back to semantic-index.json keyword search.", - "token_savings": "Pre-computed mappings eliminate navigation overhead. Agents skip INDEX.md and category READMEs, directly loading relevant docs.", - "workflow_order_importance": "Order matters for context building. Load docs sequentially in specified order for optimal comprehension.", - "prerequisites_check": "Verify prerequisites before starting remediation. Prevents failures mid-workflow." - } -} diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/INDEX.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/INDEX.md deleted file mode 100644 index 4fa6725f..00000000 --- a/evaluation/without_skills/rh-sre__system-context/environment/docs/INDEX.md +++ /dev/null @@ -1,389 +0,0 @@ ---- -title: Red Hat Remediation Agent - Documentation Index -category: meta -sources: - - title: Red Hat Product Documentation - url: https://docs.redhat.com - sections: RHEL, OpenShift, Ansible Automation Platform, Red Hat Lightspeed - date_accessed: 2026-02-24 -last_updated: 2026-02-24 ---- - -# Red Hat Remediation Agent - Documentation Index - -This knowledge base provides comprehensive Red Hat-specific patterns for CVE remediation on Kubernetes-managed RHEL systems. - -## Quick Navigation - -### Priority P0 (Core Documentation) -- **[CVE Remediation Playbook Templates](ansible/cve-remediation-templates.md)** ⭐ HIGHEST VALUE - - 6 production-ready Ansible playbook templates - - Package updates, kernel updates, service restarts, SELinux, batch remediation - -- **[RHEL Package Management](rhel/package-management.md)** - - DNF/YUM workflows for RHEL 7/8/9 - - Systemd service management - - Reboot detection and handling - -### Priority P1 (Extended Documentation) -- **[Red Hat Lightspeed Vulnerability Logic](insights/vulnerability-logic.md)** ✅ - - CVE risk assessment methodology - - CVSS score interpretation - - System inventory correlation - -- **[CVSS Scoring Reference](references/cvss-scoring.md)** ✅ - - CVSS v3.1 metrics breakdown - - Red Hat severity mappings (Critical/Important/Moderate/Low) - - Priority decision matrix - -- **[Lightspeed MCP Parameters](references/lightspeed-mcp-parameters.md)** ✅ - - Correct parameter names for Lightspeed MCP tools (e.g. `per_page` not `page_size` for list_hosts) - - Consult before calling inventory__list_hosts to avoid validation errors - -- **[Lightspeed MCP Tool Failures](references/lightspeed-mcp-tool-failures.md)** ✅ - - Generic pattern for backend errors (e.g. explain_cves `'dnf_modules'`) — user-friendly message, workarounds, no raw error exposure - -- **RHEL Version Compatibility** (planned) - - RHEL 7/8/9 compatibility matrix - - Package naming differences - - Migration considerations - -- **SELinux Context Remediation** (planned) - - SELinux context fixes in playbooks - - `restorecon` patterns - - Policy package updates - -- **Ansible Error Handling** (planned) - - Block/rescue/always patterns - - Rollback strategies - - Idempotency best practices - -- **OpenShift Node Maintenance** (planned) - - Node drain procedures - - Maintenance mode patterns - - Uncordoning after updates - -- **RHEL 9 Security Hardening** (planned) - - RHEL 9 security baseline - - CIS benchmark alignment - - Common hardening patterns - -### Priority P2 (Reference Documentation - Planned) -- **Ansible Playbook Patterns** (planned) - - Reusable playbook components - - Variable management - - Role organization - -- **Ansible Automation Platform Integration** (planned) - - AAP/Tower workflows - - Job template configuration - - Credential management - -- **OpenShift Rolling Updates** (planned) - - Deployment strategies - - StatefulSet handling - - Health check verification - -- **OpenShift Security & Compliance** (planned) - - OCP security best practices - - Compliance scanning - - Security context constraints - -- **Compliance Frameworks** (planned) - - PCI-DSS requirements - - SOC 2 controls - - NIST guidelines - -- **RHEL Systemd Services** (planned) - - Service management patterns - - Service restart logic - - Health checks - -## Documentation Structure - -``` -docs/ -├── INDEX.md (this file) ✅ -├── SOURCES.md (official Red Hat source attribution) ✅ -├── rhel/ # RHEL-specific patterns -│ ├── README.md ✅ -│ ├── package-management.md (P0) ✅ -│ ├── selinux-context.md (P1 - planned) -│ ├── systemd-services.md (P2 - planned) -│ ├── version-compatibility.md (P1 - planned) -│ └── security-hardening-rhel9.md (P1 - planned) -├── ansible/ # Ansible playbook patterns -│ ├── README.md ✅ -│ ├── cve-remediation-templates.md (P0) ⭐ ✅ -│ ├── playbook-patterns.md (P2 - planned) -│ ├── error-handling.md (P1 - planned) -│ ├── idempotency.md (P2 - planned) -│ └── aap-integration.md (P2 - planned) -├── insights/ # Red Hat Lightspeed patterns -│ ├── README.md ✅ -│ ├── vulnerability-logic.md (P1) ✅ -│ ├── remediation-workflow.md (P2 - planned) -│ └── system-inventory.md (P2 - planned) -├── references/ # Reference documentation -│ ├── README.md ✅ -│ ├── cvss-scoring.md (P1) ✅ -│ ├── compliance-frameworks.md (P2 - planned) -│ └── glossary.md (P2 - planned) -└── .ai-index/ # AI inference optimization - ├── semantic-index.json ✅ - ├── task-to-docs-mapping.json ✅ - ├── cross-reference-graph.json ✅ - └── generate-index.py (planned) -``` - -## How to Use This Documentation (For AI Agents) - -### 1. Intelligent Document Discovery - -**Always start by reading the semantic index**: -``` -Read: docs/.ai-index/semantic-index.json (~200 tokens) -``` - -The semantic index enables: -- **Query-based discovery**: Match semantic keywords to your task -- **Task mapping shortcuts**: Pre-computed doc sets for common workflows -- **CVE type inference**: Automatic doc selection based on CVE characteristics -- **System type detection**: Context-aware doc loading (K8s vs bare metal) - -### 2. Task-Based Document Loading - -**Example Workflow - Kernel CVE**: -``` -1. Read semantic-index.json -2. Detect: CVE type = "kernel" (requires reboot) -3. Load from task_mappings["kernel_cve"]: - - ansible/cve-remediation-templates.md (Template 4: Kernel Update) - - rhel/package-management.md (DNF/YUM workflows) -4. Generate playbook using patterns from loaded docs -``` - -**Token Savings**: ~2,500-4,000 tokens (85% reduction in navigation overhead) - -### 3. Progressive Disclosure Pattern - -**Load docs incrementally as needed**: -- **Phase 1 (Validation)**: Load vulnerability-logic.md for risk assessment -- **Phase 2 (Context)**: Load package-management.md for RHEL-specific considerations -- **Phase 3 (Generation)**: Load cve-remediation-templates.md for playbook patterns - -### 4. Cross-Reference Navigation - -Use the cross-reference graph to find related documentation: -``` -If reading: ansible/cve-remediation-templates.md -Also consider: - - rhel/package-management.md (complements: DNF patterns) ✅ - - insights/vulnerability-logic.md (prerequisite: for risk assessment) ✅ -``` - -## Common Remediation Workflows - -### Workflow 1: Simple Package CVE -**Task**: "Remediate CVE-2024-XXXX affecting httpd package on RHEL 8" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 1: Package Update) ✅ -2. `rhel/package-management.md` (DNF workflows) ✅ - -### Workflow 2: Kernel CVE -**Task**: "Remediate kernel CVE on RHEL production nodes" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 4: Kernel Update) ✅ -2. `rhel/package-management.md` (kernel update procedures) ✅ - -### Workflow 3: Batch Remediation -**Task**: "Remediate 5 CVEs across 20 RHEL servers" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 6: Batch) ✅ -2. `rhel/package-management.md` (for RHEL-specific patterns) ✅ - -### Workflow 4: Risk Assessment -**Task**: "Analyze impact of CVE-2024-YYYY" - -**Required Docs**: -1. `insights/vulnerability-logic.md` (Red Hat risk methodology) ✅ -2. `references/cvss-scoring.md` (CVSS interpretation) ✅ - -### Workflow 5: SELinux CVE -**Task**: "Fix SELinux context vulnerability" - -**Required Docs**: -1. `ansible/cve-remediation-templates.md` (Template 5: SELinux) ✅ -2. `rhel/package-management.md` (for RHEL-specific SELinux package handling) ✅ - -## Documentation Quality Standards - -All documents follow these standards: - -### YAML Frontmatter (Required) -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Red Hat Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keyword1, keyword2, keyword3] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: [keyword phrases for AI discovery] -use_cases: [use_case_ids for task mapping] -related_docs: [cross-references] -last_updated: YYYY-MM-DD ---- -``` - -### Content Structure (Required) -```markdown -# [Title] - -## Overview -[2-3 sentence summary] - -## When to Use This -[Specific scenarios] - -## [Main Content Sections] -### [Subsection] -**Context**: [When this applies] -**Pattern**: [How to implement] -**Example**: -```yaml -[Code block with working example] -``` -**Pitfalls**: [Common mistakes to avoid] - -## Related Documentation -- [Cross-references to other docs] - -## Quick Reference -[Summary table or bullet points] -``` - -### Code Examples -- **Lead with code**: Show working examples first, explain after -- **Production-ready**: Use real-world patterns (not toy examples) -- **Complete**: Include error handling, logging, verification -- **Tested**: Patterns validated on actual RHEL/OpenShift systems - -## Official Source Attribution - -**All documentation in this knowledge base is derived from official Red Hat sources**. - -See [SOURCES.md](SOURCES.md) for complete source attribution table including: -- Official Red Hat Product Documentation URLs -- Red Hat Customer Portal knowledge base articles -- OpenShift official documentation -- Red Hat Lightspeed documentation -- Red Hat security advisories and bulletins - -**License**: Content derived from Red Hat documentation licensed under CC BY-SA 4.0 or similar. All credit to Red Hat, Inc. - -**Verification**: All sources verified active and current as of 2026-02-24. - -## AI Inference Optimization - -This knowledge base includes an AI-optimized indexing layer in `docs/.ai-index/`: - -### Semantic Index (`semantic-index.json`) -- Document metadata with semantic keywords -- Use case mappings for task-based discovery -- RHEL version applicability -- System type applicability (bare metal, VM, K8s, OpenShift) -- Token estimates for each document -- Related docs cross-references - -### Task-to-Docs Mapping (`task-to-docs-mapping.json`) -- Pre-computed doc sets for common remediation workflows -- Required vs optional doc indicators -- Workflow execution order -- Estimated token usage per workflow - -### Cross-Reference Graph (`cross-reference-graph.json`) -- Document relationship graph -- Complement relationships (docs that enhance each other) -- Prerequisite relationships (foundational docs) -- Specialization relationships (conditional docs) -- Confidence scores for relationships - -### Index Generation (`generate-index.py`) -- Auto-generates indexes from YAML frontmatter -- Validates doc structure -- Updates semantic keywords -- Rebuilds cross-reference graph - -## Performance Benefits - -**Token Savings**: -- Simple Package CVE: 21% reduction (~1,000 tokens saved) -- Kernel CVE on K8s: 30% reduction (~1,900 tokens saved) -- Batch Remediation: 31% reduction (~1,800 tokens saved) -- Risk Assessment: 34% reduction (~1,100 tokens saved) -- **Average**: 29% reduction across all task types - -**Response Time**: -- 85% reduction in navigation overhead -- 30-40% faster end-to-end response time -- Fewer Read tool calls (direct doc access) - -**Accuracy**: -- Zero missed related docs (cross-reference graph ensures completeness) -- Zero irrelevant doc reads (semantic matching prevents false positives) -- 85% improvement in doc discovery accuracy - -## Quick Reference Tables - -### RHEL Version Support Matrix -| RHEL Version | Package Manager | Systemd | SELinux | Python | Status | -|--------------|-----------------|---------|---------|--------|--------| -| RHEL 7 | yum | 219 | Enforcing | 2.7 | Supported | -| RHEL 8 | dnf (yum alias) | 239 | Enforcing | 3.6 | Supported | -| RHEL 9 | dnf (yum alias) | 252 | Enforcing | 3.9 | Current | - -### OpenShift Version Support Matrix -| OCP Version | Kubernetes | RHEL CoreOS | Status | -|-------------|------------|-------------|--------| -| 4.7 | 1.20 | 8.x | Legacy | -| 4.8 | 1.21 | 8.x | Supported | -| 4.10 | 1.23 | 8.x | Supported | -| 4.16 | 1.29 | 9.x | Current | - -### CVE Severity Mapping (Red Hat) -| CVSS Score | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -## Documentation Maintenance - -### Update Process -1. Update or add markdown documentation -2. Update YAML frontmatter with sources and metadata -3. Run `python docs/.ai-index/generate-index.py` to regenerate indexes -4. Verify source URLs in SOURCES.md are current -5. Update "Last Verified" dates - -### Source Verification Schedule -- **Monthly**: Verify all source URLs are active -- **Quarterly**: Check for updated Red Hat documentation versions -- **Per CVE**: Validate remediation patterns against latest RH advisories - -## Support - -For questions about this documentation: -- Review [SOURCES.md](SOURCES.md) for original Red Hat documentation -- Consult official Red Hat Customer Portal: https://access.redhat.com -- Check Red Hat Product Documentation: https://docs.redhat.com - -**Important**: This is a derivative work for operational use. For authoritative information, always consult official Red Hat documentation at the URLs listed in SOURCES.md. diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/SOURCES.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/SOURCES.md deleted file mode 100644 index 06478094..00000000 --- a/evaluation/without_skills/rh-sre__system-context/environment/docs/SOURCES.md +++ /dev/null @@ -1,107 +0,0 @@ -# Red Hat Documentation Sources - -This document provides attribution for all official Red Hat documentation sources used in the Remediation Agent knowledge base. - -## Source Attribution Table - -| Category | Document Title | Official Source URL | Sections Referenced | Last Verified | -|----------|---------------|---------------------|-------------------|---------------| -| **RHEL Package Management** | Managing Software with the DNF Tool (RHEL 9) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index) | DNF commands, updating packages, repository management | 2026-01-20 | -| **RHEL Package Management** | Software Management in RHEL 9 Adoption Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9) | RHEL 7/8/9 compatibility, migration considerations | 2026-01-20 | -| **RHEL Package Management** | RHEL 9 Release Notes | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/9.4_release_notes/index) | Version-specific package management features | 2026-01-20 | -| **Ansible CVE Remediation** | Red Hat Lightspeed Remediations Guide | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index) | Creating remediation plans, playbook generation | 2026-01-20 | -| **Ansible CVE Remediation** | Creating and Managing Remediation Plans | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide) | Playbook templates, execution patterns | 2026-01-20 | -| **Ansible CVE Remediation** | Automation Controller User Guide (AAP 2.4) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4/html/automation_controller_user_guide/controller-setting-up-insights) | Setting up Lightspeed for AAP remediations | 2026-01-20 | -| **Ansible CVE Remediation** | Creating Remediation Playbooks (RHEL 7 Security Guide) | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities) | Ansible playbook patterns for security compliance | 2026-01-20 | -| **OpenShift Pod Eviction** | Node Maintenance (OpenShift Virtualization 4.8-4.10) | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/virtualization/node-maintenance) | Node maintenance operator, draining nodes | 2026-01-20 | -| **OpenShift Pod Eviction** | Evicting Pods Using the Descheduler | [docs.openshift.com](https://docs.openshift.com/en/container-platform/4.8/nodes/scheduling/nodes-descheduler.html) | Pod eviction strategies, descheduler policies | 2026-01-20 | -| **OpenShift Pod Eviction** | How to Handle Evicted Pods in OpenShift | [access.redhat.com](https://access.redhat.com/solutions/3521071) | Troubleshooting evicted pods, remediation steps | 2026-01-20 | -| **OpenShift Pod Eviction** | OpenShift Container Platform 4.16 Nodes | [docs.redhat.com](https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/pdf/nodes/OpenShift_Container_Platform-4.16-Nodes-en-US.pdf) | Node management, pod disruption budgets | 2026-01-20 | -| **Lightspeed CVE Assessment** | Assessing Security Vulnerabilities on RHEL Systems | [docs.redhat.com](https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview) | CVE identification, classification, threat intelligence | 2026-01-20 | -| **Lightspeed CVE Assessment** | Generating Vulnerability Service Reports | [access.redhat.com](https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index) | Executive reports, CVE reports, data export | 2026-01-20 | -| **Lightspeed CVE Assessment** | Red Hat CVE Database | [access.redhat.com](https://access.redhat.com/security/security-updates/cve) | Official CVE entries, security updates | 2026-01-20 | -| **Lightspeed CVE Assessment** | A Complete View of System Vulnerabilities | [redhat.com/blog](https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights) | Vulnerability service overview, best practices | 2026-01-20 | -| **CVSS Scoring** | Severity Ratings | [access.redhat.com](https://access.redhat.com/security/updates/classification) | Four-point severity scale, CVSS v3.1 scoring | 2026-01-20 | -| **CVSS Scoring** | How We Classify Security Severity Levels | [access.redhat.com](https://access.redhat.com/solutions/725593) | CVSS metrics interpretation, severity guidelines | 2026-01-20 | -| **CVSS Scoring** | Security Update Policy | [access.redhat.com](https://access.redhat.com/security/lifecycle-security-update-policy) | Security lifecycle, update policies | 2026-01-20 | -| **CVSS Scoring** | Product Security Center | [access.redhat.com](https://access.redhat.com/security/) | Security advisories, bulletins, data feeds | 2026-01-20 | - -## Documentation Categories - -### RHEL (Red Hat Enterprise Linux) -- **Primary Source**: Red Hat Product Documentation (docs.redhat.com) -- **Focus**: Package management (DNF/YUM), systemd, SELinux, security hardening -- **Versions Covered**: RHEL 7, 8, 9 -- **Update Frequency**: Continuous (latest release notes include 2026 updates) - -### Ansible Automation Platform -- **Primary Source**: Red Hat Lightspeed Documentation + Ansible Automation Platform Documentation -- **Focus**: CVE remediation playbooks, automation patterns, AAP integration -- **Current Version**: Ansible Automation Platform 2.4 -- **Update Frequency**: Regular security advisories and feature updates - -### OpenShift Container Platform -- **Primary Source**: OpenShift Product Documentation (docs.redhat.com/openshift) -- **Focus**: Node maintenance, pod eviction, zero-downtime updates, security -- **Versions Covered**: OpenShift 4.7-4.16 -- **Update Frequency**: Per-release documentation updates - -### Red Hat Lightspeed -- **Primary Source**: Red Hat Lightspeed Documentation + Customer Portal -- **Focus**: Vulnerability assessment, CVE analysis, remediation planning, system inventory -- **Current Version**: 1-latest (continuously updated) -- **Update Frequency**: Real-time CVE database updates - -### Security & CVSS -- **Primary Source**: Red Hat Customer Portal - Product Security Center -- **Focus**: CVSS v3.1 scoring, severity classification, security advisories -- **Update Frequency**: Daily security bulletins and advisories - -## Attribution Format - -All documentation files in this knowledge base include YAML frontmatter with source attribution: - -```yaml ---- -title: [Document Title] -category: rhel|ansible|openshift|insights|references -sources: - - title: [Official Doc Title] - url: [Official URL] - sections: [Relevant sections] - date_accessed: YYYY-MM-DD -tags: [keywords] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -last_updated: YYYY-MM-DD ---- -``` - -## Verification - -All sources listed above were verified as active and current as of January 20, 2026. The sources are: - -1. **Official Red Hat Documentation** (docs.redhat.com) - Authoritative product documentation -2. **Red Hat Customer Portal** (access.redhat.com) - Knowledge base articles and security data -3. **Red Hat Corporate Website** (redhat.com) - Official blog posts and technical articles -4. **OpenShift Documentation** (docs.openshift.com) - OpenShift Container Platform guides - -## License and Usage - -This knowledge base is derived from official Red Hat documentation licensed under Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0) or similar Red Hat documentation licenses. All credit for the original content belongs to Red Hat, Inc. and its contributors. - -**Important**: This knowledge base is a derivative work for educational and operational purposes. For the most up-to-date and authoritative information, always consult the official Red Hat documentation at the URLs listed above. - -## Source Maintenance - -This source list is maintained as part of the Remediation Agent plugin. When documentation is updated or new sources are added: - -1. Update this SOURCES.md file with new entries -2. Update the YAML frontmatter in affected documentation files -3. Regenerate the semantic index using `docs/.ai-index/generate-index.py` -4. Update the "Last Verified" date in the table above - -## Contact - -For questions about Red Hat documentation sources or to report broken links: -- Red Hat Customer Portal: https://access.redhat.com/support -- Red Hat Documentation Feedback: https://docs.redhat.com (feedback links on each page) diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/README.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/README.md deleted file mode 100644 index 846ef682..00000000 --- a/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Ansible Documentation Overview -category: ansible -last_updated: 2026-01-20 ---- - -# Ansible Documentation Overview - -This directory contains Ansible playbook patterns and best practices for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[cve-remediation-templates.md](cve-remediation-templates.md)** ⭐ **HIGHEST VALUE** - - 6 production-ready playbook templates - - Package updates, kernel updates, service restarts - - SELinux, batch remediation patterns - - Error handling, rollback, audit logging - -### Future Enhancements (P1-P2 Priority) -- **error-handling.md** - Block/rescue/always patterns (planned) -- **idempotency.md** - Safe re-run patterns (planned) -- **playbook-patterns.md** - Reusable components (planned) -- **aap-integration.md** - Ansible Automation Platform workflows (planned) - -## When to Use These Docs - -**Use cve-remediation-templates.md when**: -- Generating CVE remediation playbooks -- Need production-ready patterns with error handling -- Working with Kubernetes/OpenShift systems (includes pod eviction integration) -- Handling kernel updates requiring reboots -- Implementing batch remediation across multiple systems - -## Template Selection Guide - -| CVE Type | Template | Complexity | -|----------|----------|------------| -| User-space package | Template 1: Package Update | Low | -| Service config | Template 2: Service Restart | Low | -| System config | Template 3: Config Update | Low | -| Kernel CVE | Template 4: Kernel Update | High | -| SELinux issue | Template 5: SELinux Context | Medium | -| Multiple CVEs | Template 6: Batch Remediation | High | - -## Quick Links - -- Red Hat Lightspeed Remediations: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index -- Ansible Automation Platform docs: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.4 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/aap-job-execution.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/aap-job-execution.md deleted file mode 100644 index 386fd114..00000000 --- a/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/aap-job-execution.md +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: AAP Job Execution Guide -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Job Templates - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-job-templates - date_accessed: 2026-02-24 -tags: [aap, job-execution, playbook, dry-run, check-mode] -semantic_keywords: [aap job execution, ansible check mode, dry-run remediation, job template requirements, aap url structure] -use_cases: [playbook-executor, remediation] -related_docs: [playbook-integration-aap.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# AAP Job Execution Guide - -## Overview - -This guide covers executing Ansible remediation playbooks through AAP (Ansible Automation Platform), including dry-run testing, job monitoring, and result interpretation. - -## Job Template Requirements for Remediation - -### Minimum Requirements - -For a job template to be suitable for CVE remediation, it must have: - -1. **Inventory**: Contains target systems identified in CVE analysis -2. **Project**: Contains or can receive remediation playbooks from Git -3. **Credentials**: - - Machine credential (SSH) for host access - - Privilege escalation enabled (sudo/become) -4. **Execution Environment**: Compatible with RHEL versions of target systems - -### Recommended Settings - -- **Prompt on Launch - Variables**: Allow passing CVE-specific parameters -- **Prompt on Launch - Limit**: Allow targeting specific hosts at runtime -- **Job Type**: Should support both "Run" and "Check" modes -- **Verbosity**: Set to at least "1 (Verbose)" for debugging -- **Timeout**: Set generous timeout (30+ minutes for large-scale remediations) -- **Enable Webhook**: Optional for CI/CD integration - -### Example Template Configuration - -```yaml -Name: CVE Remediation Template -Job Type: Run -Inventory: Production Servers -Project: Remediation Playbooks -Playbook: playbooks/remediation/remediation-template.yml -Credentials: - - SSH Credential (Machine) - - Privilege Escalation: Yes -Prompt on Launch: - - Variables: Yes - - Limit: Yes -Options: - - Enable Privilege Escalation: Yes - - Allow Simultaneous: No -``` - -## Dry-Run vs Production Execution - -### Dry-Run (Check Mode) - -**Purpose**: Simulate playbook execution without making actual changes. - -**Use When**: -- Testing new remediation playbooks -- Validating changes before production -- Identifying potential issues (permissions, package availability, dependencies) -- Understanding impact scope - -**How to Execute**: -```json -{ - "job_type": "check", - "extra_vars": {...} -} -``` - -**What It Does**: -- Gathers facts from target systems -- Evaluates conditionals and variables -- Simulates task execution -- Reports **would change** counts -- Does NOT apply any changes - -**Limitations**: -- Some modules don't support check mode (command, shell, raw) -- Services that would restart are not actually restarted -- Can't detect runtime failures that occur during actual execution -- Package dependencies may not be fully validated - -**Output Interpretation**: -``` -PLAY RECAP ************************************************************* -prod-web-01 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-02 : ok=8 changed=3 unreachable=0 failed=0 -prod-web-03 : ok=8 changed=3 unreachable=0 failed=0 - -"changed=3" means 3 tasks WOULD make changes -"failed=0" means no errors detected in check mode -``` - -### Production Execution (Run Mode) - -**Purpose**: Apply actual changes to systems. - -**Use When**: -- Dry-run passed successfully -- User has approved changes -- Maintenance window scheduled (if required) -- Backups completed - -**How to Execute**: -```json -{ - "job_type": "run", - "extra_vars": {...} -} -``` - -**What It Does**: -- Executes all playbook tasks -- Applies actual changes (package updates, config modifications, service restarts) -- Reports real results -- Can trigger system reboots if specified - -**Best Practices**: -1. Always run dry-run first -2. Review dry-run results carefully -3. Ensure maintenance window if downtime expected -4. Have rollback plan ready -5. Monitor execution in real-time -6. Verify success after completion - -## Job Type Parameter - -### job_type: "check" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "check" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml --check -``` - -**Behavior**: -- Runs in check mode (dry-run) -- No actual changes applied -- Reports what WOULD happen -- Useful for validation - -### job_type: "run" - -**API Parameter**: -```json -{ - "id": "10", - "requestBody": { - "job_type": "run" - } -} -``` - -**Equivalent Command Line**: -```bash -ansible-playbook playbook.yml -``` - -**Behavior**: -- Runs in execution mode -- Applies actual changes -- Reports what DID happen -- Production execution - -## Interpreting Job Results - -### Job Status Values - -| Status | Meaning | Action | -|--------|---------|--------| -| `pending` | Job queued, not yet started | Wait for execution | -| `waiting` | Waiting for resources/dependencies | Monitor for start | -| `running` | Currently executing | Monitor progress | -| `successful` | Completed without errors | Verify changes | -| `failed` | Completed with errors | Review error logs | -| `error` | Job could not execute | Check configuration | -| `canceled` | User cancelled job | N/A | - -### Per-Host Statistics - -**ok**: Number of tasks that executed successfully without changes -**changed**: Number of tasks that made actual changes -**failed**: Number of tasks that failed -**unreachable**: Number of hosts that couldn't be reached -**rescued**: Number of tasks that recovered from failures -**ignored**: Number of failed tasks that were ignored - -**Success Criteria**: -- `failed: 0` AND `unreachable: 0` = Success -- `changed > 0` = Remediation applied changes -- `ok > 0` = Some tasks ran successfully - -**Failure Indicators**: -- `failed > 0` = At least one task failed -- `unreachable > 0` = Host connectivity issues -- `ok: 0` AND `changed: 0` = Nothing executed successfully - -### Task Timeline Interpretation - -Example timeline: -``` -1. ✅ Gather Facts (2s) -2. ✅ Check disk space (1s) -3. ✅ Backup configuration (3s) -4. ✅ Update package httpd (45s) -5. ⚠️ Restart httpd service (FAILED on prod-web-03) -6. ✅ Verify service status (2s) -``` - -**Analysis**: -- Tasks 1-4: Successful across all hosts -- Task 5: Failed on one host (prod-web-03) -- Task 6: Likely skipped on failed host - -**Action**: -- Investigate why httpd restart failed on prod-web-03 -- Check logs for that specific host -- Verify httpd package was actually installed -- Relaunch job for failed host after fixing issue - -## AAP URL Structure - -### Job Details URL - -**Format**: -``` -https://{your-aap-instance}/#/jobs/playbook/{JOB_ID} -``` - -**Example**: -``` -https://aap.example.com/#/jobs/playbook/1235 -``` - -**What It Shows**: -- Real-time job status -- Live output stream -- Per-host statistics -- Task-level details -- Error messages -- Job parameters used - -### Template URL - -**Format**: -``` -https://{your-aap-instance}/#/templates/job_template/{TEMPLATE_ID}/details -``` - -**Example**: -``` -https://aap.example.com/#/templates/job_template/10/details -``` - -### Project URL - -**Format**: -``` -https://{your-aap-instance}/#/projects/{PROJECT_ID}/details -``` - -## Troubleshooting Common Execution Failures - -### Connection Failures - -**Symptoms**: -- `unreachable: 1` in host statistics -- "SSH timeout" errors -- "Connection refused" messages - -**Common Causes**: -1. SSH service not running on target -2. Firewall blocking port 22 -3. Network connectivity issues -4. Wrong SSH credentials - -**Troubleshooting Steps**: -```bash -# Test SSH connectivity -ssh -i /path/to/key user@target-host - -# Check SSH service -systemctl status sshd - -# Verify firewall rules -firewall-cmd --list-all - -# Test network connectivity -ping target-host -``` - -**Resolution**: -- Fix SSH service or network issues -- Update credentials in AAP -- Relaunch job after fixing - -### Permission Errors - -**Symptoms**: -- `failed: 1` with "Permission denied" errors -- "sudo: required but not available" messages -- "This command has to be run under the root user" errors - -**Common Causes**: -1. Privilege escalation not enabled -2. User doesn't have sudo rights -3. SELinux blocking operation -4. File permissions incorrect - -**Troubleshooting Steps**: -```bash -# Check sudo access -sudo -l - -# Test privilege escalation -sudo whoami - -# Check SELinux status -getenforce - -# Review SELinux denials -ausearch -m avc -ts recent -``` - -**Resolution**: -- Enable "Privilege Escalation" in job template -- Grant sudo rights to SSH user -- Adjust SELinux policies -- Fix file permissions - -### Package Manager Issues - -**Symptoms**: -- "No package X available" errors -- "Repository not found" messages -- "Dependency problems" errors -- Package installation timeouts - -**Common Causes**: -1. Repository not configured or unavailable -2. Package name incorrect -3. Network issues accessing repos -4. Insufficient disk space - -**Troubleshooting Steps**: -```bash -# Check repository configuration -dnf repolist - -# Test package availability -dnf info httpd - -# Check disk space -df -h - -# Verify repository URLs -dnf repolist -v -``` - -**Resolution**: -- Configure required repositories -- Verify package names -- Fix network issues -- Free up disk space - -### Service Restart Failures - -**Symptoms**: -- `failed: 1` on service restart tasks -- "Failed to restart X.service" errors -- "Unit not found" messages -- Service timeout errors - -**Common Causes**: -1. Service not installed -2. Configuration errors -3. Service dependencies not met -4. Systemd issues - -**Troubleshooting Steps**: -```bash -# Check if service exists -systemctl status httpd - -# Verify service file -systemctl cat httpd - -# Check service logs -journalctl -u httpd -n 50 - -# Test manual restart -systemctl restart httpd -``` - -**Resolution**: -- Ensure service is installed -- Fix configuration errors -- Start required dependencies first -- Review systemd logs - -### Disk Space Issues - -**Symptoms**: -- "No space left on device" errors -- Package installation failures -- Download failures - -**Common Causes**: -1. /var partition full -2. /tmp partition full -3. Log files consuming space - -**Troubleshooting Steps**: -```bash -# Check disk usage -df -h - -# Find large files -du -sh /var/* | sort -hr | head -10 - -# Check package cache size -du -sh /var/cache/dnf -``` - -**Resolution**: -- Clean package cache: `dnf clean all` -- Remove old logs: `journalctl --vacuum-time=7d` -- Remove unused packages: `dnf autoremove` - -## Job Monitoring Best Practices - -### Real-Time Monitoring - -1. **Watch AAP Web UI**: Real-time output and status -2. **Monitor Task Progress**: Track which tasks are running -3. **Check Per-Host Stats**: Identify failing hosts early -4. **Review Event Log**: See task-level events as they occur - -### Alert Configuration - -Configure notifications for: -- Job failures -- Long-running jobs (timeout warnings) -- Partial successes (some hosts failed) - -### Post-Execution Verification - -After job completes: -1. **Review per-host statistics**: Ensure all hosts succeeded -2. **Check full output**: Look for warnings or errors -3. **Verify actual changes**: Confirm packages updated, services restarted -4. **Run remediation-verifier**: Validate CVE status changed - -## Performance Optimization - -### Parallelism - -AAP can run tasks in parallel across multiple hosts. Configure: -- **Forks**: Number of parallel processes (default: 5) -- **Instance Groups**: Distribute jobs across multiple AAP nodes -- **Job Slicing**: Split large inventories into parallel jobs - -### Timeout Settings - -Set appropriate timeouts based on: -- Number of target systems -- Package size to download -- Network bandwidth -- System resources - -**Recommended Timeouts**: -- Small remediations (1-10 hosts): 10 minutes -- Medium remediations (10-50 hosts): 30 minutes -- Large remediations (50+ hosts): 60+ minutes - -## Security Considerations - -### Credential Management - -- Use AAP credential vault for secrets -- Rotate credentials regularly -- Limit credential scope to necessary hosts -- Never hardcode credentials in playbooks - -### Audit Logging - -AAP automatically logs: -- Who launched the job -- When it was launched -- What parameters were used -- Full execution output -- Final job status - -**Retention**: Configure appropriate log retention for compliance. - -### Change Control - -Integrate AAP jobs with change management: -- Require approval workflows for production -- Document job execution in change tickets -- Link jobs to CVE remediation tracking -- Maintain audit trail - -## Related Documentation - -- [Playbook Integration with AAP](./playbook-integration-aap.md) - How to add playbooks to AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package update best practices diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/cve-remediation-templates.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/cve-remediation-templates.md deleted file mode 100644 index d612b2f6..00000000 --- a/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/cve-remediation-templates.md +++ /dev/null @@ -1,1500 +0,0 @@ ---- -title: CVE Remediation Playbook Templates -category: ansible -sources: - - title: Red Hat Lightspeed Remediations Guide - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index - sections: Creating remediation plans, playbook generation - date_accessed: 2026-02-24 - - title: Creating and Managing Remediation Plans - url: https://docs.redhat.com/en/documentation/red_hat_lightspeed/1-latest/html-single/red_hat_lightspeed_remediations_guide/index#creating-remediation-plans_red-hat-lightspeed-remediation-guide - sections: Playbook templates, execution patterns - date_accessed: 2026-02-24 - - title: Creating Remediation Playbooks (RHEL 7 Security Guide) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - sections: Ansible playbook patterns for security compliance - date_accessed: 2026-02-24 -tags: [cve, remediation, playbooks, ansible, templates, package-update, kernel, service-restart, selinux, batch] -applies_to: [rhel7, rhel8, rhel9, openshift4.x] -semantic_keywords: - - "package update" - - "kernel update" - - "service restart" - - "configuration change" - - "SELinux context" - - "batch remediation" - - "reboot handling" - - "rollback strategy" - - "idempotent playbook" - - "CVE patch" -use_cases: - - "package_update_cve" - - "kernel_cve" - - "service_restart_cve" - - "config_file_cve" - - "selinux_cve" - - "batch_remediation" -related_docs: - - "rhel/package-management.md" - - "ansible/error-handling.md" - - "rhel/version-compatibility.md" -last_updated: 2026-02-24 ---- - -# CVE Remediation Playbook Templates - -This document provides 6 production-ready Ansible playbook templates for CVE remediation on RHEL systems. All templates are based on official Red Hat Lightspeed remediation patterns and best practices. - -## Overview - -Red Hat Lightspeed automatically generates Ansible playbooks for CVE remediation. These templates represent the core patterns used in production environments, enhanced with: - -- **Error handling**: Block/rescue/always patterns for safe execution -- **Rollback capability**: Automated rollback on failure -- **Audit logging**: Comprehensive remediation tracking -- **Reboot handling**: Safe reboot detection and execution -- **Idempotency**: Safe to re-run without side effects -- **Health checks**: Pre-flight and post-flight validation - -## Template Index - -1. **[Template 1: Package Update](#template-1-package-update-most-common)** - Standard package updates (most common) -2. **[Template 2: Service Restart](#template-2-service-restart)** - Service configuration CVEs -3. **[Template 3: Configuration File Update](#template-3-configuration-file-update)** - Config file modifications -4. **[Template 4: Kernel Update with Reboot](#template-4-kernel-update-with-reboot)** - Kernel CVEs requiring reboot -5. **[Template 5: SELinux Context Update](#template-5-selinux-context-update)** - SELinux-related CVEs -6. **[Template 6: Batch Remediation](#template-6-batch-remediation)** - Multiple CVEs across fleet - -## When to Use Each Template - -| CVE Type | Affected Component | Template | Reboot Required? | K8s Considerations | -|----------|-------------------|----------|------------------|-------------------| -| Package vulnerability | User-space package | Template 1 | Usually no | Service restart if pod | -| Service configuration | Service daemon | Template 2 | No | Pod restart recommended | -| Config file issue | System config | Template 3 | Usually no | ConfigMap update if K8s | -| Kernel vulnerability | Linux kernel | Template 4 | Yes | Node drain required | -| SELinux context | File contexts | Template 5 | No | Rare in containerized | -| Multiple CVEs | Various | Template 6 | Depends | Batch node updates | - ---- - -## Template 1: Package Update (Most Common) - -### Use Case -Standard package updates for CVEs affecting user-space packages (httpd, openssl, glibc, etc.). This is the most common CVE remediation pattern. - -### When to Use -- CVE affects an installed package -- Fix available via package update (DNF/YUM) -- No kernel or system-critical components involved -- Service restart sufficient (no reboot needed) - -### Key Features -- RHEL version validation (7/8/9 compatibility) -- Automatic backup creation (RHEL 8/9 snapshots) -- Package update with cache refresh -- Reboot detection -- Conditional service restarts -- Audit logging -- Idempotent (safe to re-run) - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Package Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - vulnerable_packages: - - package-name-1 - - package-name-2 - affected_services: [] # Optional: services to restart after update - - pre_tasks: - - name: Gather package facts - package_facts: - manager: auto - - - name: Verify system is RHEL - assert: - that: - - ansible_distribution == "RedHat" - - ansible_distribution_major_version in ["7", "8", "9"] - fail_msg: "This playbook is for Red Hat Enterprise Linux systems only" - success_msg: "RHEL {{ ansible_distribution_major_version }} detected" - - - name: Check current vulnerable package versions - debug: - msg: > - Package {{ item }} current version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Create backup point (RHEL 8/9 with Boom) - command: > - boom create --title "pre-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: - - ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: backup_result - - - name: Log backup creation - debug: - msg: "Backup created: {{ backup_result.stdout | default('skipped or failed') }}" - - tasks: - - name: Update vulnerable packages - block: - - name: Update packages using DNF (RHEL 8/9) - dnf: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update packages using YUM (RHEL 7) - yum: - name: "{{ vulnerable_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package update success - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed" - success_msg: "Packages updated successfully" - - rescue: - - name: Log package update failure - debug: - msg: "Package update failed: {{ package_update.msg | default('unknown error') }}" - - - name: Fail playbook on update error - fail: - msg: "CVE remediation failed - package update error" - - - name: Check if reboot is required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check if reboot is required (systemd method) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Gather updated package facts - package_facts: - manager: auto - - - name: Verify package versions after update - debug: - msg: > - Package {{ item }} updated version: - {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages - - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - needs_restarting.rc == 0 | default(true) - - - name: Wait for services to be active - systemd: - name: "{{ item }}" - state: started - loop: "{{ affected_services | default([]) }}" - when: - - affected_services is defined - - affected_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ package_update.results | length | default(0) }} packages updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Notify if reboot required - debug: - msg: | - ⚠️ REBOOT REQUIRED ⚠️ - System: {{ inventory_hostname }} - Reason: Package update requires reboot - Action: Schedule maintenance window for reboot - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) - - - name: Create reboot notification file - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Packages Updated: {{ vulnerable_packages | join(', ') }} - Reason: Package update requires system reboot - dest: "/root/REBOOT_REQUIRED_{{ cve_id }}.txt" - mode: '0644' - when: > - reboot_required_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) -``` - -### Usage Example - -```bash -# Create inventory file -cat > inventory.ini < remediate-cve-2024-1234.yml < - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ service_name }} service configuration updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Service: {{ service_name }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate sshd CVE requiring config changes -- name: CVE-2024-5678 Remediation - SSH Weak Ciphers - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-5678" - service_name: "sshd" - config_file: "/etc/ssh/sshd_config" - - tasks: - - name: Disable weak SSH ciphers - lineinfile: - path: "{{ config_file }}" - regexp: '^#?Ciphers.*' - line: 'Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr' - backup: true - - - name: Disable weak MACs - lineinfile: - path: "{{ config_file }}" - regexp: '^#?MACs.*' - line: 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com' - backup: true -``` - -### Pitfalls to Avoid - -❌ **Don't skip configuration validation**: Always test config syntax before restart -❌ **Don't forget backups**: Configuration rollback impossible without backups -❌ **Don't skip health checks**: Verify service functionality after restart -❌ **Don't use `state: reloaded` for critical changes**: Use `state: restarted` for security fixes - ---- - -## Template 3: Configuration File Update - -### Use Case -CVEs requiring modifications to system configuration files (not service-specific configs). - -### When to Use -- CVE affects system-level configuration (/etc/sysctl.conf, /etc/security/limits.conf, etc.) -- Fix involves file content changes -- No package updates or service restarts required -- Configuration takes effect via sysctl reload or next boot - -### Key Features -- Configuration file backup -- Atomic file updates -- Configuration validation -- Sysctl reload for kernel parameters -- Audit trail - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - System Configuration Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - config_file: "/etc/sysctl.conf" # Example: sysctl, limits, etc. - config_backup_dir: "/var/backups/cve-remediation" - sysctl_reload: true # Set to true if sysctl configuration - - pre_tasks: - - name: Create backup directory - file: - path: "{{ config_backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current configuration - copy: - src: "{{ config_file }}" - dest: "{{ config_backup_dir }}/{{ config_file | basename }}.{{ ansible_date_time.epoch }}.bak" - remote_src: true - mode: preserve - register: backup_config - - - name: Record current configuration checksum - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: original_config_stat - - tasks: - - name: Update configuration file - block: - # Example: Disable source routing (CVE mitigation) - - name: Set kernel parameter - Disable source routing - sysctl: - name: net.ipv4.conf.all.accept_source_route - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - - name: Set kernel parameter - Disable ICMP redirects - sysctl: - name: net.ipv4.conf.all.accept_redirects - value: '0' - state: present - reload: "{{ sysctl_reload }}" - sysctl_file: "{{ config_file }}" - when: config_file contains 'sysctl' - - # For non-sysctl configs, use lineinfile or blockinfile - - name: Update non-sysctl configuration - lineinfile: - path: "{{ config_file }}" - regexp: '^#?ParameterName.*' - line: 'ParameterName value' - backup: true - when: config_file not contains 'sysctl' - register: config_update - - - name: Verify configuration change - stat: - path: "{{ config_file }}" - checksum_algorithm: sha256 - register: new_config_stat - - - name: Assert configuration was modified - assert: - that: - - new_config_stat.stat.checksum != original_config_stat.stat.checksum - fail_msg: "Configuration file was not modified" - success_msg: "Configuration successfully updated" - - rescue: - - name: Log configuration failure - debug: - msg: "Configuration update failed - initiating rollback" - - - name: Restore configuration from backup - copy: - src: "{{ backup_config.dest }}" - dest: "{{ config_file }}" - remote_src: true - mode: preserve - - - name: Reload original sysctl configuration - command: sysctl -p {{ config_file }} - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Fail playbook after rollback - fail: - msg: "CVE remediation failed - configuration rolled back" - - post_tasks: - - name: Verify sysctl parameters (if applicable) - command: sysctl net.ipv4.conf.all.accept_source_route net.ipv4.conf.all.accept_redirects - register: sysctl_verify - when: sysctl_reload and (config_file contains 'sysctl') - changed_when: false - - - name: Display sysctl values - debug: - msg: "{{ sysctl_verify.stdout_lines }}" - when: sysctl_reload and (config_file contains 'sysctl') - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - {{ config_file }} updated on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document configuration changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - Configuration File: {{ config_file }} - Backup Location: {{ backup_config.dest }} - Changes: Kernel parameters hardened per CVE mitigation - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate kernel parameter CVE -- name: CVE-2024-9012 Remediation - Kernel Parameter Hardening - hosts: all_servers - become: true - - vars: - cve_id: "CVE-2024-9012" - config_file: "/etc/sysctl.conf" - sysctl_reload: true - - tasks: - - name: Disable IP forwarding - sysctl: - name: net.ipv4.ip_forward - value: '0' - state: present - reload: true - sysctl_file: /etc/sysctl.conf - - - name: Enable SYN cookies - sysctl: - name: net.ipv4.tcp_syncookies - value: '1' - state: present - reload: true - sysctl_file: /etc/sysctl.conf -``` - -### Pitfalls to Avoid - -❌ **Don't skip sysctl reload**: Changes won't take effect until reload or reboot -❌ **Don't modify production configs without backup**: Always create backups -❌ **Don't forget validation**: Verify sysctl values after reload -❌ **Don't use persistent=no for security fixes**: Security parameters should persist across reboots - ---- - -## Template 4: Kernel Update with Reboot - -### Use Case -Critical kernel CVEs requiring kernel package updates and system reboots. - -### When to Use -- CVE affects the Linux kernel -- Fix available via kernel package update -- System reboot mandatory for remediation -- High-impact operation requiring maintenance window - -### Key Features -- Kernel version validation -- Grub configuration backup -- Safe reboot with timeout -- Kubernetes node draining (if applicable) -- Post-reboot verification -- Health checks after reboot - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - Kernel Update with Reboot - hosts: affected_systems - become: true - gather_facts: true - serial: 1 # Update one host at a time for safety - - vars: - cve_id: "CVE-YYYY-NNNNN" - kernel_package: "kernel" - reboot_timeout: 600 # 10 minutes - post_reboot_delay: 30 # Wait 30 seconds after reboot - kubernetes_node: false # Set to true if K8s node - drain_timeout: 300 # 5 minutes for node drain - - pre_tasks: - - name: Record current kernel version - command: uname -r - register: current_kernel - changed_when: false - - - name: Display current kernel - debug: - msg: "Current kernel: {{ current_kernel.stdout }}" - - - name: Check if system is a Kubernetes node - stat: - path: /etc/kubernetes/kubelet.conf - register: k8s_check - - - name: Set kubernetes_node fact - set_fact: - kubernetes_node: "{{ k8s_check.stat.exists }}" - - - name: Drain Kubernetes node (if applicable) - command: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data --force --timeout={{ drain_timeout }}s - delegate_to: localhost - when: kubernetes_node - register: drain_result - - - name: Log node drain - debug: - msg: "Node drained: {{ drain_result.stdout }}" - when: kubernetes_node - - - name: Create pre-update snapshot (RHEL 8/9) - command: > - boom create --title "pre-kernel-{{ cve_id }}-{{ ansible_date_time.epoch }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - register: snapshot_result - - - name: Backup GRUB configuration - copy: - src: /etc/default/grub - dest: /var/backups/grub.{{ ansible_date_time.epoch }}.bak - remote_src: true - mode: preserve - - tasks: - - name: Update kernel package - block: - - name: Update kernel using DNF (RHEL 8/9) - dnf: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update kernel using YUM (RHEL 7) - yum: - name: "{{ kernel_package }}" - state: latest - update_cache: true - register: kernel_update - when: ansible_distribution_major_version == "7" - - - name: Verify kernel update - assert: - that: - - kernel_update is changed or kernel_update is success - fail_msg: "Kernel update failed" - success_msg: "Kernel updated successfully" - - rescue: - - name: Log kernel update failure - debug: - msg: "Kernel update failed: {{ kernel_update.msg | default('unknown error') }}" - - - name: Uncordon Kubernetes node on failure - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - ignore_errors: true - - - name: Fail playbook - fail: - msg: "CVE remediation failed - kernel update error" - - - name: Create reboot notification - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Current Kernel: {{ current_kernel.stdout }} - Action: System will reboot to apply kernel update - dest: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - mode: '0644' - - - name: Log pre-reboot state - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} - Pre-reboot - - Current kernel: {{ current_kernel.stdout }} - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Reboot system to apply kernel update - reboot: - reboot_timeout: "{{ reboot_timeout }}" - post_reboot_delay: "{{ post_reboot_delay }}" - msg: "Rebooting for CVE-{{ cve_id }} kernel update" - register: reboot_result - - post_tasks: - - name: Verify system is back online - wait_for_connection: - timeout: 300 - - - name: Gather facts after reboot - setup: - - - name: Record new kernel version - command: uname -r - register: new_kernel - changed_when: false - - - name: Verify kernel was updated - assert: - that: - - new_kernel.stdout != current_kernel.stdout - fail_msg: "Kernel was not updated - still running {{ current_kernel.stdout }}" - success_msg: "Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }}" - - - name: Verify all services are running - service_facts: - - - name: Check critical services - assert: - that: - - ansible_facts.services['sshd.service'].state == 'running' - fail_msg: "Critical services not running after reboot" - success_msg: "System health check passed" - - - name: Uncordon Kubernetes node - command: kubectl uncordon {{ inventory_hostname }} - delegate_to: localhost - when: kubernetes_node - register: uncordon_result - - - name: Wait for node to be ready - command: kubectl wait --for=condition=Ready node/{{ inventory_hostname }} --timeout=300s - delegate_to: localhost - when: kubernetes_node - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - Kernel updated from {{ current_kernel.stdout }} to {{ new_kernel.stdout }} on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document kernel update - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Previous Kernel: {{ current_kernel.stdout }} - New Kernel: {{ new_kernel.stdout }} - Reboot Duration: {{ reboot_result.elapsed }} seconds - Kubernetes Node: {{ kubernetes_node }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' - - - name: Clean up notification files - file: - path: "/root/REBOOTING_FOR_{{ cve_id }}.txt" - state: absent -``` - -### Usage Example - -```bash -# Kernel CVE remediation with Kubernetes integration -ansible-playbook -i k8s-nodes.ini remediate-kernel-cve.yml \ - --extra-vars "cve_id=CVE-2024-3456 kubernetes_node=true" \ - --limit production-worker-nodes \ - --ask-become-pass - -# Serial execution (one node at a time) is built into the playbook -# with 'serial: 1' directive -``` - -### Kubernetes/OpenShift Considerations - -**Before running this playbook on K8s nodes**: - -1. **Review pod disruption budgets**: Ensure critical apps can tolerate node drain -2. **Check StatefulSets**: May require special handling -3. **Verify cluster capacity**: Ensure remaining nodes can handle workload -4. **Schedule maintenance window**: Kernel updates are high-impact - -### Pitfalls to Avoid - -❌ **Don't reboot all nodes simultaneously**: Use `serial: 1` for safety -❌ **Don't forget post-reboot verification**: Ensure new kernel is running -❌ **Don't set short reboot timeouts**: Kernel updates can take 5-10 minutes - ---- - -## Template 5: SELinux Context Update - -### Use Case -CVEs affecting SELinux file contexts or requiring SELinux policy updates. - -### When to Use -- CVE involves SELinux context issues -- Fix requires `restorecon` or policy updates -- SELinux denials blocking remediation -- No package updates required (context-only fixes) - -### Key Features -- SELinux status validation -- Context backup -- Safe context restoration -- Policy module management -- AVC denial checking -- Rollback capability - -### Complete Playbook - -```yaml ---- -- name: CVE-YYYY-NNNNN Remediation - SELinux Context Update - hosts: affected_systems - become: true - gather_facts: true - - vars: - cve_id: "CVE-YYYY-NNNNN" - affected_paths: - - /path/to/affected/file1 - - /path/to/affected/dir/ - selinux_type: "httpd_sys_content_t" # Example context - backup_dir: "/var/backups/selinux-contexts" - - pre_tasks: - - name: Check SELinux status - command: getenforce - register: selinux_status - changed_when: false - failed_when: selinux_status.stdout not in ['Enforcing', 'Permissive'] - - - name: Verify SELinux is not disabled - assert: - that: - - selinux_status.stdout != 'Disabled' - fail_msg: "SELinux is disabled - this playbook requires SELinux to be enabled" - success_msg: "SELinux mode: {{ selinux_status.stdout }}" - - - name: Create backup directory - file: - path: "{{ backup_dir }}" - state: directory - mode: '0755' - - - name: Backup current SELinux contexts - shell: > - ls -Z {{ item }} > {{ backup_dir }}/{{ item | basename }}.{{ ansible_date_time.epoch }}.context - loop: "{{ affected_paths }}" - changed_when: false - - - name: Check for recent AVC denials - command: ausearch -m avc -ts recent - register: avc_denials - failed_when: false - changed_when: false - - - name: Log AVC denials if present - debug: - msg: "Recent AVC denials detected - review /var/log/audit/audit.log" - when: avc_denials.rc == 0 - - tasks: - - name: Apply SELinux context fixes - block: - - name: Restore default SELinux contexts - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - register: restorecon_result - - - name: Set specific SELinux type (if needed) - sefcontext: - target: '{{ item }}(/.*)?' - setype: "{{ selinux_type }}" - state: present - loop: "{{ affected_paths }}" - when: selinux_type is defined - register: sefcontext_result - - - name: Apply new context after semanage - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - when: sefcontext_result is changed - - - name: Verify contexts were applied - shell: ls -Z {{ item }} - loop: "{{ affected_paths }}" - register: context_verify - changed_when: false - - - name: Display new contexts - debug: - msg: "{{ context_verify.results | map(attribute='stdout_lines') | list }}" - - rescue: - - name: Log SELinux context failure - debug: - msg: "SELinux context update failed - review AVC denials" - - - name: Display backup location - debug: - msg: "Context backups available in {{ backup_dir }}" - - - name: Fail playbook - fail: - msg: "CVE remediation failed - SELinux context update error" - - post_tasks: - - name: Check for new AVC denials - command: ausearch -m avc -ts recent - register: new_avc_denials - failed_when: false - changed_when: false - - - name: Warn if new AVC denials - debug: - msg: "⚠️ New AVC denials detected - manual review required" - when: new_avc_denials.rc == 0 - - - name: Verify SELinux is still enforcing - command: getenforce - register: final_selinux_status - changed_when: false - - - name: Assert SELinux mode unchanged - assert: - that: - - final_selinux_status.stdout == selinux_status.stdout - fail_msg: "SELinux mode changed unexpectedly" - success_msg: "SELinux mode stable: {{ final_selinux_status.stdout }}" - - - name: Log remediation success - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - {{ cve_id }} remediated - - SELinux contexts updated for {{ affected_paths | length }} paths on {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Document SELinux changes - copy: - content: | - CVE: {{ cve_id }} - Date: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - Affected Paths: {{ affected_paths | join(', ') }} - SELinux Type Applied: {{ selinux_type | default('default') }} - Backup Location: {{ backup_dir }} - Status: Remediation successful - dest: "/root/CVE_REMEDIATION_{{ cve_id }}.txt" - mode: '0644' -``` - -### Usage Example - -```yaml -# Remediate httpd SELinux context CVE -- name: CVE-2024-7890 Remediation - httpd SELinux Context - hosts: web_servers - become: true - - vars: - cve_id: "CVE-2024-7890" - affected_paths: - - /var/www/html - - /var/www/cgi-bin - selinux_type: "httpd_sys_content_t" - - tasks: - - name: Restore default contexts for web content - command: restorecon -Rv {{ item }} - loop: "{{ affected_paths }}" - - - name: Allow httpd network connections (if needed) - seboolean: - name: httpd_can_network_connect - state: true - persistent: true -``` - -### Pitfalls to Avoid - -❌ **Don't disable SELinux as a "fix"**: This defeats security -❌ **Don't use `setenforce 0`**: Keep SELinux enforcing -❌ **Don't skip AVC denial review**: Understand why denials occurred -❌ **Don't forget `restorecon -R`**: Recursive is critical for directories -❌ **Don't use `-F` flag casually**: Forces relabeling, may have side effects - ---- - -## Template 6: Batch Remediation - -### Use Case -Remediating multiple CVEs across a fleet of systems efficiently. - -### When to Use -- Multiple CVEs need remediation -- Large number of affected systems -- Want to minimize maintenance windows -- Need comprehensive remediation reporting - -### Key Features -- Multi-CVE handling -- Consolidated package updates -- Batch service restarts -- Progress tracking -- Detailed reporting -- Failure isolation (continues on non-critical errors) - -### Complete Playbook - -```yaml ---- -- name: Batch CVE Remediation - Multiple CVEs - hosts: affected_systems - become: true - gather_facts: true - serial: "{{ batch_size | default(5) }}" # Configurable batch size - - vars: - batch_id: "batch-{{ ansible_date_time.epoch }}" - batch_size: 5 # Process 5 systems concurrently - - cves: - - cve_id: "CVE-2024-1111" - packages: ["httpd", "httpd-tools"] - services: ["httpd"] - priority: "critical" - - - cve_id: "CVE-2024-2222" - packages: ["openssl", "openssl-libs"] - services: [] - priority: "important" - - - cve_id: "CVE-2024-3333" - packages: ["glibc", "glibc-common"] - services: [] - priority: "important" - - all_packages: "{{ cves | map(attribute='packages') | flatten | unique | list }}" - all_services: "{{ cves | map(attribute='services') | flatten | unique | select | list }}" - - report_dir: "/var/log/cve-remediation/{{ batch_id }}" - - pre_tasks: - - name: Create report directory - file: - path: "{{ report_dir }}" - state: directory - mode: '0755' - - - name: Log batch remediation start - copy: - content: | - Batch ID: {{ batch_id }} - Start Time: {{ ansible_date_time.iso8601 }} - System: {{ inventory_hostname }} - CVEs to Remediate: {{ cves | map(attribute='cve_id') | join(', ') }} - Total Packages: {{ all_packages | length }} - Total Services: {{ all_services | length }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_start.txt" - mode: '0644' - - - name: Gather current package versions - package_facts: - manager: auto - - - name: Record pre-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_pre_versions.txt" - mode: '0644' - - - name: Create backup snapshot (RHEL 8/9) - command: > - boom create --title "batch-{{ batch_id }}" - when: ansible_distribution_major_version in ["8", "9"] - ignore_errors: true - - tasks: - - name: Batch update all vulnerable packages - block: - - name: Update all packages in one transaction (RHEL 8/9) - dnf: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - - - name: Update all packages in one transaction (RHEL 7) - yum: - name: "{{ all_packages }}" - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" - - - name: Verify package updates - package_facts: - manager: auto - - - name: Record post-update package versions - copy: - content: | - {% for pkg in all_packages %} - {% if pkg in ansible_facts.packages %} - {{ pkg }}: {{ ansible_facts.packages[pkg][0].version }} - {% else %} - {{ pkg }}: not installed - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_post_versions.txt" - mode: '0644' - - rescue: - - name: Log package update failures - copy: - content: | - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Status: FAILED - Error: {{ package_update.msg | default('Package update failed') }} - Failed Packages: {{ package_update.failures | default([]) | join(', ') }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - mode: '0644' - - - name: Continue despite package failures - debug: - msg: "Package updates failed on {{ inventory_hostname }} - see {{ report_dir }}/{{ inventory_hostname }}_FAILED.txt" - - - name: Check if reboot required - stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - post_tasks: - - name: Restart affected services (if no reboot needed) - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ all_services }}" - when: - - all_services | length > 0 - - not (reboot_required_file.stat.exists | default(false)) - - (needs_restarting.rc == 0) | default(true) - ignore_errors: true - register: service_restarts - - - name: Log service restart failures - copy: - content: | - {% for result in service_restarts.results | default([]) %} - {% if result.failed %} - Service: {{ result.item }} - Status: FAILED - Error: {{ result.msg | default('Unknown error') }} - {% endif %} - {% endfor %} - dest: "{{ report_dir }}/{{ inventory_hostname }}_service_failures.txt" - mode: '0644' - when: service_restarts.failed | default(false) - - - name: Generate CVE remediation summary - copy: - content: | - ======================================== - BATCH CVE REMEDIATION SUMMARY - ======================================== - Batch ID: {{ batch_id }} - System: {{ inventory_hostname }} - Completion Time: {{ ansible_date_time.iso8601 }} - - CVEs Remediated: - {% for cve in cves %} - - {{ cve.cve_id }} ({{ cve.priority }}) - Packages: {{ cve.packages | join(', ') }} - Services: {{ cve.services | join(', ') if cve.services else 'None' }} - {% endfor %} - - Package Updates: {{ all_packages | length }} packages - Service Restarts: {{ all_services | length }} services - - Reboot Required: {{ 'YES' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'NO' }} - - {% if reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false)) %} - ⚠️ REBOOT REQUIRED ⚠️ - Schedule maintenance window to reboot this system. - {% endif %} - - Status: {{ 'SUCCESS' if not (package_update.failed | default(false)) else 'PARTIAL' }} - - Reports Location: {{ report_dir }} - dest: "{{ report_dir }}/{{ inventory_hostname }}_SUMMARY.txt" - mode: '0644' - - - name: Log to central remediation log - lineinfile: - path: /var/log/cve-remediation.log - line: > - {{ ansible_date_time.iso8601 }} - Batch {{ batch_id }} - - Remediated {{ cves | length }} CVEs - - {{ all_packages | length }} packages updated - - {{ 'REBOOT REQUIRED' if (reboot_required_file.stat.exists or (needs_restarting.rc != 0 | default(false))) else 'No reboot' }} - - {{ inventory_hostname }} - create: true - mode: '0644' - - - name: Display remediation summary - debug: - msg: "{{ lookup('file', report_dir + '/' + inventory_hostname + '_SUMMARY.txt') }}" -``` - -### Usage Example - -```bash -# Remediate multiple CVEs across production fleet -ansible-playbook -i production-inventory.ini batch-remediation.yml \ - --extra-vars "batch_size=10" \ - --limit web_servers - -# Generate consolidated report -grep "Batch batch-" /var/log/cve-remediation.log | \ - awk '{print $1, $2, $NF}' | \ - sort > batch-remediation-summary.txt -``` - -### Reporting - -After execution, find detailed reports in `/var/log/cve-remediation//`: - -``` -/ -├── host1_start.txt # Remediation start details -├── host1_pre_versions.txt # Package versions before update -├── host1_post_versions.txt # Package versions after update -├── host1_SUMMARY.txt # Remediation summary -├── host1_FAILED.txt # Failures (if any) -└── host1_service_failures.txt # Service restart failures (if any) -``` - -### Pitfalls to Avoid - -❌ **Don't set batch_size too high**: Limits blast radius if failures occur -❌ **Don't skip pre/post version recording**: Critical for audit trail -❌ **Don't fail entire batch on single package error**: Use `ignore_errors` strategically -❌ **Don't batch kernel updates**: Kernel CVEs require serial execution (Template 4) -❌ **Don't forget consolidated reporting**: Management needs overall status - ---- - -## Cross-Reference Guide - -### For RHEL-Specific Operations -- **[package-management.md](../rhel/package-management.md)** - DNF/YUM workflows, reboot detection -- **[version-compatibility.md](../rhel/version-compatibility.md)** - RHEL 7/8/9 differences -- **[selinux-context.md](../rhel/selinux-context.md)** - SELinux remediation patterns -- **[systemd-services.md](../rhel/systemd-services.md)** - Service management patterns - -### For Error Handling & Rollback -- **[error-handling.md](./error-handling.md)** - Block/rescue/always patterns, rollback strategies -- **[idempotency.md](./idempotency.md)** - Safe re-run patterns - -### For Execution & Deployment -- **[aap-integration.md](./aap-integration.md)** - Ansible Automation Platform workflows - -### For Risk Assessment -- **[vulnerability-logic.md](../insights/vulnerability-logic.md)** - Red Hat Lightspeed risk methodology -- **[cvss-scoring.md](../references/cvss-scoring.md)** - CVSS interpretation - ---- - -## Official Red Hat Sources - -This document is derived from: - -1. **Red Hat Lightspeed Remediations Guide** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html-single/red_hat_insights_remediations_guide/index - -2. **Creating and Managing Remediation Plans** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/red_hat_insights_remediations_guide/creating-managing-playbooks_red-hat-insights-remediation-guide - -3. **Creating Remediation Playbooks (RHEL 7 Security Guide)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/security_guide/creating-a-remediation-ansible-playbook-to-align-the-system-with-baseline_scanning-the-system-for-configuration-compliance-and-vulnerabilities - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-02-24 - ---- - -## Quick Reference Table - -| Template | Use Case | Reboot? | K8s Impact | Priority | Complexity | -|----------|----------|---------|------------|----------|------------| -| 1. Package Update | User-space packages | Rare | Low | High | Low | -| 2. Service Restart | Service configs | No | Medium | Medium | Low | -| 3. Config Update | System configs | No | Low | Medium | Low | -| 4. Kernel Update | Kernel CVEs | Yes | High | Critical | High | -| 5. SELinux Context | SELinux issues | No | Low | Medium | Medium | -| 6. Batch Remediation | Multiple CVEs | Varies | Varies | High | High | - -## Template Selection Decision Tree - -``` -Is CVE affecting kernel? -├─ YES → Use Template 4 (Kernel Update) -└─ NO → Is CVE affecting multiple systems? - ├─ YES → Use Template 6 (Batch Remediation) - └─ NO → What component is affected? - ├─ Package → Use Template 1 (Package Update) - ├─ Service config → Use Template 2 (Service Restart) - ├─ System config → Use Template 3 (Config Update) - └─ SELinux context → Use Template 5 (SELinux) -``` - ---- - -**Document Version**: 1.0 -**Last Updated**: 2026-02-24 -**Maintained By**: Remediation Agent Knowledge Base -**Official Sources**: See SOURCES.md for complete attribution diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/playbook-integration-aap.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/playbook-integration-aap.md deleted file mode 100644 index 806841f5..00000000 --- a/evaluation/without_skills/rh-sre__system-context/environment/docs/ansible/playbook-integration-aap.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -title: Playbook Integration with AAP -category: ansible -sources: - - title: Red Hat Ansible Automation Platform Documentation - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6 - date_accessed: 2026-02-24 - - title: AAP Projects - url: https://docs.redhat.com/en/documentation/red_hat_ansible_automation_platform/2.6/html/using_automation_execution/controller-projects - date_accessed: 2026-02-24 -tags: [aap, playbooks, git-integration, project-sync, version-control] -semantic_keywords: [aap project integration, playbook git workflow, project sync process, playbook versioning] -use_cases: [playbook-executor, job-template-creator] -related_docs: [aap-job-execution.md, cve-remediation-templates.md] -last_updated: 2026-02-24 ---- - -# Playbook Integration with AAP - -## Overview - -This guide explains how to integrate generated Ansible remediation playbooks with AAP (Ansible Automation Platform) through Git-based projects. AAP requires playbooks to be stored in version control (Git) and synced to projects before execution. - -## Workflow Overview - -```mermaid -graph LR - A[Generate Playbook] --> B[Add to Git Repo] - B --> C[Commit & Push] - C --> D[Sync AAP Project] - D --> E[Playbook Available in AAP] - E --> F[Create/Use Job Template] - F --> G[Execute Playbook] -``` - -## Git Repository Structure - -### Recommended Directory Layout - -``` -ansible-remediation-playbooks/ -├── README.md -├── .gitignore -├── playbooks/ -│ ├── remediation/ -│ │ ├── remediation-CVE-2025-49794.yml -│ │ ├── remediation-CVE-2025-50123.yml -│ │ └── remediation-template.yml -│ ├── verification/ -│ │ └── verify-remediation.yml -│ └── rollback/ -│ └── rollback-template.yml -├── roles/ -│ ├── common/ -│ ├── package-update/ -│ └── service-restart/ -├── inventories/ -│ ├── production.ini -│ ├── staging.ini -│ └── development.ini -├── group_vars/ -│ └── all.yml -└── host_vars/ -``` - -**Key Directories**: -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (optional if using AAP inventories) -- `group_vars/` and `host_vars/` - Variable files - -### .gitignore Configuration - -```gitignore -# Ansible -*.retry -.vault_pass -*.swp -*~ - -# Logs -*.log - -# Credentials (NEVER commit) -**/credentials.* -**/secrets.* -**/.env - -# Temporary files -/tmp/ -.DS_Store -``` - -## Adding Playbooks to Git Repository - -### Method 1: Existing Repository - -If you already have a Git repository configured in AAP: - -#### Step 1: Clone Repository (if not already local) - -```bash -# Clone the repository -git clone https://github.com/your-org/ansible-remediation-playbooks.git -cd ansible-remediation-playbooks -``` - -#### Step 2: Add Generated Playbook - -```bash -# Create remediation directory if it doesn't exist -mkdir -p playbooks/remediation - -# Add the playbook (replace with actual content) -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' ---- -- name: Remediate CVE-2025-49794 - hosts: all - become: true - - tasks: - - name: Check disk space - # ... playbook content ... -EOF -``` - -#### Step 3: Commit Changes - -```bash -# Stage the new playbook -git add playbooks/remediation/remediation-CVE-2025-49794.yml - -# Create descriptive commit message -git commit -m "Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Requires: Brief service restart (~10s downtime) -" -``` - -#### Step 4: Push to Remote - -```bash -# Push to main branch (or your default branch) -git push origin main -``` - -#### Step 5: Sync AAP Project - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Find your project (e.g., "Remediation Playbooks") -3. Click the **Sync** button (🔄 icon) -4. Wait for status to change to "Successful" (green checkmark) -5. Verify playbook appears in project's playbook list - -**Via AAP API** (if available): -```bash -curl -X POST \ - "${AAP_MCP_SERVER}/api/controller/v2/projects/${PROJECT_ID}/update/" \ - -H "Authorization: Bearer ${AAP_API_TOKEN}" -``` - -### Method 2: New Repository - -If you need to create a new repository for remediation playbooks: - -#### Step 1: Initialize Local Repository - -```bash -# Create project directory -mkdir ansible-remediation-playbooks -cd ansible-remediation-playbooks - -# Initialize Git -git init -``` - -#### Step 2: Create Directory Structure - -```bash -# Create directory structure -mkdir -p playbooks/{remediation,verification,rollback} -mkdir -p roles -mkdir -p inventories -mkdir -p {group_vars,host_vars} -``` - -#### Step 3: Create README - -```bash -cat > README.md << 'EOF' -# Ansible Remediation Playbooks - -CVE remediation playbooks for Red Hat Enterprise Linux systems. - -## Directory Structure - -- `playbooks/remediation/` - CVE remediation playbooks -- `playbooks/verification/` - Post-remediation verification -- `playbooks/rollback/` - Rollback procedures -- `roles/` - Shared Ansible roles -- `inventories/` - Inventory files (if not using AAP inventories) - -## Naming Convention - -Remediation playbooks: `remediation-CVE-YYYY-NNNNN.yml` - -## Usage - -Playbooks are executed via AAP job templates. See internal documentation -for execution procedures. - -## Best Practices - -1. Always test in non-production first -2. Review playbook in dry-run (check) mode -3. Backup systems before remediation -4. Verify remediation success after execution -5. Document changes in commit messages -EOF -``` - -#### Step 4: Create .gitignore - -```bash -cat > .gitignore << 'EOF' -*.retry -.vault_pass -*.swp -*~ -*.log -**/credentials.* -**/secrets.* -**/.env -/tmp/ -.DS_Store -EOF -``` - -#### Step 5: Add First Playbook - -```bash -# Add your generated playbook -cat > playbooks/remediation/remediation-CVE-2025-49794.yml << 'EOF' -# [Your playbook content here] -EOF -``` - -#### Step 6: Initial Commit - -```bash -# Stage all files -git add . - -# Create initial commit -git commit -m "Initial commit: Add remediation playbooks structure - -- Directory structure for remediation, verification, rollback -- README with project documentation -- .gitignore for security -- First remediation playbook: CVE-2025-49794 -" -``` - -#### Step 7: Create Remote Repository - -**On GitHub**: -1. Go to https://github.com/new -2. Enter repository name: `ansible-remediation-playbooks` -3. Choose visibility (Private recommended for security) -4. **Do NOT** initialize with README (you already have one) -5. Click "Create repository" -6. Copy the repository URL - -**On GitLab**: -1. Go to "New Project" -2. Enter project name -3. Choose visibility -4. **Uncheck** "Initialize with README" -5. Create project -6. Copy the repository URL - -#### Step 8: Connect and Push - -```bash -# Add remote -git remote add origin - -# Rename branch to main (if needed) -git branch -M main - -# Push to remote -git push -u origin main -``` - -#### Step 9: Add Project to AAP - -**Via AAP Web UI**: -1. Navigate to **Automation Execution** → **Projects** -2. Click **Add** button -3. Fill in project form: - - **Name**: "Remediation Playbooks" - - **Organization**: Select your organization - - **Source Control Type**: Git - - **Source Control URL**: `` - - **Source Control Branch**: `main` - - **Source Control Credential**: (if private repo) -4. Click **Save** -5. AAP will automatically sync -6. Wait for status "Successful" - -## Project Sync Process - -### Understanding Project Sync - -**What Happens During Sync**: -1. AAP connects to Git repository -2. Fetches latest commits from specified branch -3. Downloads playbooks and related files -4. Updates project playbook list -5. Makes playbooks available for job templates - -**Sync Triggers**: -- Manual: Click Sync button in AAP Web UI -- Automatic: Configured update interval (optional) -- Webhook: Git push triggers AAP sync (optional) -- Pre-launch: Job template can auto-sync before execution - -### Sync Verification - -**Check Sync Status**: -```bash -# Via MCP tool -projects_list(search="Remediation") - -# Look for: -# - status: "successful" -# - scm_revision: Latest commit SHA -# - last_update_failed: false -``` - -**Verify Playbook Available**: -1. In AAP Web UI, go to Projects -2. Click on your project -3. View "Playbooks" tab -4. Confirm new playbook appears in list - -### Troubleshooting Sync Issues - -**Sync Failed - Authentication**: -``` -Error: Authentication failed -``` -**Cause**: Invalid or missing Git credentials -**Fix**: -- Update Source Control Credential in project settings -- Verify credential has read access to repository -- For private repos, ensure SSH key or token is valid - -**Sync Failed - Network**: -``` -Error: Failed to connect to repository -``` -**Cause**: Network connectivity issues or firewall -**Fix**: -- Verify repository URL is correct -- Check AAP server can reach Git server -- Review firewall rules - -**Sync Failed - Branch Not Found**: -``` -Error: Branch 'main' not found -``` -**Cause**: Specified branch doesn't exist -**Fix**: -- Verify branch name in project settings -- Check repository has commits on that branch -- Update branch name to match repository - -**Playbook Not Appearing**: -``` -Sync successful but playbook not in list -``` -**Cause**: Playbook not in correct path or format -**Fix**: -- Verify playbook is in repository root or subdirectory -- Check playbook has .yml or .yaml extension -- Ensure playbook is valid Ansible syntax -- Re-sync project after fixing - -## Playbook Versioning Strategy - -### Semantic Versioning for Playbooks - -**Approach 1: Git Tags** -```bash -# Tag specific playbook versions -git tag -a remediate-CVE-2025-49794-v1.0 -m "Initial version" -git push origin remediate-CVE-2025-49794-v1.0 - -# Update for new version -git tag -a remediate-CVE-2025-49794-v1.1 -m "Fixed service restart timeout" -git push origin remediate-CVE-2025-49794-v1.1 -``` - -**Approach 2: Filename Versioning** -``` -playbooks/remediation/ -├── remediation-CVE-2025-49794-v1.yml -├── remediation-CVE-2025-49794-v2.yml -└── remediation-CVE-2025-49794.yml # Latest (symlink or copy) -``` - -**Approach 3: Branch-Based** -```bash -# Create feature branch for new playbook -git checkout -b remediate-cve-2025-49794 - -# Develop and test -git add playbooks/remediation/remediation-CVE-2025-49794.yml -git commit -m "Add CVE-2025-49794 remediation" - -# Merge to main after testing -git checkout main -git merge remediate-cve-2025-49794 -git push origin main -``` - -### Recommended Versioning Approach - -**For Production**: -1. Use Git tags for major versions -2. Keep playbook filenames stable -3. Document changes in commit messages -4. Use branches for development/testing -5. Merge to main only after validation - -**Version Format**: -``` -CVE-{YEAR}-{NUMBER}-v{MAJOR}.{MINOR} - -Examples: -- CVE-2025-49794-v1.0 (Initial release) -- CVE-2025-49794-v1.1 (Bug fix) -- CVE-2025-49794-v2.0 (Major changes) -``` - -## Best Practices - -### Commit Message Guidelines - -**Format**: -``` -: - - - - -``` - -**Example**: -``` -feat: Add remediation playbook for CVE-2025-49794 - -- Target CVE: CVE-2025-49794 (Critical, CVSS 9.8) -- Affected package: httpd -- Remediation: Update to httpd-2.4.57-8.el9 -- Target systems: Production web servers -- Impact: Brief service restart (~10s downtime) -- Tested on: RHEL 9.3, 9.4 -- Validation: Passed dry-run on 50 staging systems - -Refs: TICKET-12345 -``` - -**Commit Types**: -- `feat:` - New playbook -- `fix:` - Bug fix in existing playbook -- `refactor:` - Code restructuring without behavior change -- `docs:` - Documentation updates -- `test:` - Test-related changes -- `chore:` - Maintenance tasks - -### Security Best Practices - -1. **Never Commit Credentials**: - - Use AAP credential vault - - Reference credentials via AAP, not in playbooks - - Add credential files to .gitignore - -2. **Sensitive Variables**: - ```yaml - # Bad - hardcoded password - - name: Connect to database - vars: - db_password: "MyPassword123" - - # Good - reference AAP credential - - name: Connect to database - vars: - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - ``` - -3. **Audit Trail**: - - Descriptive commit messages - - Link to change tickets - - Document testing performed - - Tag production versions - -### Code Review Process - -**Before Merging to Main**: -1. **Syntax Validation**: - ```bash - ansible-playbook --syntax-check playbook.yml - ``` - -2. **Linting**: - ```bash - ansible-lint playbook.yml - ``` - -3. **Dry-Run Testing**: - - Test on staging systems first - - Run in check mode - - Review output for errors - -4. **Peer Review**: - - Create pull request - - Have colleague review changes - - Address feedback - - Approve and merge - -## AAP Project Configuration - -### Project Settings - -**Optimal Configuration**: -```yaml -Name: Remediation Playbooks -Organization: Default -Source Control Type: Git -Source Control URL: https://github.com/org/ansible-remediation-playbooks.git -Source Control Branch: main -Source Control Credential: Git-ReadOnly-Credential - -Options: - Clean: Yes (remove local modifications) - Delete: Yes (delete before sync) - Track submodules: No (unless needed) - Update Revision on Launch: Yes (auto-sync before jobs) - -Update Cache Timeout: 0 (always fetch latest) -``` - -**Update on Launch**: -- **Enabled**: AAP syncs project before each job launch -- **Pros**: Always uses latest playbook version -- **Cons**: Slight delay before job starts -- **Recommendation**: Enable for dynamic environments - -### Multiple Projects Strategy - -**Option 1: Single Project for All Playbooks** -``` -Project: "Remediation Playbooks" -Contains: All remediation, verification, rollback playbooks -Pros: Simple management, single sync point -Cons: All teams share same repository -``` - -**Option 2: Separate Projects by Purpose** -``` -Project: "CVE Remediation" - - playbooks/remediation/ - -Project: "Verification Playbooks" - - playbooks/verification/ - -Project: "Rollback Procedures" - - playbooks/rollback/ - -Pros: Clear separation, different access controls -Cons: More complex, multiple syncs needed -``` - -**Option 3: Separate Projects by Team/Environment** -``` -Project: "Production Remediation" - - Branch: main - -Project: "Staging Remediation" - - Branch: staging - -Project: "Development Remediation" - - Branch: develop - -Pros: Environment isolation, safe testing -Cons: Need to promote across branches -``` - -## Automation and CI/CD Integration - -### Automated Testing Pipeline - -**Example GitHub Actions**: -```yaml -name: Playbook Validation - -on: - push: - branches: [ main, develop ] - paths: - - 'playbooks/**/*.yml' - pull_request: - branches: [ main ] - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install Ansible - run: | - pip install ansible ansible-lint - - - name: Syntax Check - run: | - ansible-playbook --syntax-check playbooks/**/*.yml - - - name: Ansible Lint - run: | - ansible-lint playbooks/ - - - name: Check for Secrets - run: | - git secrets --scan -``` - -### Webhook Integration - -**Trigger AAP Sync on Git Push**: - -1. **Configure Webhook in Git**: - - URL: `https://aap.example.com/api/controller/v2/projects/{id}/github/` - - Events: Push events - - Secret: Generate in AAP - -2. **Enable Webhook in AAP**: - - Project settings → Enable Webhook - - Copy webhook URL and secret - - Add to Git repository settings - -**Result**: Git push automatically triggers AAP project sync. - -## Related Documentation - -- [AAP Job Execution Guide](./aap-job-execution.md) - Executing playbooks in AAP -- [CVE Remediation Templates](./cve-remediation-templates.md) - Playbook patterns -- [Package Management](../rhel/package-management.md) - RHEL package updates diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/insights/README.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/insights/README.md deleted file mode 100644 index 661065fd..00000000 --- a/evaluation/without_skills/rh-sre__system-context/environment/docs/insights/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Red Hat Lightspeed Documentation Overview -category: insights -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Documentation Overview - -This directory contains Red Hat Lightspeed-specific guidance for CVE vulnerability assessment and remediation planning. - -## Available Documentation - -### Current Documentation -- **[vulnerability-logic.md](vulnerability-logic.md)** - How Red Hat Lightspeed assesses CVE risk - - CVE identification and classification (Vulnerable vs Affected) - - Security Rules designation - - Red Hat severity ratings (Critical/Important/Moderate/Low) - - Risk assessment and prioritization methodology - - Priority decision matrix - - Integration with remediation workflows - -### Future Enhancements (P2 Priority) -- **remediation-workflow.md** - End-to-end remediation process (planned) -- **system-inventory.md** - Inventory management patterns (planned) - -## When to Use These Docs - -**Use vulnerability-logic.md when**: -- Performing CVE impact analysis -- Need to understand Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk assessments to management - -## Quick Links - -- Red Hat Lightspeed: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest -- CVE Assessment Guide: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/insights/vulnerability-logic.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/insights/vulnerability-logic.md deleted file mode 100644 index 0edc1ebd..00000000 --- a/evaluation/without_skills/rh-sre__system-context/environment/docs/insights/vulnerability-logic.md +++ /dev/null @@ -1,568 +0,0 @@ ---- -title: Red Hat Lightspeed Vulnerability Assessment Logic -category: insights -sources: - - title: Assessing and Monitoring Security Vulnerabilities on RHEL Systems - url: https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - sections: CVE identification, classification, threat intelligence - date_accessed: 2026-01-20 - - title: Generating Vulnerability Service Reports - url: https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - sections: Executive reports, CVE reports, data export - date_accessed: 2026-01-20 - - title: Red Hat CVE Database - url: https://access.redhat.com/security/security-updates/cve - sections: Official CVE entries, security updates - date_accessed: 2026-01-20 - - title: A Complete View of System Vulnerabilities - url: https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - sections: Vulnerability service overview, best practices - date_accessed: 2026-01-20 -tags: [insights, vulnerability, cve, risk-assessment, threat-intelligence] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "Red Hat Lightspeed" - - "CVE assessment" - - "vulnerability classification" - - "threat intelligence" - - "security rules" - - "affected but not vulnerable" - - "vulnerable status" - - "CVSS score" - - "severity rating" - - "remediation priority" -use_cases: - - "risk_assessment" - - "cve_impact_analysis" - - "remediation_prioritization" - - "vulnerability_reporting" -related_docs: - - "references/cvss-scoring.md" - - "ansible/cve-remediation-templates.md" - - "references/compliance-frameworks.md" -last_updated: 2026-01-20 ---- - -# Red Hat Lightspeed Vulnerability Assessment Logic - -This document explains how Red Hat Lightspeed assesses CVE vulnerabilities, classifies risk, and prioritizes remediation for RHEL systems. - -## Overview - -The Red Hat Lightspeed vulnerability service provides automated CVE assessment for RHEL systems registered with Lightspeed. It analyzes CVE impact, determines exploitability, and generates remediation recommendations. - -**Key Capabilities**: -- Automated CVE identification on registered systems -- Threat intelligence integration -- Remediation playbook generation -- Executive and technical reporting -- Compliance tracking - -## CVE Classification - -### Vulnerable vs Affected But Not Vulnerable - -Red Hat Lightspeed categorizes CVEs into two primary classifications: - -#### Vulnerable -**Definition**: System is running flawed code with an **open path to exploitation**. - -**Characteristics**: -- CVE exists in installed package -- No mitigating factors present -- Exploit path is viable -- System is exposed to risk - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with CVE-2024-1234 affecting versions < 2.4.37-2.el8 -- Package is vulnerable -- No compensating controls -- **Status**: Vulnerable ✗ - -**Action Required**: Immediate remediation recommended - -#### Affected But Not Vulnerable -**Definition**: System has the vulnerable code, but **exploitation is not currently possible**. - -**Characteristics**: -- CVE exists in installed package -- Mitigating factors prevent exploitation -- Configuration blocks exploit path -- Feature not enabled/used - -**Mitigating Factors**: -- SELinux blocking exploit path -- Firewall preventing network access -- Service not enabled/running -- Feature compiled out or disabled -- Red Hat backported fix without version number change - -**Example**: RHEL 8 system running `httpd-2.4.37-1.el8` with network-based CVE -- Package contains vulnerable code -- Firewall blocks external HTTP access -- **Status**: Affected but not vulnerable ⚠ - -**Action Required**: Lower priority, monitor for configuration changes - -### Security Rules - -**Security Rules** are CVEs that receive **elevated visibility** due to heightened risk. - -**Criteria for Security Rule Status**: -- Significant media coverage -- Active exploitation in the wild -- High CVSS score (typically ≥ 8.0) -- Affects critical infrastructure -- Red Hat Product Security team escalation - -**Example**: Log4Shell (CVE-2021-44228) -- CVSS 10.0 -- Widespread exploitation -- Affects many RHEL systems -- **Designated as Security Rule** - -**Impact**: -- Appears in executive dashboards -- Priority remediation recommended -- Enhanced reporting and tracking -- Compliance audit visibility - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** in addition to CVSS scores: - -| Severity | CVSS Range | Description | Response Time | -|----------|-----------|-------------|---------------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, no user interaction | 24-48 hours | -| **Important** | 7.0-8.9 | Exploitable with moderate effort, significant impact | 7 days | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited impact | 30 days | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | - -**Red Hat Severity ≠ CVSS Score** - -Red Hat severity considers: -- **Exploitability**: How easily can it be exploited? -- **Impact**: What damage can it cause? -- **Affected configurations**: How common is the vulnerable configuration? -- **Compensating controls**: Are mitigations available? -- **Customer environment**: How are customers actually using the software? - -**Example**: -- **CVSS Score**: 8.5 (Important) -- **Red Hat Severity**: Moderate -- **Reason**: Requires non-default configuration rarely used in production - -## Vulnerability Assessment Workflow - -### Step 1: System Registration - -```bash -# Register system with Red Hat Lightspeed -insights-client --register - -# Verify registration -insights-client --status -``` - -**What Lightspeed Collects**: -- Installed packages and versions -- Running services -- System configuration -- SELinux status -- Firewall rules (high-level) -- Subscription data - -**Privacy**: Lightspeed collects system metadata only, not application data or files. - -### Step 2: CVE Identification - -Lightspeed automatically: -1. Scans installed packages against Red Hat CVE database -2. Compares package versions to affected ranges -3. Identifies all applicable CVEs -4. Updates daily as new CVEs published - -**CVE Sources**: -- Red Hat Product Security team -- NVD (National Vulnerability Database) -- Red Hat engineering analysis -- Customer-reported vulnerabilities - -### Step 3: Exploitability Analysis - -For each identified CVE, Lightspeed determines exploitability: - -**Factors Analyzed**: -- SELinux policies (can SELinux block the exploit?) -- Service status (is the vulnerable service running?) -- Network exposure (is the service accessible remotely?) -- Feature enablement (is the vulnerable feature enabled?) -- Configuration (does config prevent exploitation?) - -**Example Analysis**: -``` -CVE-2024-1234: httpd remote code execution -Package: httpd-2.4.37-1.el8 (vulnerable version) - -Exploitability Check: -✓ Service running: YES (httpd.service active) -✓ Network accessible: YES (port 80/443 open) -✓ SELinux blocking: NO (httpd_can_network_connect enabled) -✓ Feature enabled: YES (mod_cgi loaded) - -Conclusion: VULNERABLE -``` - -### Step 4: Threat Intelligence Integration - -Lightspeed integrates external threat intelligence: - -**Data Sources**: -- Known exploits in the wild -- CISA Known Exploited Vulnerabilities (KEV) catalog -- Security researcher disclosures -- Red Hat threat intelligence feeds - -**Impact on Priority**: -- **Active exploitation** → Escalate to Critical -- **Proof-of-concept available** → Increase priority -- **Theoretical only** → Standard priority - -### Step 5: Remediation Recommendation - -Lightspeed generates remediation recommendations: - -**Automated Playbook Available**: -``` -CVE-2024-1234 Remediation -───────────────────────── -Status: Automated remediation available -Package: httpd-2.4.37-1.el8 → httpd-2.4.37-2.el8 -Reboot Required: No -Estimated Time: 5 minutes -Ansible Playbook: ✓ Available - -Actions: -1. Download playbook from Lightspeed console -2. Review and customize for your environment -3. Execute via Ansible Automation Platform -4. Verify remediation success -``` - -**Manual Remediation Required**: -``` -CVE-2024-5678 Remediation -───────────────────────── -Status: Manual remediation required -Package: custom-app-1.0 (not in Red Hat repos) -Guidance: Contact vendor for patch - -Actions: -1. Review vendor security advisory -2. Test vendor-provided patch in staging -3. Schedule maintenance window -4. Apply patch manually -5. Re-scan with Lightspeed to verify -``` - -## Remediation Prioritization - -### Priority Decision Matrix - -Lightspeed prioritizes CVEs based on multiple factors: - -| Factor | Weight | Description | -|--------|--------|-------------| -| **Severity** | 40% | Red Hat severity rating (Critical/Important/Moderate/Low) | -| **Exploitability** | 30% | Vulnerable vs Affected but not vulnerable | -| **Business Impact** | 20% | System criticality (production vs dev/test) | -| **Active Threats** | 10% | Known exploitation in the wild | - -**Example Calculation**: -``` -CVE-2024-1234: -- Severity: Important (7.0) → Score: 7.0 × 0.40 = 2.8 -- Exploitability: Vulnerable → Score: 10 × 0.30 = 3.0 -- Business Impact: Production → Score: 10 × 0.20 = 2.0 -- Active Threats: PoC available → Score: 7.0 × 0.10 = 0.7 - -Total Priority Score: 8.5 / 10 → P1 (High Priority) -``` - -### Recommended Response Times - -| Priority | Response Time | Typical Scenarios | -|----------|---------------|-------------------| -| **P0** | 24 hours | Critical + Vulnerable + Production + Active exploitation | -| **P1** | 7 days | Important + Vulnerable + Production | -| **P2** | 30 days | Moderate + Vulnerable or Important + Affected but not vulnerable | -| **P3** | 90 days | Low severity or non-production systems | - -## Reporting Capabilities - -### Executive Reports (PDF) - -**Purpose**: High-level summary for leadership -**Contents**: -- Total CVE count by severity -- Trend analysis (improving/worsening) -- Top 10 critical CVEs -- Remediation progress metrics -- Compliance posture - -**Generated via Lightspeed Console or API** - -### CVE Reports (PDF) - -**Purpose**: Detailed technical analysis -**Contents**: -- Filtered CVE list (by severity, date, system) -- Per-CVE details (CVSS, description, affected systems) -- Remediation guidance -- Timeline for resolution - -### Vulnerability Data Export (CSV/JSON) - -**Purpose**: Integration with other tools (SIEM, ticketing) -**Contents**: -- Complete CVE dataset -- System-to-CVE mapping -- Remediation status -- Custom fields - -**Example Export**: -```csv -CVE,Severity,CVSS,Affected_Systems,Remediation_Available,Status -CVE-2024-1234,Important,7.5,15,Yes,Vulnerable -CVE-2024-5678,Moderate,5.3,8,No,Affected_but_not_vulnerable -``` - -## Integration with Remediation Agent - -When using the remediation-agent plugin: - -### Step 1: Risk Assessment (Use This Doc) - -```yaml -# CVE Impact Skill uses vulnerability-logic.md -- Retrieve CVE data from Lightspeed -- Understand Red Hat severity rating -- Check if "vulnerable" or "affected but not vulnerable" -- Assess business impact -- Determine priority (P0/P1/P2/P3) -``` - -### Step 2: Remediation (Use Templates) - -```yaml -# Remediator Agent uses cve-remediation-templates.md -- If priority = P0/P1: Generate playbook immediately -- If priority = P2: Schedule for maintenance window -- If priority = P3: Add to backlog - -# Use appropriate template based on CVE type -- Package CVE → Template 1 (Package Update) -- Kernel CVE → Template 4 (Kernel Update) -- Service CVE → Template 2 (Service Restart) -``` - -## API Integration - -### Lightspeed API Endpoints - -**Get CVE Information**: -```bash -# Via lightspeed-mcp tool -vulnerability_get_cve_info(cve_id="CVE-2024-1234") - -# Returns: -{ - "cve_id": "CVE-2024-1234", - "severity": "Important", - "cvss_score": 7.5, - "publish_date": "2024-01-15", - "description": "...", - "affected_packages": ["httpd-2.4.37"], - "remediation_available": true -} -``` - -**List Affected Systems**: -```bash -vulnerability_list_cves(cve_id="CVE-2024-1234") - -# Returns: -{ - "systems": [ - { - "uuid": "abc-123", - "hostname": "web-server-01", - "status": "vulnerable", - "package": "httpd-2.4.37-1.el8" - } - ] -} -``` - -## Best Practices - -### 1. Regular Scanning - -- **Daily scans**: Automated via `insights-client` systemd timer -- **On-demand scans**: After patching to verify remediation -- **Post-change scans**: After system configuration changes - -```bash -# Force immediate scan -insights-client --force-reregister -``` - -### 2. Understand Context - -Don't remediate solely based on CVSS: -- ✅ Check Red Hat severity rating -- ✅ Verify "vulnerable" vs "affected but not vulnerable" -- ✅ Consider system criticality -- ✅ Review threat intelligence -- ❌ Don't auto-patch based on CVSS alone - -### 3. Prioritize Production - -``` -Production Vulnerable > Production Affected > Non-Prod Vulnerable -``` - -### 4. Track Remediation Progress - -- Use Lightspeed dashboards -- Export metrics for management -- Set SLAs per priority level -- Audit remediation completion - -### 5. Compliance Integration - -Map CVE remediation to compliance frameworks: -- **PCI-DSS**: High/Critical CVEs must be remediated per timeline -- **SOC 2**: Vulnerability management process documented -- **NIST**: CVE scoring aligns with NIST guidelines - -See: [compliance-frameworks.md](../references/compliance-frameworks.md) - -## Common Scenarios - -### Scenario 1: Security Rule Announced - -``` -Notification: CVE-2024-9999 designated as Security Rule -Severity: Critical (CVSS 9.8) -Affected Systems: 127 systems - -Workflow: -1. Read vulnerability-logic.md to understand Security Rule criteria -2. Assess: 127 systems × Critical severity = P0 priority -3. Generate batch remediation playbook (Template 6) -4. Execute rolling updates (serial: 5 for safety) -5. Verify all systems remediated within 24 hours -``` - -### Scenario 2: Affected But Not Vulnerable - -``` -CVE: CVE-2024-1111 -Status: Affected but not vulnerable -Reason: SELinux prevents exploitation - -Workflow: -1. Read vulnerability-logic.md to understand classification -2. Document why not vulnerable (SELinux policy blocks exploit) -3. Monitor for configuration changes (if SELinux disabled → becomes vulnerable) -4. Remediate during next maintenance window (not urgent) -5. Update documentation for audit purposes -``` - -### Scenario 3: No Automated Remediation - -``` -CVE: CVE-2024-2222 -Package: third-party-app-1.0 -Remediation: Manual steps required - -Workflow: -1. Check Red Hat CVE database for guidance -2. Contact vendor for security advisory -3. Create custom remediation playbook -4. Test in staging environment -5. Document manual steps for audit trail -``` - -## Troubleshooting - -### Issue: System Not Showing CVEs - -**Possible Causes**: -- System not registered with Lightspeed -- `insights-client` service not running -- Subscription not active -- Network connectivity issues - -**Resolution**: -```bash -# Check registration -insights-client --status - -# Re-register if needed -insights-client --register - -# Force update -insights-client --force-reregister -``` - -### Issue: CVE Shows "Affected But Not Vulnerable" When It Shouldn't - -**Possible Causes**: -- Lightspeed detected mitigating factor (SELinux, firewall) -- Configuration analysis outdated -- False positive - -**Resolution**: -1. Review system configuration -2. Verify mitigating factors actually present -3. If Lightspeed wrong: Contact Red Hat support -4. Override classification if necessary (document reason) - -### Issue: Remediation Playbook Failed - -**Possible Causes**: -- Package dependency conflicts -- Repository not accessible -- System requires reboot (not detected) - -**Resolution**: -1. Review playbook execution logs -2. Check `package_update.results` for errors -3. Consult [package-management.md](../rhel/package-management.md) for troubleshooting -4. Re-run with increased verbosity: `ansible-playbook -vvv` - -## Related Documentation - -- **[CVSS Scoring Reference](../references/cvss-scoring.md)** - Detailed CVSS interpretation -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates for remediation -- **[Compliance Frameworks](../references/compliance-frameworks.md)** - Compliance requirements per CVE severity - -## Official Red Hat Sources - -This document is derived from: - -1. **Assessing and Monitoring Security Vulnerabilities on RHEL Systems** - https://docs.redhat.com/en/documentation/red_hat_insights/1-latest/html/assessing_and_monitoring_security_vulnerabilities_on_rhel_systems/vuln-cves_vuln-overview - -2. **Generating Vulnerability Service Reports** - https://access.redhat.com/documentation/en-us/red_hat_insights/1-latest/html-single/generating_vulnerability_service_reports/index - -3. **Red Hat CVE Database** - https://access.redhat.com/security/security-updates/cve - -4. **A Complete View of System Vulnerabilities Using Red Hat Lightspeed** - https://www.redhat.com/en/blog/complete-view-system-vulnerabilities-using-red-hat-insights - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/references/README.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/references/README.md deleted file mode 100644 index 4c6481e1..00000000 --- a/evaluation/without_skills/rh-sre__system-context/environment/docs/references/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Reference Documentation Overview -category: references -last_updated: 2026-01-20 ---- - -# Reference Documentation Overview - -This directory contains reference materials supporting CVE remediation decisions. - -## Available Documentation - -### Current Documentation -- **[cvss-scoring.md](cvss-scoring.md)** - CVSS v3.1 interpretation and Red Hat severity mappings - - All 8 CVSS v3.1 metrics explained (AV, AC, PR, UI, S, C, I, A) - - Red Hat 4-point severity scale (Critical/Important/Moderate/Low) - - Why Red Hat severity ≠ CVSS score - - Priority decision matrix - - Real-world CVE examples (Log4Shell, Spectre, Sudo) - - Compliance framework mappings (PCI-DSS, SOC 2, NIST 800-53) - -### Future Enhancements (P2 Priority) -- **compliance-frameworks.md** - Detailed PCI-DSS, SOC 2, NIST requirements (planned) -- **glossary.md** - Red Hat terminology reference (planned) - -## When to Use These Docs - -**Use cvss-scoring.md when**: -- Interpreting CVSS scores and vector strings -- Understanding Red Hat severity ratings -- Prioritizing CVEs for remediation -- Explaining risk to stakeholders -- Mapping to compliance requirements (PCI-DSS, SOC 2, NIST) - -## Quick Links - -- Red Hat Security: https://access.redhat.com/security/ -- CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/references/cvss-scoring.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/references/cvss-scoring.md deleted file mode 100644 index e0ccd3dc..00000000 --- a/evaluation/without_skills/rh-sre__system-context/environment/docs/references/cvss-scoring.md +++ /dev/null @@ -1,636 +0,0 @@ ---- -title: CVSS Scoring and Red Hat Severity Mappings -category: references -sources: - - title: Severity Ratings (Red Hat Customer Portal) - url: https://access.redhat.com/security/updates/classification - sections: Red Hat severity ratings, CVSS usage - date_accessed: 2026-01-20 - - title: How We Classify Security Severity Levels - url: https://access.redhat.com/solutions/725593 - sections: Severity classification methodology - date_accessed: 2026-01-20 - - title: Security Update Policy - url: https://access.redhat.com/security/lifecycle-security-update-policy - sections: Security lifecycle, update policies - date_accessed: 2026-01-20 - - title: Product Security Center - url: https://access.redhat.com/security/ - sections: Security advisories, bulletins, CVSS data - date_accessed: 2026-01-20 -tags: [cvss, severity, scoring, risk-assessment, priority] -applies_to: [rhel6, rhel7, rhel8, rhel9] -semantic_keywords: - - "CVSS score" - - "severity rating" - - "Red Hat severity" - - "vulnerability scoring" - - "risk assessment" - - "priority matrix" - - "CVSS v3.1" - - "attack vector" - - "exploitability" - - "impact metrics" -use_cases: - - "risk_assessment" - - "cve_prioritization" - - "compliance_reporting" - - "stakeholder_communication" -related_docs: - - "insights/vulnerability-logic.md" - - "references/compliance-frameworks.md" - - "ansible/cve-remediation-templates.md" -last_updated: 2026-01-20 ---- - -# CVSS Scoring and Red Hat Severity Mappings - -This document explains CVSS (Common Vulnerability Scoring System) scoring and how Red Hat maps CVSS scores to severity ratings for prioritization. - -## Overview - -**CVSS (Common Vulnerability Scoring System)** is an industry-standard framework for assessing vulnerability severity. Red Hat uses **CVSS v3.1** for all vulnerabilities published since 2016. - -**Key Concepts**: -- **CVSS Base Score**: 0.0 (no risk) to 10.0 (critical) - measures inherent vulnerability characteristics -- **CVSS Temporal Score**: Adjusts base score for current exploit availability -- **CVSS Environmental Score**: Organization-specific adjustments -- **Red Hat Severity**: Critical/Important/Moderate/Low - Red Hat's assessment for their products - -**Important**: Red Hat severity ratings may differ from CVSS scores based on real-world exploitability in RHEL environments. - -## CVSS v3.1 Metrics - -CVSS base scores are calculated from 8 metrics across 3 categories: - -### Exploitability Metrics (How Easy to Exploit) - -#### 1. Attack Vector (AV) -**Question**: How is the vulnerability exploited? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Network (N)** | Highest | Remotely exploitable over network | Web service vulnerability | -| **Adjacent (A)** | High | Exploitable from adjacent network | WiFi attack, same subnet | -| **Local (L)** | Medium | Requires local access to system | Privilege escalation | -| **Physical (P)** | Lowest | Requires physical access | Console-only exploit | - -**Impact on Priority**: Network vulnerabilities are highest priority (remotely exploitable). - -#### 2. Attack Complexity (AC) -**Question**: How difficult is the attack to execute? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Low (L)** | Higher | Easy to exploit reliably | Standard buffer overflow | -| **High (H)** | Lower | Requires specific conditions | Race condition, timing-dependent | - -**Impact on Priority**: Low complexity = easier exploitation = higher priority. - -#### 3. Privileges Required (PR) -**Question**: What privileges must attacker have? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Highest | Unauthenticated attack | Public web service exploit | -| **Low (L)** | Medium | Basic user privileges needed | Authenticated user exploit | -| **High (H)** | Lowest | Admin/root privileges needed | Root-only privilege escalation | - -**Impact on Priority**: None required = widest attack surface = highest priority. - -#### 4. User Interaction (UI) -**Question**: Does exploitation require user action? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **None (N)** | Higher | Fully automated | Wormable exploit | -| **Required (R)** | Lower | User must perform action | Phishing, clicking malicious link | - -**Impact on Priority**: No interaction = self-propagating = highest priority. - -### Scope Metric - -#### 5. Scope (S) -**Question**: Can the exploit impact resources beyond the vulnerable component? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **Changed (C)** | Higher | Impacts other components | Container escape, VM escape | -| **Unchanged (U)** | Lower | Impacts only vulnerable component | Single service compromise | - -**Impact on Priority**: Changed scope = broader impact = higher priority. - -### Impact Metrics (What Damage) - -#### 6. Confidentiality (C) -**Question**: How much data can be disclosed? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total information disclosure | Database dump, /etc/shadow leak | -| **Low (L)** | 0.22 | Limited disclosure | Single file read | -| **None (N)** | 0.00 | No confidentiality impact | Denial of service only | - -#### 7. Integrity (I) -**Question**: How much can data/systems be modified? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total data modification | Root access, arbitrary code execution | -| **Low (L)** | 0.22 | Limited modification | Single file write | -| **None (N)** | 0.00 | No integrity impact | Read-only exploit | - -#### 8. Availability (A) -**Question**: How much is service availability impacted? - -| Value | Score | Description | Example | -|-------|-------|-------------|---------| -| **High (H)** | 0.56 | Total service disruption | System crash, complete DoS | -| **Low (L)** | 0.22 | Reduced performance | Resource exhaustion, slowdown | -| **None (N)** | 0.00 | No availability impact | Information disclosure only | - -## CVSS Score Calculation - -### Base Score Formula - -The CVSS v3.1 base score is calculated using a complex formula: - -``` -Impact = 1 - [(1 - C) × (1 - I) × (1 - A)] - -If Scope Unchanged: - BaseScore = Roundup(Minimum[(Impact + Exploitability), 10]) - -If Scope Changed: - BaseScore = Roundup(Minimum[1.08 × (Impact + Exploitability), 10]) - -Where: -Exploitability = 8.22 × AV × AC × PR × UI -``` - -**You don't need to calculate manually** - use CVSS calculators: -- FIRST CVSS Calculator: https://www.first.org/cvss/calculator/3.1 -- Red Hat CVE pages include calculated scores - -### CVSS Severity Ratings (FIRST Standard) - -| CVSS Score | Severity | Description | -|------------|----------|-------------| -| 9.0 - 10.0 | **Critical** | Easily exploitable, severe impact | -| 7.0 - 8.9 | **High** | Exploitable with moderate effort, significant impact | -| 4.0 - 6.9 | **Medium** | Requires specific conditions, moderate impact | -| 0.1 - 3.9 | **Low** | Difficult to exploit, limited impact | -| 0.0 | **None** | No impact | - -## Red Hat Severity Ratings - -Red Hat uses a **4-point severity scale** that may differ from CVSS: - -### Red Hat Severity Levels - -| Severity | Typical CVSS Range | Description | Response Time | Example | -|----------|-------------------|-------------|---------------|---------| -| **Critical** | 9.0-10.0 | Easily exploitable, widespread impact, remote code execution | 24-48 hours | Heartbleed, Shellshock | -| **Important** | 7.0-8.9 | Exploitable, significant impact, requires moderate effort | 7 days | Privilege escalation, data leak | -| **Moderate** | 4.0-6.9 | Requires specific conditions, limited scope | 30 days | Local DoS, limited info disclosure | -| **Low** | 0.1-3.9 | Difficult to exploit, minimal impact | 90 days | Minor information leaks | - -### Why Red Hat Severity ≠ CVSS Score - -Red Hat Product Security considers additional factors: - -**Exploitability in Real World**: -- Is the vulnerable code path actually used in typical RHEL deployments? -- Are default RHEL configurations vulnerable? -- Do RHEL security features (SELinux, firewall) mitigate the vulnerability? - -**Example 1: CVSS Higher Than Red Hat Severity** -``` -CVE: CVE-2024-XXXX -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate - -Reason: -- Vulnerability requires non-default configuration -- Feature rarely enabled in production RHEL systems -- SELinux blocks exploitation in enforcing mode -- Red Hat downgrades to Moderate based on real-world risk -``` - -**Example 2: Red Hat Severity Higher Than CVSS** -``` -CVE: CVE-2024-YYYY -CVSS Score: 6.5 (Medium) -Red Hat Severity: Important - -Reason: -- CVSS calculated for generic software -- In RHEL context, vulnerability is more exploitable -- Commonly used RHEL service affected -- Red Hat elevates to Important based on customer environment -``` - -## Priority Decision Matrix - -Combine Red Hat severity with other factors to determine priority: - -### Priority Calculation - -| Red Hat Severity | Vulnerable Status | Production System | Known Exploit | Priority | Response | -|------------------|------------------|-------------------|---------------|----------|----------| -| Critical | Vulnerable | Yes | Yes | **P0** | 24 hours | -| Critical | Vulnerable | Yes | No | **P0** | 48 hours | -| Critical | Vulnerable | No | - | **P1** | 7 days | -| Important | Vulnerable | Yes | Yes | **P1** | 7 days | -| Important | Vulnerable | Yes | No | **P1** | 14 days | -| Important | Vulnerable | No | - | **P2** | 30 days | -| Moderate | Vulnerable | Yes | - | **P2** | 30 days | -| Moderate | Vulnerable | No | - | **P3** | 90 days | -| Low | Vulnerable | - | - | **P3** | 90 days | -| Any | Affected but not vulnerable | - | - | **P3** | Next maintenance window | - -### Decision Tree - -``` -1. What is Red Hat Severity? - ├─ Critical → Continue to 2 - ├─ Important → Continue to 2 - ├─ Moderate → P2 (unless production + exploit = P2) - └─ Low → P3 - -2. Is system Vulnerable or Affected but not vulnerable? - ├─ Vulnerable → Continue to 3 - └─ Affected but not vulnerable → P3 - -3. Is this a production system? - ├─ Yes → Continue to 4 - └─ No → P1 (Critical) or P2 (Important) - -4. Is exploit known/available? - ├─ Yes → P0 - └─ No → P0 (Critical) or P1 (Important) -``` - -## CVSS Vector String - -CVSS scores include a **vector string** encoding all metrics: - -**Example Vector String**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Decoded**: -- **AV:N** - Attack Vector: Network -- **AC:L** - Attack Complexity: Low -- **PR:N** - Privileges Required: None -- **UI:N** - User Interaction: None -- **S:C** - Scope: Changed -- **C:H** - Confidentiality Impact: High -- **I:H** - Integrity Impact: High -- **A:H** - Availability Impact: High - -**Score**: 10.0 (Critical) - -**Interpretation**: Remotely exploitable, no authentication, no user interaction, impacts all CIA aspects - **highest severity**. - -## Common CVE Patterns - -### Pattern 1: Critical Remote Code Execution - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact) - -**CVSS Score**: 9.0-10.0 -**Red Hat Severity**: Critical -**Example**: Heartbleed (CVE-2014-0160) - -**Priority**: P0 - Immediate remediation - -### Pattern 2: Privilege Escalation - -**Typical Metrics**: -- AV:L (Local) -- AC:L (Low complexity) -- PR:L (Low privileges) -- UI:N (No interaction) -- C:H/I:H/A:H (Full impact after escalation) - -**CVSS Score**: 7.8 -**Red Hat Severity**: Important -**Example**: Dirty COW (CVE-2016-5195) - -**Priority**: P1 - 7 day remediation window - -### Pattern 3: Information Disclosure - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:H/I:N/A:N (Confidentiality only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Important or Moderate (depends on data sensitivity) - -**Priority**: P1-P2 depending on data criticality - -### Pattern 4: Denial of Service - -**Typical Metrics**: -- AV:N (Network) -- AC:L (Low complexity) -- PR:N (No privileges) -- UI:N (No interaction) -- C:N/I:N/A:H (Availability only) - -**CVSS Score**: 7.5 -**Red Hat Severity**: Moderate (unless critical service) - -**Priority**: P2 - 30 days (unless high-availability requirement) - -## Compliance Framework Mappings - -### PCI-DSS Requirements - -**PCI-DSS v4.0 Requirement 6.3.3**: Patch critical and high vulnerabilities within defined timeframes - -| CVSS Score | PCI-DSS Timeline | Remediation Deadline | -|------------|------------------|----------------------| -| 9.0-10.0 (Critical) | Within 30 days | **30 days max** | -| 7.0-8.9 (High) | Within 30 days | **30 days max** | -| 4.0-6.9 (Medium) | Risk-based approach | Organization-defined | -| 0.1-3.9 (Low) | Risk-based approach | Organization-defined | - -**Compliance Note**: PCI-DSS uses CVSS scores, not Red Hat severity ratings. - -### SOC 2 Trust Service Criteria - -**CC7.1**: Detect and respond to security incidents - -- **All CVSS 7.0+**: Must be tracked, remediated, documented -- **Remediation process**: Must demonstrate timely response -- **Audit trail**: Document priority decisions and remediation timeline - -### NIST 800-53 - -**SI-2**: Flaw Remediation - -- **High-impact systems**: Remediate high/critical within 30 days -- **Moderate-impact**: Remediate within 60 days -- **Low-impact**: Remediate within 90 days - -Map CVSS to NIST impact: -- CVSS 9.0-10.0 → High impact -- CVSS 7.0-8.9 → Moderate impact -- CVSS < 7.0 → Low impact - -See: [compliance-frameworks.md](compliance-frameworks.md) for detailed mappings - -## Real-World Examples - -### Example 1: Log4Shell (CVE-2021-44228) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Network (remotely exploitable) -- Attack Complexity: Low (trivial to exploit) -- Privileges Required: None (unauthenticated) -- User Interaction: None (fully automated) -- Scope: Changed (RCE impacts entire system) -- Impact: H/H/H (full compromise) - -**Score**: 10.0 -**Red Hat Severity**: Critical -**Response**: P0 - Immediate remediation (24 hours) - -**Why P0**: -- Remotely exploitable -- No authentication -- Widespread usage (logging library) -- Active exploitation in wild -- Full system compromise possible - -### Example 2: Spectre Variant 1 (CVE-2017-5753) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:C/C:H/I:N/A:N -``` - -**Metrics**: -- Attack Vector: Local (requires local access) -- Attack Complexity: High (timing attack, difficult) -- Privileges Required: Low (unprivileged user) -- Scope: Changed (can read across privilege boundaries) -- Impact: C:H (information disclosure), I:N, A:N - -**Score**: 5.6 (Medium by CVSS) -**Red Hat Severity**: Important (elevated due to CPU-level impact) -**Response**: P1 - 7 day remediation window - -**Why Red Hat Elevated**: -- Affects all modern CPUs -- Hardware-level vulnerability -- Difficult to fully mitigate -- Widespread impact across cloud/virtualization - -### Example 3: Sudo Heap Overflow (CVE-2021-3156) - -**CVSS v3.1 Vector**: -``` -CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H -``` - -**Metrics**: -- Attack Vector: Local -- Attack Complexity: Low (reliable exploit available) -- Privileges Required: Low (any local user) -- Scope: Unchanged -- Impact: H/H/H (root access) - -**Score**: 7.8 (High) -**Red Hat Severity**: Important -**Response**: P1 - 7 days - -**Why Important**: -- Local privilege escalation to root -- sudo installed by default on all RHEL systems -- Any local user can exploit -- Reliable exploits available - -## Integration with Remediation Agent - -### CVE Impact Skill Workflow - -```yaml -# Step 1: Retrieve CVE data (use vulnerability-logic.md) -- Get CVSS score from Red Hat Lightspeed -- Get Red Hat severity rating -- Check vulnerable vs affected status - -# Step 2: Interpret CVSS (use THIS document) -- Understand CVSS metrics (AV, AC, PR, UI, S, C, I, A) -- Map CVSS to Red Hat severity -- Check for discrepancies (explain to user) - -# Step 3: Calculate priority (use THIS document) -- Apply priority decision matrix -- Consider: severity + vulnerable status + production + exploit -- Output: P0/P1/P2/P3 with reasoning - -# Step 4: Recommend action -- P0: Immediate remediation (use cve-remediation-templates.md) -- P1: Schedule within 7 days -- P2: Next maintenance window -- P3: Backlog -``` - -### Remediator Agent Integration - -```yaml -# Input from CVE Impact Skill: -priority: P0 -cvss_score: 9.8 -red_hat_severity: Critical -reasoning: "Network RCE, no auth required, production systems affected" - -# Remediator Agent Decision: -if priority == "P0": - - Generate playbook immediately (Template 1 or 4) - - Recommend emergency change process - - Execute with minimal delay - -if priority == "P1": - - Generate playbook - - Schedule maintenance window - - Allow time for testing in staging - -if priority == "P2" or "P3": - - Add to remediation backlog - - Batch with other low-priority CVEs - - Include in next quarterly patching cycle -``` - -## Best Practices - -### 1. Trust Red Hat Severity Over Raw CVSS - -Red Hat Product Security analyzes vulnerabilities in the context of RHEL systems. Their severity ratings account for: -- Default RHEL configurations -- SELinux policies -- Firewall defaults -- Real-world usage patterns - -✅ **Use Red Hat severity for prioritization decisions** -⚠️ CVSS is a helpful reference but not the sole factor - -### 2. Document Prioritization Decisions - -Create an audit trail: -```markdown -CVE-2024-XXXX Prioritization Decision -Date: 2024-01-15 -CVSS Score: 8.5 (High) -Red Hat Severity: Moderate -Assigned Priority: P2 - -Rationale: -- CVSS high due to network vector -- Red Hat downgraded to Moderate (non-default config required) -- No active exploitation detected -- Non-production systems only affected -- Decision: P2 (30-day remediation window) - -Approved by: [Security Team Lead] -``` - -### 3. Re-evaluate on New Information - -Priorities can change: -- **Exploit published**: P2 → P1 -- **Active exploitation**: Any → P0 -- **Configuration change**: Affected but not vulnerable → Vulnerable - -Set up alerts for CVE status changes. - -### 4. Communicate in Business Terms - -Translate CVSS for stakeholders: - -**For Technical Teams**: -``` -CVE-2024-XXXX: CVSS 9.8 (Critical) -Attack Vector: Network, No Auth Required -Impact: Remote Code Execution -Priority: P0 - Patch within 24 hours -``` - -**For Management**: -``` -CVE-2024-XXXX: Critical Severity -Risk: Attackers can remotely compromise our web servers -Business Impact: Customer data exposure, service outage -Action Required: Emergency patching tonight -Estimated Downtime: 30 minutes per server -``` - -## Quick Reference - -### CVSS to Red Hat Severity (Typical Mapping) - -| CVSS Range | Red Hat Severity | Priority | Response Time | -|------------|------------------|----------|---------------| -| 9.0-10.0 | Critical | P0 | 24-48 hours | -| 7.0-8.9 | Important | P1 | 7 days | -| 4.0-6.9 | Moderate | P2 | 30 days | -| 0.1-3.9 | Low | P3 | 90 days | - -**Note**: Actual Red Hat severity may differ - always check CVE page. - -### Priority Override Conditions - -Escalate priority if: -- **Active exploitation** detected (any severity → P0) -- **Production critical system** affected (+1 priority level) -- **Compliance deadline** approaching (adjust to meet deadline) -- **Wormable vulnerability** (self-propagating → P0) - -Downgrade priority if: -- **Affected but not vulnerable** (-1 priority level) -- **Non-production only** (-1 priority level) -- **Compensating controls** in place (-1 priority level) - -## Related Documentation - -- **[Red Hat Lightspeed Vulnerability Logic](../insights/vulnerability-logic.md)** - How Lightspeed assesses CVEs -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbooks for remediation -- **[Compliance Frameworks](compliance-frameworks.md)** - PCI-DSS, SOC 2, NIST requirements - -## Official Red Hat Sources - -This document is derived from: - -1. **Severity Ratings** - https://access.redhat.com/security/updates/classification - -2. **How We Classify Security Severity Levels** - https://access.redhat.com/solutions/725593 - -3. **Security Update Policy** - https://access.redhat.com/security/lifecycle-security-update-policy - -4. **Product Security Center** - https://access.redhat.com/security/ - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-parameters.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-parameters.md deleted file mode 100644 index b1465f4f..00000000 --- a/evaluation/without_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-parameters.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Red Hat Lightspeed MCP - Parameter Reference -category: references -sources: - - title: Red Hat Lightspeed MCP - url: https://github.com/redhat/lightspeed-mcp - date_accessed: 2026-02-26 -tags: [lightspeed, mcp, parameters, inventory] -last_updated: 2026-02-26 ---- - -# Lightspeed MCP Parameter Reference - -Correct parameter names and types for Red Hat Lightspeed MCP tools. **Using wrong parameters causes validation errors.** - -## inventory__list_hosts - -**Purpose**: List hosts with filtering and sorting options. - -**CRITICAL**: Use `per_page` (integer), NOT `page_size`. The Lightspeed inventory API uses different parameter names than AAP MCP. - -| Parameter | Type | Required | Example | Notes | -|-----------|------|----------|---------|-------| -| `per_page` | **integer** | No | `10` | Use 10 on first call to avoid performance issues. NOT `page_size`. | -| `display_name` | string | No | `""` | Filter by display name. Pass empty string if no filter. | -| `page` | integer | No | `1` | Pagination page number. | - -**Correct**: -``` -inventory__list_hosts(per_page=10, display_name="") -``` - -**Wrong** (causes "Unexpected keyword argument" error): -``` -inventory__list_hosts(page_size=100) # ❌ Use per_page, not page_size -inventory__list_hosts(per_page="100") # ❌ Use integer, not string -``` - -## AAP MCP vs Lightspeed MCP - -| Server | Pagination Parameter | Type | -|--------|---------------------|------| -| lightspeed-mcp (inventory) | `per_page` | integer | -| aap-mcp-job-management | `page_size` | integer | -| aap-mcp-inventory-management | `page_size` | integer | - -Do not mix parameter names between servers. - -## vulnerability__get_cves - -**Purpose**: List CVEs affecting the account with filtering. - -**⚠️ Known issue**: Some MCP clients serialize `limit` as `limit_`, causing "Unexpected keyword argument" errors. For connectivity tests, call with no parameters. For CVE queries, if you see this error, omit `limit` (default 10) or use other parameters only. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `impact` | string | `"7,6"` | Comma-separated impact IDs: 7=Critical, 6=High, 5=Important, 4=Moderate | -| `sort` | string | `"-cvss_score"` | Use `-` prefix for descending | -| `limit` | integer | `20` | Max records per page. **Note**: Some clients bug: pass as `limit`; if error, omit (default 10) | -| `advisory_available` | string | `"true"` | Filter remediatable CVEs: `"true"` = only with available advisory, `"true,false"` = all | - -**For remediatable CVEs** (user asks "which CVEs can I remediate?"): -``` -vulnerability__get_cves(impact="7,6", sort="-cvss_score", limit=20, advisory_available="true") -``` - -**For remediatable CVEs on a specific system**: `get_system_cves` does NOT support `advisory_available` as a request param. Paginate with `limit=100`, `offset=0,100,200,...` until empty; filter client-side for `attributes.advisory_available === true`. **HITL required** before pagination—confirm with user (systems with 1,700+ CVEs ≈ 18 API calls). - -## vulnerability__get_system_cves - -**Purpose**: List CVEs affecting a specific system. Supports pagination. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `system_uuid` | string (UUID) | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Required | -| `limit` | integer | `100` | Records per page (default 10) | -| `offset` | integer | `0`, `100`, `200` | Pagination offset | -| `sort` | string | `"-public_date"` | Use `-` for descending | - -**Pagination**: Loop with `offset += limit` until `len(data) < limit`. Response includes `attributes.advisory_available` per CVE—filter client-side for remediatable. - -## vulnerability__get_cve_systems - -**Purpose**: List systems affected by a CVE. Supports `system_uuid` filter to check if a specific system is affected. - -| Parameter | Type | Example | Notes | -|-----------|------|---------|-------| -| `cve` | string | `"CVE-2024-1234"` | Required | -| `system_uuid` | string | `"68ce32aa-57da-49b7-8ded-dc4ad54e520a"` | Filter to check if this system is affected | diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-tool-failures.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-tool-failures.md deleted file mode 100644 index f2c661b4..00000000 --- a/evaluation/without_skills/rh-sre__system-context/environment/docs/references/lightspeed-mcp-tool-failures.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Lightspeed MCP Tool Failures — Handling and Workarounds -category: references -tags: [lightspeed, mcp, troubleshooting, errors] -last_updated: 2026-03-02 ---- - -# Lightspeed MCP Tool Failures - -When Lightspeed MCP tools fail with cryptic backend errors (e.g. KeyError, missing keys), follow this pattern instead of exposing the raw error. - -## Generic Pattern - -1. **Do NOT** expose the raw error to the user (e.g. `'dnf_modules'`, `KeyError: 'xyz'`) -2. **Show** a user-friendly message explaining what happened and what we know -3. **Use** alternative tools to achieve the same goal when possible -4. **Do NOT** retry the failing tool—backend errors typically persist for the same request - -## Known Failures and Workarounds - -### vulnerability__get_cves — `limit_` Unexpected keyword argument - -**Error**: `1 validation error for call[get_cves] limit_ Unexpected keyword argument [type=unexpected_keyword_argument]` - -**Cause**: Some MCP clients incorrectly serialize the `limit` parameter as `limit_`. The Lightspeed MCP server expects `limit` (no underscore). - -**Workaround**: For connectivity tests, call with **no parameters**—the tool uses default `limit=10`: -``` -vulnerability__get_cves() -``` -Or pass only parameters that don't trigger the bug (e.g. `impact`, `sort`, `advisory_available`). Avoid passing `limit` when the client may serialize it as `limit_`. - -**Skills affected**: mcp-lightspeed-validator (connectivity test), cve-impact (account-level CVE queries). - -### vulnerability__explain_cves — `'dnf_modules'` (or similar KeyError) - -**Error**: `Error calling tool 'explain_cves': 'dnf_modules'` - -**Cause**: Backend expects a `dnf_modules` key in the system profile; some systems don't include it. - -**User-friendly message**: -``` -⚠️ CVE explanation unavailable for this system - -The detailed "why this CVE affects your system" explanation could not be retrieved. -This sometimes happens when the system profile is missing module data. - -**What we know** (from other sources): -- CVE: [CVE-ID] -- Affected system: [hostname] -- Severity: [from get_cve] -- Affected packages: [from get_cve] - -**Next steps**: Proceeding with available data. The playbook will still address the CVE correctly. -``` - -**Workaround**: Synthesize from `get_cve` + `get_host_details`: -1. `get_cve(cve_id)` → affected_packages, severity, advisory -2. `get_host_details(system_id)` → installed_packages -3. Match and explain: "CVE-X affects system Y because [package] is installed at [version]. Fix: [advisory]." - -**Skills**: cve-impact (if explaining why CVE affects system); remediation and system-context do NOT use explain_cves. - -### Other Tool Failures - -When a different tool fails with a similar cryptic error: -1. Apply the generic pattern (no raw error, user-friendly message) -2. Identify alternative tools that provide equivalent data -3. Add the failure and workaround to this doc for future reference diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/references/skill-invocation.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/references/skill-invocation.md deleted file mode 100644 index 132854c8..00000000 --- a/evaluation/without_skills/rh-sre__system-context/environment/docs/references/skill-invocation.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Skill Invocation Reference -category: references -tags: [skills, invocation, troubleshooting] -last_updated: 2026-03-02 ---- - -# Skill Invocation Reference - -Guidance for correctly invoking skills in the rh-sre pack across different AI hosts (Cursor, Claude Code, etc.). - -## Invoking Skills (All Sub-Skills) - -When the remediation skill (or other orchestrators) invokes any sub-skill—validators, cve-validation, cve-impact, system-context, playbook-generator, playbook-executor, remediation-verifier: - -- **Use the Skill tool** with the skill name. Format may vary by host: - - Cursor: `Skill(rh-sre:mcp-lightspeed-validator)` or similar - - Claude Code: `/mcp-lightspeed-validator` or `Skill(mcp-lightspeed-validator)` -- **Wait for the skill to complete**—skills typically return output directly. Do NOT proceed to the next step until you have the skill's actual result (e.g. validation PASSED/FAILED). "Successfully loaded skill" indicates the skill was loaded, not that it finished—wait for the validation outcome before continuing. -- **Do NOT use "Task Output" with the skill name as the task ID.** If you see "No task found with ID: mcp-lightspeed-validator" (or cve-validation, cve-impact, etc.), you are passing the skill name to a Task Output tool. Task Output expects the task ID returned from an async invocation (e.g. a UUID), NOT the skill name. Skill names are not task IDs. - -## If Validator Invocation Fails - -If validator invocation returns "No task found" or similar: - -1. **Do NOT block the workflow.** Proceed with a warning. -2. **Inform the user**: "Validator invocation encountered an issue. Proceeding with remediation workflow—MCP operations in later steps will confirm connectivity." -3. **Continue to Step 2** (cve-validation). The `get_cve` call will fail if Lightspeed MCP is unavailable. -4. **Continue to Step 5** (playbook-executor). AAP MCP calls will fail if AAP is unavailable. - -The workflow is resilient: actual MCP tool calls in later steps serve as implicit validation. Do not retry Task Output with the skill name. - -## Validation Freshness - -If validation was performed earlier in the same session and succeeded, you may skip re-invoking validators. See each validator skill's "Validation Freshness Policy" section. diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/rhel/README.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/rhel/README.md deleted file mode 100644 index 3f8526e9..00000000 --- a/evaluation/without_skills/rh-sre__system-context/environment/docs/rhel/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: RHEL Documentation Overview -category: rhel -last_updated: 2026-01-20 ---- - -# RHEL Documentation Overview - -This directory contains Red Hat Enterprise Linux-specific guidance for CVE remediation. - -## Available Documentation - -### Priority P0 (Core) -- **[package-management.md](package-management.md)** - DNF/YUM workflows, reboot detection, service restarts - - RHEL 7/8/9 compatibility - - Package update patterns - - Repository management - - Subscription Manager integration - -### Future Enhancements (P1-P2 Priority) -- **selinux-context.md** - SELinux remediation patterns (planned) -- **security-hardening-rhel9.md** - RHEL 9 security baseline (planned) -- **version-compatibility.md** - RHEL 7/8/9 comparison matrix (planned) -- **systemd-services.md** - Service management patterns (planned) - -## When to Use These Docs - -**Use package-management.md when**: -- Creating playbooks that update packages -- Need to detect if reboot is required (needs-restarting) -- Working across multiple RHEL versions (7/8/9) -- Handling DNF/YUM differences -- Managing service restarts after package updates -- Troubleshooting repository or subscription issues - -## Quick Links - -- Official Red Hat RHEL 9 docs: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9 -- Package management guide: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index -- Source attribution: [../SOURCES.md](../SOURCES.md) diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/rhel/package-management.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/rhel/package-management.md deleted file mode 100644 index f7e4252e..00000000 --- a/evaluation/without_skills/rh-sre__system-context/environment/docs/rhel/package-management.md +++ /dev/null @@ -1,738 +0,0 @@ ---- -title: RHEL Package Management for CVE Remediation -category: rhel -sources: - - title: Managing Software with the DNF Tool (RHEL 9) - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - sections: DNF commands, updating packages, repository management - date_accessed: 2026-01-20 - - title: Software Management in RHEL 9 Adoption Guide - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - sections: RHEL 7/8/9 compatibility, migration considerations - date_accessed: 2026-01-20 - - title: Updating RHEL 9 Content - url: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - sections: Package update procedures, reboot detection - date_accessed: 2026-01-20 -tags: [dnf, yum, package-management, rhel, updates, systemd, reboot-detection] -applies_to: [rhel7, rhel8, rhel9] -semantic_keywords: - - "DNF package manager" - - "YUM package manager" - - "package update" - - "repository management" - - "reboot detection" - - "systemd service management" - - "needs-restarting" - - "subscription manager" -use_cases: - - "package_update_cve" - - "rhel_version_compatibility" - - "reboot_detection" - - "service_restart_after_update" -related_docs: - - "ansible/cve-remediation-templates.md" - - "rhel/version-compatibility.md" - - "rhel/systemd-services.md" -last_updated: 2026-01-20 ---- - -# RHEL Package Management for CVE Remediation - -This document provides comprehensive guidance on package management across RHEL 7, 8, and 9 for CVE remediation scenarios. - -## Overview - -Red Hat Enterprise Linux uses different package managers across versions: -- **RHEL 7**: YUM (Yellowdog Updater Modified) -- **RHEL 8**: DNF (Dandified YUM) with `yum` as an alias -- **RHEL 9**: DNF with `yum` as an alias - -**Key Insight**: In RHEL 8 and 9, `yum` is a symbolic link to `dnf` for backward compatibility. All YUM commands work identically in RHEL 8/9 via DNF. - -## DNF vs YUM Command Compatibility - -### Command Equivalence Table - -| Operation | RHEL 7 (YUM) | RHEL 8/9 (DNF) | Notes | -|-----------|--------------|----------------|-------| -| Update package | `yum update httpd` | `dnf update httpd` or `yum update httpd` | Identical behavior | -| Install package | `yum install httpd` | `dnf install httpd` or `yum install httpd` | Identical behavior | -| Remove package | `yum remove httpd` | `dnf remove httpd` or `yum remove httpd` | Identical behavior | -| Search packages | `yum search keyword` | `dnf search keyword` or `yum search keyword` | Identical behavior | -| List installed | `yum list installed` | `dnf list installed` or `yum list installed` | Identical behavior | -| Clean cache | `yum clean all` | `dnf clean all` or `yum clean all` | Identical behavior | -| Check updates | `yum check-update` | `dnf check-update` or `yum check-update` | Identical behavior | - -### Ansible Module Compatibility - -```yaml -# RHEL 7 - Use yum module -- name: Update packages (RHEL 7) - yum: - name: httpd - state: latest - when: ansible_distribution_major_version == "7" - -# RHEL 8/9 - Use dnf module (preferred) -- name: Update packages (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] - -# Universal approach - yum module works on all versions -- name: Update packages (All RHEL versions) - package: - name: httpd - state: latest - # Uses appropriate package manager automatically -``` - -## Package Update Patterns for CVE Remediation - -### Pattern 1: Single Package Update - -**Use Case**: CVE affects a specific package (e.g., httpd, openssl, glibc) - -```yaml -- name: Update vulnerable package - dnf: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version in ["8", "9"] - -- name: Update vulnerable package (RHEL 7) - yum: - name: httpd - state: latest - update_cache: true - register: package_update - when: ansible_distribution_major_version == "7" -``` - -**Key Options**: -- `state: latest` - Updates to newest available version -- `update_cache: true` - Refreshes repository metadata before update -- `register: package_update` - Captures update results for verification - -### Pattern 2: Multiple Related Packages - -**Use Case**: CVE affects a package and its dependencies (e.g., openssl + openssl-libs) - -```yaml -- name: Update vulnerable packages and dependencies - dnf: - name: - - openssl - - openssl-libs - - openssl-devel - state: latest - update_cache: true - register: package_update -``` - -**Why This Matters**: Some CVEs affect shared libraries. Updating only the main package may leave vulnerabilities in dependent libraries. - -### Pattern 3: Kernel Package Updates - -**Use Case**: Kernel CVEs requiring reboot - -```yaml -- name: Update kernel package - dnf: - name: kernel - state: latest - update_cache: true - register: kernel_update - -- name: Record current kernel before reboot - command: uname -r - register: current_kernel - changed_when: false - -# Reboot will be handled separately -# See: Template 4 in cve-remediation-templates.md -``` - -**Important**: Kernel updates **always** require a reboot. New kernel is not active until system restarts. - -### Pattern 4: Security-Only Updates - -**Use Case**: Apply only security updates, not all available updates - -```bash -# RHEL 8/9 - Security updates only -dnf update --security - -# RHEL 7 - Requires yum-plugin-security -yum update --security -``` - -**Ansible Equivalent**: -```yaml -- name: Apply security updates only (RHEL 8/9) - command: dnf update -y --security - register: security_updates - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Repository Management - -### Enabling/Disabling Repositories - -```yaml -- name: Enable repository for specific package - command: subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms - when: ansible_distribution_major_version == "9" - -- name: Update package from specific repo - dnf: - name: httpd - state: latest - enablerepo: rhel-9-for-x86_64-appstream-rpms -``` - -### Repository List (RHEL 9) - -Common repositories for CVE remediation: -- `rhel-9-for-x86_64-baseos-rpms` - Base OS packages -- `rhel-9-for-x86_64-appstream-rpms` - Application streams -- `rhel-9-for-x86_64-supplementary-rpms` - Supplementary packages - -### Verifying Repository Configuration - -```yaml -- name: List enabled repositories - command: subscription-manager repos --list-enabled - register: enabled_repos - changed_when: false - -- name: Display enabled repos - debug: - msg: "{{ enabled_repos.stdout_lines }}" -``` - -## Reboot Detection Patterns - -### Method 1: Check for Reboot-Required File - -```yaml -- name: Check if reboot is required (file-based) - stat: - path: /var/run/reboot-required - register: reboot_required_file - -- name: Notify if reboot needed - debug: - msg: "⚠️ System reboot required" - when: reboot_required_file.stat.exists -``` - -**Note**: Not all RHEL systems create this file. More reliable method below. - -### Method 2: needs-restarting Command (RHEL 8/9) - -**Most Reliable Method for RHEL 8/9** - -```yaml -- name: Check if reboot is required (needs-restarting) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Determine reboot requirement - set_fact: - reboot_required: "{{ needs_restarting.rc != 0 }}" - when: ansible_distribution_major_version in ["8", "9"] - -- name: Display reboot status - debug: - msg: "Reboot required: {{ reboot_required }}" -``` - -**Exit Codes**: -- `0` - No reboot required -- `1` - Reboot required (kernel, glibc, systemd, or other core component updated) - -### Method 3: Check Specific Package Updates - -```yaml -- name: Check if kernel was updated - shell: | - LATEST_KERNEL=$(rpm -q kernel --last | head -1 | awk '{print $1}') - RUNNING_KERNEL=$(uname -r) - if [[ "$LATEST_KERNEL" != "kernel-$RUNNING_KERNEL" ]]; then - echo "reboot_needed" - fi - register: kernel_check - changed_when: false - -- name: Set reboot flag if kernel changed - set_fact: - reboot_required: true - when: "'reboot_needed' in kernel_check.stdout" -``` - -### Comprehensive Reboot Detection - -**Recommended Pattern for CVE Remediation**: - -```yaml -- name: Comprehensive reboot detection - block: - - name: Check needs-restarting (RHEL 8/9) - command: needs-restarting -r - register: needs_restarting - failed_when: false - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - - - name: Check reboot-required file - stat: - path: /var/run/reboot-required - register: reboot_file - - - name: Check if kernel was updated - shell: | - rpm -q --last kernel | head -1 | \ - grep -q "$(uname -r)" || echo "kernel_updated" - register: kernel_check - changed_when: false - failed_when: false - - - name: Determine final reboot requirement - set_fact: - reboot_required: > - {{ - reboot_file.stat.exists | default(false) or - (needs_restarting.rc != 0 | default(false)) or - ('kernel_updated' in kernel_check.stdout) - }} - - - name: Display reboot requirement - debug: - msg: | - Reboot Required: {{ reboot_required }} - Reason: {% if reboot_file.stat.exists %}reboot-required file exists{% elif needs_restarting.rc != 0 %}needs-restarting check{% elif 'kernel_updated' in kernel_check.stdout %}kernel update{% else %}unknown{% endif %} -``` - -## Service Restart After Package Updates - -### Pattern 1: Restart Specific Services - -```yaml -- name: Restart httpd after package update - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required - -- name: Wait for service to be active - systemd: - name: httpd - state: started - retries: 3 - delay: 5 -``` - -### Pattern 2: Restart Services Requiring Updates (RHEL 8/9) - -```yaml -- name: Find services that need restarting - command: needs-restarting -s - register: services_to_restart - changed_when: false - when: ansible_distribution_major_version in ["8", "9"] - -- name: Parse service names - set_fact: - service_list: "{{ services_to_restart.stdout_lines | map('regex_replace', '^(.+)\\.service$', '\\1') | list }}" - when: services_to_restart.stdout_lines | length > 0 - -- name: Restart services that need it - systemd: - name: "{{ item }}" - state: restarted - loop: "{{ service_list }}" - when: - - service_list is defined - - not reboot_required - ignore_errors: true -``` - -**`needs-restarting -s` Output Example**: -``` -httpd.service -NetworkManager.service -sshd.service -``` - -### Pattern 3: Conditional Service Restart Based on Package - -```yaml -- name: Map packages to services - set_fact: - package_service_map: - httpd: httpd - nginx: nginx - sshd: sshd - openssl: [httpd, nginx, sshd] # Multiple services may use openssl - -- name: Restart services for updated packages - systemd: - name: "{{ package_service_map[item] }}" - state: restarted - loop: "{{ package_update.results | map(attribute='item') | list }}" - when: - - package_update is changed - - item in package_service_map - - not reboot_required -``` - -## Package Version Verification - -### Pre/Post Update Version Comparison - -```yaml -- name: Gather package facts before update - package_facts: - manager: auto - -- name: Record pre-update versions - set_fact: - pre_update_versions: "{{ ansible_facts.packages }}" - -- name: Update packages - dnf: - name: "{{ vulnerable_packages }}" - state: latest - register: package_update - -- name: Gather package facts after update - package_facts: - manager: auto - -- name: Compare versions - debug: - msg: | - Package: {{ item }} - Before: {{ pre_update_versions[item][0].version | default('not installed') }} - After: {{ ansible_facts.packages[item][0].version | default('not installed') }} - loop: "{{ vulnerable_packages }}" - when: item in ansible_facts.packages -``` - -### Verify Specific Package Version - -```yaml -- name: Verify package is at required version - shell: | - rpm -q {{ package_name }} --queryformat '%{VERSION}-%{RELEASE}' - register: package_version - changed_when: false - -- name: Assert minimum version - assert: - that: - - package_version.stdout is version(minimum_version, '>=') - fail_msg: "Package {{ package_name }} is {{ package_version.stdout }}, required >= {{ minimum_version }}" - success_msg: "Package {{ package_name }} version {{ package_version.stdout }} meets requirements" -``` - -## Rollback and Backup Strategies - -### RHEL 8/9 Snapshot with Boom - -```yaml -- name: Install boom-boot (if not present) - dnf: - name: boom-boot - state: present - when: ansible_distribution_major_version in ["8", "9"] - -- name: Create pre-update snapshot - command: boom create --title "pre-cve-{{ cve_id }}-{{ ansible_date_time.epoch }}" - register: snapshot_result - ignore_errors: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Log snapshot creation - debug: - msg: "Snapshot created: {{ snapshot_result.stdout }}" - when: snapshot_result is success -``` - -### Package Downgrade (Emergency Rollback) - -```yaml -- name: Downgrade package to previous version - dnf: - name: httpd-2.4.37-1.el8 - state: present - allow_downgrade: true - when: ansible_distribution_major_version in ["8", "9"] - -- name: Downgrade package (RHEL 7) - yum: - name: httpd-2.4.37-1.el7 - state: present - when: ansible_distribution_major_version == "7" -``` - -**Warning**: Downgrades should be rare and only for emergency rollback. May cause dependency issues. - -## Subscription Manager Integration - -### Verify System Registration - -```yaml -- name: Check subscription status - command: subscription-manager status - register: subscription_status - changed_when: false - failed_when: false - -- name: Assert system is registered - assert: - that: - - "'Overall Status: Current' in subscription_status.stdout or 'Overall Status: Simple Content Access' in subscription_status.stdout" - fail_msg: "System is not properly subscribed to Red Hat repositories" - success_msg: "System subscription is current" -``` - -### Refresh Subscription - -```yaml -- name: Refresh subscription data - command: subscription-manager refresh - when: subscription_status.rc != 0 - -- name: Update repository metadata - command: dnf clean all && dnf makecache - when: ansible_distribution_major_version in ["8", "9"] -``` - -## RHEL Version-Specific Considerations - -### RHEL 7 - -- **Package Manager**: YUM (Python 2.7-based) -- **Systemd Version**: 219 -- **Reboot Detection**: No `needs-restarting -r`, use alternative methods -- **Security Updates**: Requires `yum-plugin-security` package - -```yaml -- name: Install security plugin (RHEL 7) - yum: - name: yum-plugin-security - state: present - when: ansible_distribution_major_version == "7" -``` - -### RHEL 8 - -- **Package Manager**: DNF 4.x (Python 3.6-based), `yum` is alias -- **Systemd Version**: 239 -- **Reboot Detection**: `needs-restarting -r` available -- **Module Streams**: AppStream concept introduced - -```yaml -- name: Enable module stream (RHEL 8) - command: dnf module enable httpd:2.4 -y - when: ansible_distribution_major_version == "8" -``` - -### RHEL 9 - -- **Package Manager**: DNF 4.x (Python 3.9-based), `yum` is alias -- **Systemd Version**: 252 -- **Reboot Detection**: `needs-restarting -r` available -- **New in 9.7**: Multisig DNF plugin for quantum-safe RPM verification - -```yaml -- name: Install multisig plugin (RHEL 9.7+) - dnf: - name: python3-dnf-plugin-multisig - state: present - when: - - ansible_distribution_major_version == "9" - - ansible_distribution_version is version('9.7', '>=') -``` - -## Common Pitfalls and Solutions - -### Pitfall 1: Not Refreshing Repository Cache - -**Problem**: Updates fail or don't detect new packages -**Solution**: Always use `update_cache: true` - -```yaml -# ❌ Bad - may miss new package versions -- dnf: - name: httpd - state: latest - -# ✅ Good - ensures latest metadata -- dnf: - name: httpd - state: latest - update_cache: true -``` - -### Pitfall 2: Ignoring Reboot Requirements - -**Problem**: CVE remains exploitable after "update" -**Solution**: Always check and handle reboots - -```yaml -# ✅ Complete pattern -- name: Update package - dnf: - name: kernel - state: latest - -- name: Check reboot requirement - command: needs-restarting -r - register: needs_reboot - failed_when: false - -- name: Notify if reboot needed - debug: - msg: "⚠️ REBOOT REQUIRED - CVE not fully remediated until reboot" - when: needs_reboot.rc != 0 -``` - -### Pitfall 3: Not Verifying Package Update Success - -**Problem**: Package update silently fails, CVE remains -**Solution**: Use `register` and verify changes - -```yaml -- name: Update package - dnf: - name: httpd - state: latest - register: package_update - -- name: Verify update occurred - assert: - that: - - package_update is changed or package_update is success - fail_msg: "Package update failed - CVE remediation incomplete" -``` - -### Pitfall 4: Restarting Services When Reboot Required - -**Problem**: Wasted effort, service restart won't apply kernel updates -**Solution**: Conditional service restarts - -```yaml -- name: Restart service only if no reboot needed - systemd: - name: httpd - state: restarted - when: - - package_update is changed - - not reboot_required # Don't restart if rebooting anyway -``` - -### Pitfall 5: Using Wrong Package Manager Module - -**Problem**: Playbook fails on different RHEL versions -**Solution**: Use version-conditional tasks or `package` module - -```yaml -# ✅ Best - works on all RHEL versions -- name: Update package (universal) - package: - name: httpd - state: latest - -# ✅ Also good - version-specific -- name: Update package (RHEL 8/9) - dnf: - name: httpd - state: latest - when: ansible_distribution_major_version in ["8", "9"] -``` - -## Quick Reference Commands - -### Package Operations -```bash -# Update single package -dnf update httpd - -# Update all security patches -dnf update --security - -# Update multiple packages -dnf update httpd httpd-tools - -# Check for available updates -dnf check-update - -# List installed packages -dnf list installed - -# Show package info -dnf info httpd - -# Search for package -dnf search webserver -``` - -### Reboot Detection -```bash -# Check if reboot needed (RHEL 8/9) -needs-restarting -r - -# List services needing restart -needs-restarting -s - -# Check current kernel vs installed -uname -r -rpm -q kernel --last | head -1 -``` - -### Repository Management -```bash -# List enabled repos -subscription-manager repos --list-enabled - -# Enable specific repo -subscription-manager repos --enable=repo-name - -# Refresh repo metadata -dnf clean all && dnf makecache -``` - -## Related Documentation - -- **[CVE Remediation Templates](../ansible/cve-remediation-templates.md)** - Playbook templates using these patterns -- **[RHEL Version Compatibility](version-compatibility.md)** - Detailed RHEL 7/8/9 differences -- **[Systemd Services](systemd-services.md)** - Service management patterns -- **[SELinux Context](selinux-context.md)** - SELinux considerations for package updates - -## Official Red Hat Sources - -This document is derived from: - -1. **Managing Software with the DNF Tool (RHEL 9)** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html-single/managing_software_with_the_dnf_tool/index - -2. **Software Management in RHEL 9 Adoption Guide** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/considerations_in_adopting_rhel_9/assembly_software-management_considerations-in-adopting-rhel-9 - -3. **Updating RHEL 9 Content** - https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/managing_software_with_the_dnf_tool/assembly_updating-rhel-9-content_managing-software-with-the-dnf-tool - -**License**: Content derived from Red Hat documentation under CC BY-SA 4.0 -**Last Verified**: 2026-01-20 diff --git a/evaluation/without_skills/rh-sre__system-context/environment/docs/testing/aap-integration-test-guide.md b/evaluation/without_skills/rh-sre__system-context/environment/docs/testing/aap-integration-test-guide.md deleted file mode 100644 index 6c122770..00000000 --- a/evaluation/without_skills/rh-sre__system-context/environment/docs/testing/aap-integration-test-guide.md +++ /dev/null @@ -1,649 +0,0 @@ ---- -title: AAP Integration Test Guide -category: testing -sources: - - title: Internal Testing Documentation - date_accessed: 2026-02-24 -tags: [testing, aap-integration, workflow-verification, remediation-testing] -semantic_keywords: [aap integration testing, workflow verification, remediation test] -use_cases: [remediation, playbook-executor] -related_docs: [aap-job-execution.md, playbook-integration-aap.md] -last_updated: 2026-02-24 ---- - -# AAP Integration Test Guide - -## Overview - -This guide provides a comprehensive testing plan for the AAP MCP integration, covering the complete CVE remediation workflow from analysis through execution to verification. - -## Prerequisites for Testing - -### Required Setup - -1. **AAP Environment**: - - AAP 2.4+ instance accessible - - Valid API token with appropriate permissions - - At least one project configured - - At least one inventory with test systems - - At least one job template (or ability to create one) - -2. **Environment Variables**: - ```bash - export AAP_MCP_SERVER="https://your-aap-mcp-endpoint.com" - export AAP_API_TOKEN="your-api-token" - ``` - -3. **Test Systems**: - - At least 2-3 RHEL systems in AAP inventory - - Systems registered with Red Hat Lightspeed - - Systems have known CVEs for testing - - SSH access configured with credentials in AAP - -4. **MCP Configuration**: - - `rh-sre/.mcp.json` configured with AAP MCP servers - - `lightspeed-mcp` configured and working - - All environment variables set - -### Verification Checklist - -Before starting tests, verify: - -- [ ] AAP Web UI accessible (your AAP instance URL) -- [ ] Can log in with your credentials -- [ ] API token has been generated -- [ ] Environment variables are set (run: `env | grep AAP`) -- [ ] Test systems visible in AAP inventory -- [ ] Test systems have CVEs in Red Hat Lightspeed -- [ ] Git repository available for playbook storage - -## Test Plan Structure - -``` -Test Phase 1: Component Testing -├─ Test 1.1: AAP MCP Validator -├─ Test 1.2: Job Template Lister -├─ Test 1.3: Playbook Generator -└─ Test 1.4: Inventory Access - -Test Phase 2: Integration Testing -├─ Test 2.1: Template Selection Workflow -├─ Test 2.2: Dry-Run Execution -├─ Test 2.3: Production Execution -└─ Test 2.4: Error Handling - -Test Phase 3: End-to-End Testing -├─ Test 3.1: Full Remediator Workflow -├─ Test 3.2: Multi-CVE Remediation -└─ Test 3.3: Partial Failure Recovery - -Test Phase 4: Performance Testing -└─ Test 4.1: Large-Scale Execution -``` - -## Test Phase 1: Component Testing - -### Test 1.1: AAP MCP Validator - -**Objective**: Verify AAP MCP server connectivity and resource availability. - -**Steps**: -1. Invoke the mcp-aap-validator skill -2. Observe validation checks -3. Confirm all checks pass - -**Expected Results**: -``` -✓ AAP MCP Validation: PASSED - -Configuration: -✓ MCP server aap-mcp-job-management configured -✓ MCP server aap-mcp-inventory-management configured -✓ Environment variable AAP_MCP_SERVER is set -✓ Environment variable AAP_API_TOKEN is set -✓ Job management server connectivity verified -✓ Inventory management server connectivity verified - -Resources: -✓ Found N job template(s) available -✓ Found M inventory/inventories available - -Ready to execute AAP operations. -``` - -**Pass Criteria**: -- All configuration checks pass -- Both MCP servers connect successfully -- At least 1 job template found -- At least 1 inventory found - -**Troubleshooting**: -- If fails: Review error message and fix configuration -- If partial: Note warnings but may proceed if resources exist -- If connection fails: Check AAP server status and credentials - -### Test 1.2: Job Template Lister - -**Objective**: Verify ability to list and filter job templates. - -**Test Command**: Use `job_templates_list` MCP tool via skill - -**Steps**: -1. Request list of all job templates -2. Verify response contains expected templates -3. Note template IDs for later tests - -**Expected Results**: -- List of templates with IDs, names, projects, inventories -- At least 1 template suitable for remediation - -**Pass Criteria**: -- Tool returns valid response -- Template data includes required fields -- Can identify suitable template for testing - -### Test 1.3: Playbook Generator - -**Objective**: Verify playbook generation from CVE data. - -**Steps**: -1. Invoke playbook-generator skill with a known CVE -2. Review generated playbook -3. Verify playbook has required sections - -**Test Input**: -- CVE ID: Use a real CVE affecting your test systems -- Target systems: Your test system UUIDs - -**Expected Results**: -- Valid Ansible YAML playbook generated -- Includes: pre-flight checks, package updates, service restarts -- Follows Red Hat best practices -- Has proper error handling - -**Pass Criteria**: -- Playbook is syntactically valid YAML -- Contains all remediation tasks -- Includes backup/rollback steps -- Has audit logging - -### Test 1.4: Inventory Access - -**Objective**: Verify ability to query AAP inventories and hosts. - -**Test Command**: Use `inventories_list` and `hosts_list` MCP tools - -**Steps**: -1. List all inventories -2. Select test inventory -3. List hosts in that inventory -4. Verify test systems are present - -**Expected Results**: -- Inventory list returned -- Can query hosts within inventory -- Test systems visible with correct metadata - -**Pass Criteria**: -- At least 1 inventory returned -- Hosts query succeeds -- Test systems found in inventory - -## Test Phase 2: Integration Testing - -### Test 2.1: Template Selection Workflow - -**Objective**: Test the template selection and creation workflow. - -**Scenario A: Existing Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Skill lists available templates -3. Select an existing compatible template -4. Verify selection is accepted - -**Expected Results**: -``` -Found N compatible job template(s): - -1. "CVE Remediation Template" (ID: 10) - - Inventory: Production Servers (1) - - Project: Remediation Playbooks (5) - - Credentials: ✓ Configured - -Select template number or "create" for new: 1 - -✓ Using template: CVE Remediation Template (ID: 10) -``` - -**Pass Criteria**: -- Templates listed successfully -- User can select a template -- Selection is confirmed - -**Scenario B: Create New Template** - -**Steps**: -1. Invoke playbook-executor skill -2. Choose "create" option -3. Follow template creation guidance -4. Verify template appears in AAP - -**Expected Results**: -- User guided through Web UI creation -- Template created with correct settings -- Template visible in `job_templates_list` - -**Pass Criteria**: -- Guidance is clear and actionable -- Template created successfully -- Template has required configuration - -### Test 2.2: Dry-Run Execution - -**Objective**: Test check mode (dry-run) execution. - -**Steps**: -1. Generate a remediation playbook -2. Select job template -3. Choose "yes" when asked about dry-run -4. Wait for dry-run to complete -5. Review dry-run results - -**Expected Results**: -``` -⏳ Dry-run in progress... - -Job ID: 1234 -Status: running - -# Dry-Run Results - -## Job Summary -**Job ID**: 1234 -**Status**: ✓ Successful (Check Mode) -**Duration**: 2m 15s - -## Simulated Changes -| Host | Would Change | OK | Failed | Status | -|------|--------------|-----|--------|--------| -| test-01 | 2 | 6 | 0 | ✓ Ready | -| test-02 | 2 | 6 | 0 | ✓ Ready | - -✓ No errors detected in dry-run -``` - -**Pass Criteria**: -- Job launches with `job_type: "check"` -- Execution completes successfully -- Results show "would change" counts -- No actual changes made to systems -- User asked to proceed with actual execution - -### Test 2.3: Production Execution - -**Objective**: Test actual playbook execution (run mode). - -**Steps**: -1. After successful dry-run, approve actual execution -2. Monitor execution progress -3. Wait for completion -4. Review execution report - -**Expected Results**: -``` -⏳ Execution in progress... - -Job ID: 1235 -Status: running - -# Playbook Execution Report - -## Job Summary -**Job ID**: 1235 -**Status**: ✅ Successful -**Duration**: 3m 45s - -## Per-Host Results -| Host | OK | Changed | Failed | Unreachable | Status | -|------|-----|---------|--------|-------------|--------| -| test-01 | 6 | 2 | 0 | 0 | ✅ Success | -| test-02 | 6 | 2 | 0 | 0 | ✅ Success | - -**Summary**: 2 of 2 hosts successfully remediated - -## Next Steps -☐ Verify remediation with remediation-verifier skill -``` - -**Pass Criteria**: -- Job launches with `job_type: "run"` -- Real-time progress displayed -- Execution completes successfully -- All hosts show success status -- Comprehensive report generated -- AAP URL provided for detailed view - -### Test 2.4: Error Handling - -**Objective**: Test error handling and recovery. - -**Scenario A: Partial Host Failure** - -**Setup**: -- Use 3 test systems -- Cause failure on 1 system (e.g., remove package, stop service) - -**Steps**: -1. Execute remediation playbook -2. Observe partial failure -3. Review error report -4. Choose to relaunch for failed host - -**Expected Results**: -``` -⚠️ Playbook Execution Completed with Failures - -Job ID: 1236 -Systems Remediated: 2 of 3 -Failed Systems: test-03 - -## Failed Tasks Details -**Host**: test-03 -**Task**: Update package httpd -**Error**: "No package httpd available" -**Recommendation**: Check repository configuration - -Would you like to: -1. Relaunch for failed host only -2. Fix issues manually and relaunch -``` - -**Pass Criteria**: -- Failure detected and reported -- Specific error message provided -- Troubleshooting guidance given -- Relaunch option offered -- Can successfully relaunch for failed host only - -**Scenario B: Connection Failure** - -**Setup**: -- Block SSH to one test system (firewall rule) - -**Steps**: -1. Execute remediation playbook -2. Observe connection failure -3. Review error categorization - -**Expected Results**: -``` -❌ Host test-02: unreachable - -**Error Category**: Connection Failure - -**Troubleshooting**: -1. Check SSH service: systemctl status sshd -2. Verify firewall: firewall-cmd --list-all -3. Test connectivity: ping test-02 -``` - -**Pass Criteria**: -- Connection failure detected -- Categorized as connection error -- Specific troubleshooting provided - -## Test Phase 3: End-to-End Testing - -### Test 3.1: Full Remediator Workflow - -**Objective**: Test complete CVE remediation from analysis to verification. - -**Steps**: -1. **Invoke remediation skill** with a known CVE -2. **Impact Analysis**: Review CVE risk assessment -3. **CVE Validation**: Confirm CVE is valid and has remediation -4. **System Context**: Review affected systems and strategy -5. **Playbook Generation**: Review generated playbook, approve -6. **Dry-Run**: Run check mode, review results, approve production -7. **Execution**: Monitor real execution, review report -8. **Verification**: Verify CVE status updated in Lightspeed - -**Test Input**: -``` -User: "Remediate CVE-YYYY-NNNNN on my test systems" -``` - -**Expected Flow**: -1. Agent analyzes CVE impact -2. Agent validates CVE exists -3. Agent gathers system context -4. Agent generates playbook -5. Agent offers dry-run → User approves -6. Agent shows dry-run results -7. Agent asks for production execution → User approves -8. Agent executes playbook -9. Agent reports success -10. Agent suggests verification -11. User invokes remediation-verifier -12. Verifier confirms CVE resolved - -**Pass Criteria**: -- All steps complete without errors -- User prompted at appropriate points -- Dry-run shows simulated changes -- Production execution succeeds -- CVE status updated in Lightspeed -- Comprehensive report at each stage - -**Timeline**: ~10-15 minutes for full workflow - -### Test 3.2: Multi-CVE Remediation - -**Objective**: Test batch remediation of multiple CVEs. - -**Steps**: -1. Invoke remediation skill with 2-3 CVEs -2. Verify agent handles batch processing -3. Confirm single consolidated playbook generated -4. Execute remediation -5. Verify all CVEs resolved - -**Test Input**: -``` -User: "Remediate CVE-2024-1234, CVE-2024-5678, CVE-2024-9012" -``` - -**Expected Results**: -- Agent processes all CVEs -- Consolidated playbook with all fixes -- Single job execution covering all changes -- Report shows results per CVE - -**Pass Criteria**: -- Batch processing works correctly -- Playbook includes all remediation tasks -- Execution handles multiple changes -- Verification confirms all CVEs resolved - -### Test 3.3: Partial Failure Recovery - -**Objective**: Test recovery from partial failures. - -**Scenario**: 5 test systems, 2 fail during execution - -**Steps**: -1. Execute remediation on 5 systems -2. Observe 2 failures -3. Review error analysis -4. Fix issues on failed systems -5. Relaunch for failed systems only -6. Verify all systems eventually succeed - -**Expected Results**: -- Partial success reported (3 of 5) -- Failed systems identified -- Relaunch targets only failed systems -- Second execution succeeds -- Final report shows 5 of 5 success - -**Pass Criteria**: -- Partial failure handled gracefully -- Relaunch doesn't re-run successful hosts -- Ultimate success achieved -- Audit trail shows full history - -## Test Phase 4: Performance Testing - -### Test 4.1: Large-Scale Execution - -**Objective**: Test performance with larger number of systems. - -**Setup**: -- Use 20+ systems in inventory -- Single CVE affecting all systems - -**Steps**: -1. Execute remediation targeting 20+ systems -2. Monitor execution time -3. Review AAP resource usage -4. Verify all systems succeed - -**Expected Results**: -- Execution completes in reasonable time -- Progress monitoring works at scale -- All systems remediated successfully -- Report generated efficiently - -**Pass Criteria**: -- Job completes within expected timeframe -- No timeouts or performance degradation -- Monitoring provides useful progress updates -- Final report is comprehensive - -**Performance Benchmarks**: -- 10 systems: ~5-10 minutes -- 20 systems: ~10-20 minutes -- 50 systems: ~20-40 minutes -(Times vary based on package size and network) - -## Test Reporting Template - -### Test Execution Report - -```markdown -# AAP Integration Test Report - -**Date**: YYYY-MM-DD -**Tester**: [Name] -**Environment**: [AAP Server URL] -**Test Phase**: [1-4] - -## Summary -- Tests Run: N -- Tests Passed: N -- Tests Failed: N -- Pass Rate: NN% - -## Phase 1: Component Testing -- [ ] Test 1.1: AAP MCP Validator - PASS/FAIL -- [ ] Test 1.2: Job Template Lister - PASS/FAIL -- [ ] Test 1.3: Playbook Generator - PASS/FAIL -- [ ] Test 1.4: Inventory Access - PASS/FAIL - -## Phase 2: Integration Testing -- [ ] Test 2.1: Template Selection - PASS/FAIL -- [ ] Test 2.2: Dry-Run Execution - PASS/FAIL -- [ ] Test 2.3: Production Execution - PASS/FAIL -- [ ] Test 2.4: Error Handling - PASS/FAIL - -## Phase 3: End-to-End Testing -- [ ] Test 3.1: Full Remediator Workflow - PASS/FAIL -- [ ] Test 3.2: Multi-CVE Remediation - PASS/FAIL -- [ ] Test 3.3: Partial Failure Recovery - PASS/FAIL - -## Phase 4: Performance Testing -- [ ] Test 4.1: Large-Scale Execution - PASS/FAIL - -## Issues Found -1. [Issue description] - Severity: High/Medium/Low -2. [Issue description] - Severity: High/Medium/Low - -## Recommendations -1. [Recommendation] -2. [Recommendation] - -## Sign-Off -Tested by: [Name] -Approved by: [Name] -Date: YYYY-MM-DD -``` - -## Common Issues and Solutions - -### Issue: "AAP MCP Validation Failed" - -**Symptoms**: Validation fails with connection errors - -**Solutions**: -1. Verify `AAP_MCP_SERVER` environment variable is correct (must point to the MCP endpoint of the AAP server) -2. Check API token is valid and not expired -3. Ensure AAP server is accessible from your network -4. Review AAP MCP server logs for errors - -### Issue: "No Job Templates Found" - -**Symptoms**: Validation passes but no templates available - -**Solutions**: -1. Create job template via AAP Web UI -2. Ensure project is synced and contains playbooks -3. Verify inventory is configured -4. Check credentials are attached to template - -### Issue: "Dry-Run Shows No Changes" - -**Symptoms**: Dry-run completes but reports 0 changes - -**Solutions**: -1. Verify systems actually need remediation -2. Check playbook targets correct hosts -3. Ensure package names are correct -4. Review playbook conditionals (when clauses) - -### Issue: "Execution Hangs" - -**Symptoms**: Job starts but never completes - -**Solutions**: -1. Check AAP Web UI for job status -2. Review job output for stuck tasks -3. Verify systems are reachable -4. Increase job timeout in template settings - -## Sign-Off Criteria - -Before considering AAP integration complete, verify: - -- [ ] All Phase 1 tests pass -- [ ] All Phase 2 tests pass -- [ ] At least Test 3.1 passes (full workflow) -- [ ] No critical issues remain -- [ ] Documentation is accurate -- [ ] Examples work as described -- [ ] Performance is acceptable - -## Next Steps After Testing - -1. **Document Results**: Complete test report template -2. **Fix Issues**: Address any failures found -3. **Update Documentation**: Correct any inaccuracies -4. **User Acceptance**: Have users test workflow -5. **Production Rollout**: Enable for production use - -## Related Documentation - -- [AAP Job Execution Guide](../ansible/aap-job-execution.md) -- [Playbook Integration with AAP](../ansible/playbook-integration-aap.md) -- [CVE Remediation Templates](../ansible/cve-remediation-templates.md) diff --git a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/.ai-index/semantic-index.json deleted file mode 100644 index 2512c2a3..00000000 --- a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/.ai-index/semantic-index.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "index_version": "1.0.0", - "last_updated": "2026-02-17", - "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", - - "documents": [ - { - "file": "INDEX.md", - "title": "VM Troubleshooting Guide - Index", - "category": "navigation", - "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], - "use_cases": ["navigation", "tool-reference", "general-guidance"], - "token_estimate": 450, - "skills": ["all"] - }, - { - "file": "scheduling-errors.md", - "title": "VM Scheduling Errors", - "category": "scheduling", - "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], - "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], - "errors_covered": [ - "ErrorUnschedulable - Node Taints", - "ErrorUnschedulable - Insufficient Resources", - "ErrorUnschedulable - Node Selector Mismatch" - ], - "token_estimate": 820, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "storage-errors.md", - "title": "VM Storage Errors", - "category": "storage", - "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], - "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], - "errors_covered": [ - "ErrorDataVolumeNotReady - DataVolume Still Provisioning", - "ErrorDataVolumeNotReady - Storage Class Not Found", - "ErrorDataVolumeNotReady - Insufficient Storage Quota", - "ErrorPvcNotFound", - "Storage Deletion Failures", - "DataVolume Cloning Failures" - ], - "token_estimate": 1350, - "skills": ["vm-creator", "vm-delete", "vm-clone"] - }, - { - "file": "lifecycle-errors.md", - "title": "VM Lifecycle Errors", - "category": "lifecycle", - "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], - "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], - "errors_covered": [ - "VM Stuck in Terminating State", - "VM Won't Start (Non-Scheduling Issues)", - "VM Won't Stop" - ], - "token_estimate": 1150, - "skills": ["vm-delete", "vm-lifecycle-manager"] - }, - { - "file": "runtime-errors.md", - "title": "VM Runtime Errors", - "category": "runtime", - "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], - "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], - "errors_covered": [ - "CrashLoopBackOff" - ], - "token_estimate": 850, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "network-errors.md", - "title": "VM Network Errors", - "category": "networking", - "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], - "use_cases": ["vm-creation", "network-troubleshooting"], - "errors_covered": [ - "Network Attachment Failures" - ], - "token_estimate": 580, - "skills": ["vm-creator"] - } - ], - - "skill_to_docs_mapping": { - "vm-creator": { - "primary_docs": ["scheduling-errors.md", "storage-errors.md"], - "secondary_docs": ["runtime-errors.md", "network-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3650, - "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" - }, - "vm-delete": { - "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 2950, - "description": "VM deletion failures - stuck terminating, storage cleanup" - }, - "vm-clone": { - "primary_docs": ["storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 1800, - "description": "VM cloning failures - DataVolume cloning, storage provisioning" - }, - "vm-lifecycle-manager": { - "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], - "secondary_docs": ["runtime-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3270, - "description": "VM start/stop failures - lifecycle management, scheduling" - }, - "vm-inventory": { - "primary_docs": [], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 450, - "description": "General guidance for status interpretation" - } - }, - - "error_to_docs_mapping": { - "ErrorUnschedulable": "scheduling-errors.md", - "ErrorDataVolumeNotReady": "storage-errors.md", - "ErrorPvcNotFound": "storage-errors.md", - "Terminating": "lifecycle-errors.md", - "CrashLoopBackOff": "runtime-errors.md", - "NetworkAttachmentFailed": "network-errors.md" - }, - - "cross_references": { - "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], - "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], - "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], - "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], - "network-errors.md": ["scheduling-errors.md"] - }, - - "progressive_disclosure_strategy": { - "description": "Load only the minimum documentation needed for current task", - "pattern": "error_type → primary_doc → cross_references (if needed)", - "token_savings": "29% average reduction vs single-file (based on rh-sre data)", - "navigation_overhead_reduction": "85% vs single-file navigation" - } -} diff --git a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/INDEX.md b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/INDEX.md deleted file mode 100644 index a2e76963..00000000 --- a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/INDEX.md +++ /dev/null @@ -1,332 +0,0 @@ ---- -title: VM Troubleshooting Guide - Index -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 - - title: OpenShift Virtualization - Virtual Machine Status - url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html - date_accessed: 2026-02-06 - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] -semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] -use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] -last_updated: 2026-02-17 ---- - -# VM Troubleshooting Guide - Index - -## Overview - -This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: - -- **Creation**: Scheduling failures, storage provisioning issues -- **Lifecycle**: Start/stop failures, stuck states -- **Deletion**: Resources stuck in Terminating, storage cleanup failures -- **Cloning**: DataVolume cloning errors, cross-namespace issues -- **Networking**: Secondary network attachment failures -- **Runtime**: CrashLoopBackOff, guest OS boot failures - -This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. - ---- - -## 🗂️ Troubleshooting Categories - -The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: - -### 1. [Scheduling Errors](scheduling-errors.md) -**When to use**: VM fails to schedule on any node - -**Errors covered**: -- ErrorUnschedulable - Node Taints -- ErrorUnschedulable - Insufficient Resources -- ErrorUnschedulable - Node Selector Mismatch - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 2. [Storage Errors](storage-errors.md) -**When to use**: VM has storage provisioning, deletion, or cloning issues - -**Errors covered**: -- ErrorDataVolumeNotReady (all 3 subsections) -- ErrorPvcNotFound -- Storage Deletion Failures -- DataVolume Cloning Failures - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### 3. [Lifecycle Errors](lifecycle-errors.md) -**When to use**: VM has start/stop/termination issues - -**Errors covered**: -- VM Stuck in Terminating State -- VM Won't Start (Non-Scheduling Issues) -- VM Won't Stop - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### 4. [Runtime Errors](runtime-errors.md) -**When to use**: VM repeatedly crashes or fails at runtime - -**Errors covered**: -- CrashLoopBackOff - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 5. [Network Errors](network-errors.md) -**When to use**: VM has secondary network attachment failures - -**Errors covered**: -- Network Attachment Failures (Multus, SR-IOV) - -**Skills that use this**: vm-create - ---- - -## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last - -### Critical Principle: MCP-First Approach - -**All diagnostic procedures in this guide follow the MCP-first pattern**: - -``` -1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) -2. ⚠️ IF FAILS: Ask user permission to use CLI command -3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval -``` - -### Why MCP Tools First? - -- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources -- **Consistency**: Same interface across all operations -- **Better Error Handling**: MCP tools return structured errors -- **Audit Trail**: MCP tool usage is logged and trackable - -### Command Pattern Examples - -Throughout this guide, you'll see diagnostic steps formatted like this: - -**✅ CORRECT Pattern (MCP First)**: - -```markdown -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Extract**: `.metadata.finalizers` from returned JSON - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` -``` - -### Available MCP Tools - -The `openshift-virtualization` MCP server provides these tools: - -**Resource Operations**: -- `resources_get` - Get specific resource (replaces `oc get `) -- `resources_list` - List resources (replaces `oc get `) -- `resources_delete` - Delete resource (replaces `oc delete`) -- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) - -**Pod Operations**: -- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) -- `pods_get` - Get pod details (replaces `oc get pod`) -- `pods_log` - Get pod logs (replaces `oc logs`) -- `pods_exec` - Execute in pod (replaces `oc exec`) -- `pods_delete` - Delete pod (replaces `oc delete pod`) -- `pods_top` - Pod resource usage (replaces `oc top pods`) - -**Events & Monitoring**: -- `events_list` - List events (replaces `oc get events`) -- `nodes_top` - Node resource usage (replaces `oc top nodes`) -- `nodes_log` - Node logs (replaces `oc adm node-logs`) -- `nodes_stats_summary` - Detailed node stats - -**VM Operations** (KubeVirt toolset): -- `vm_create` - Create VMs -- `vm_lifecycle` - Start/stop/restart VMs - -**Namespaces**: -- `namespaces_list` - List namespaces (replaces `oc get namespaces`) -- `projects_list` - List OpenShift projects (replaces `oc get projects`) - -### When CLI Commands Are Required - -Some operations have **NO MCP equivalent** and require CLI: - -- `virtctl` commands (console, VNC access) -- `oc debug node` (node debugging) -- `oc auth can-i` (permission checks) -- `oc adm taint` (node taint management) -- Complex JSON patch operations - -For these, the guide will note: **"CLI Only - No MCP equivalent"** - -### Quick Reference: CLI → MCP Mapping - -| CLI Command | MCP Tool Equivalent | -|-------------|---------------------| -| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | -| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | -| `oc logs -n ` | `pods_log` with name="", namespace="" | -| `oc get events -n ` | `events_list` with namespace="" | -| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | - -**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. - -### How to Read Diagnostic Sections - -Each error section includes: -1. **Symptom** - What you observe -2. **Description** - What's happening -3. **Possible Causes** - Why it's happening -4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback -5. **Solutions** - **MCP tools first**, then CLI fallback -6. **Verification** - **MCP tools first**, then CLI fallback - -**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. - ---- - -## 🔍 Quick Navigation by Skill - -**vm-create**: -- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics -- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning -- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff -- [Network Errors](network-errors.md) - Network attachment failures - -**vm-delete**: -- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state -- [Storage Errors](storage-errors.md) - Storage deletion failures - -**vm-clone**: -- [Storage Errors](storage-errors.md) - DataVolume cloning failures - -**vm-lifecycle-manager**: -- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop -- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling - -**vm-inventory**: -- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed - ---- - -## 📊 Documentation Coverage & Maintenance - -### Current Coverage - -This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: - -- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) -- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures -- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop -- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) -- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems - -**Total errors documented**: 12 error types across 6 categories - ---- - -### Encountering Undocumented Errors - -#### For AI Agents (Claude Code) - -If you encounter an error **not documented** in the categories above: - -1. **Report to user** with all available details (error message, affected resources, namespace) -2. **Provide best-effort diagnostics** using MCP tools: - - `resources_get` to inspect resource status - - `pods_log` to check virt-launcher or compute container logs - - `events_list` to view Kubernetes events -3. **Suggest documentation update**: - ``` - ⚠️ This error is not yet documented in the troubleshooting guide. - - **Error**: - **Affected resource**: / - - I recommend adding this error to the troubleshooting guide: - - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file - - If it's a new error class → Create new category file in docs/troubleshooting/ - - Would you like me to help document this error for future reference? - ``` - -4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs - ---- - -#### For Human Contributors - -To document a new error: - -1. **Determine the category**: - - Scheduling issue → `scheduling-errors.md` - - Storage problem → `storage-errors.md` - - Lifecycle/start/stop → `lifecycle-errors.md` - - Runtime crash → `runtime-errors.md` - - Network issue → `network-errors.md` - - New error class → Create new category file (e.g., `security-errors.md`) - -2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): - - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback - - **Solutions**: MCP Tool approach → Implementation → CLI Fallback - - **Verification**: MCP Tool checks → Expected results → CLI Fallback - -3. **Update semantic index**: - - Edit `.ai-index/semantic-index.json` - - Add error to `error_to_docs_mapping` - - Update relevant skill's `primary_docs` or `secondary_docs` - - Increment token estimates if needed - -4. **Reference in skill files**: - - Update skill's Reference Documentation section - - Ensure skill's Document Consultation steps point to the right category - -5. **Update this INDEX.md**: - - Add error to the appropriate category section (lines 50-120) - - Update "Current Coverage" list above - ---- - -## 📚 Additional Resources - -- [rh-virt README](../../README.md) - MCP server setup and skill documentation -- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources -- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/lifecycle-errors.md deleted file mode 100644 index 6167fcc8..00000000 --- a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/lifecycle-errors.md +++ /dev/null @@ -1,869 +0,0 @@ ---- -title: VM Lifecycle Errors -category: kubevirt -sources: - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 -tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] -semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] -use_cases: [vm-deletion, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Lifecycle Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM lifecycle issues including start/stop failures and stuck termination states. - -**When to use this document**: -- VM stuck in `Terminating` state -- VM won't start (runStrategy is Always but VM never reaches Running) -- VM won't stop (runStrategy is Halted but VM never reaches Stopped) - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### VM Stuck in Terminating State - -**Symptom**: VM shows status `Terminating` but deletion never completes - -**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). - -**Possible Causes**: -- Finalizers blocking deletion -- PVC/DataVolume still attached and preventing cleanup -- VirtualMachineInstance (VMI) not terminating properly -- Custom controllers or operators blocking deletion -- Stuck virt-launcher pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check finalizers on the VM**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` - -**2. Check if VMI still exists**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name containing "virt-launcher-". - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check events for deletion issues**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results where `involvedObject.name` == "" and sort by timestamp. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' -``` - -**5. Check VM deletion timestamp**: - -Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. - -If present, VM is in deletion process. If null, VM is not being deleted. - -**Common Finalizer Patterns**: -- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer -- `foregroundDeletion` - Waits for dependent objects to be deleted -- Custom finalizers from operators - -**Solutions** (Use MCP Tools First): - -1. **Wait for dependent resources** (safest approach): - - Use `resources_list` to check if PVCs, DataVolumes are still deleting - - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) - -2. **Force delete VMI** (if VMI is stuck): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable - requires explicit user permission): - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -3. **Force delete virt-launcher pod** (if pod is stuck): - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - # Find the virt-launcher pod name first - oc get pods -n | grep virt-launcher- - - # Then delete it - oc delete pod virt-launcher--xxx -n --force --grace-period=0 - ``` - -4. **Remove finalizers** (⚠️ dangerous - use only as last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` - 2. Remove items from `.metadata.finalizers` array - 3. Update VM using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch not easily done via MCP): - ```bash - # This operation is complex for MCP - may need CLI - oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Check for protection labels** (vm-delete skill specific): - - Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. - - If value is "true", the vm-delete skill refuses deletion (this is expected behavior). - -**Verification**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error if deletion successful. - -**CLI Fallback**: -```bash -oc get vm -n -# Should return: Error from server (NotFound) -``` - ---- - - ---- - -### VM Won't Start (Non-Scheduling Issues) - -**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state - -**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. - -**Possible Causes**: -- Guest OS kernel panic or boot failure -- Cloud-init configuration errors -- Missing or corrupted disk image -- Insufficient memory for guest OS to boot -- QEMU/KVM virtualization errors -- VirtualMachineInstance (VMI) creation failures -- virt-launcher pod crashes - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VMI (VirtualMachineInstance) status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and overall VMI state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**2. Check VMI conditions for errors**: - -Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. View virt-launcher pod logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -First, get pod name from step 3, then: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -Look for QEMU errors, memory allocation failures, device errors. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**5. Check virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. - -Alternatively, use `pods_get` to get full pod details: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**6. Access VM console to see guest OS boot messages**: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -Look for kernel panic, initramfs errors, filesystem errors. - -**7. Check VM events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep -A 20 "Events:" -``` - -**Common Error Patterns**: - -1. **Guest OS Boot Failure**: - - Console shows kernel panic - - Guest hangs at GRUB or boot loader - - Cloud-init errors during first boot - -2. **Insufficient Memory**: - - Guest OS kills processes due to OOM - - VMI logs show memory allocation errors - -3. **QEMU Crashes**: - - virt-launcher logs show QEMU segfaults - - VMI repeatedly restarts - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for kernel panic, initramfs errors, filesystem errors. - -2. **Check virt-launcher pod logs for QEMU errors**: - - Use `pods_log` from diagnostic step 4 to view logs. - - Look for: - - "qemu-system-x86_64: ..." errors - - Memory allocation failures - - Device errors - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx - ``` - -3. **Increase memory if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check VMI memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -4. **Verify disk image integrity**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check PVC is bound: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - If using container disk, verify image exists and is accessible (check VMI spec). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get pvc -n - ``` - -5. **Check cloud-init configuration** (if applicable): - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. - - Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq - ``` - -6. **Restart VMI** (soft reset): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete VMI (VM controller will recreate it): - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for new VMI to start (use `resources_get` to check status). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n -w - ``` - -7. **Check virtualization extensions** (KVM): - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - - # In debug shell: - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should eventually return `Running`). - -Check VMI is running: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should show `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should eventually return: Running - -oc get vmi -n -# Should show: Running -``` - ---- - -### VM Won't Stop - -**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state - -**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. - -**Possible Causes**: -- Guest OS not responding to ACPI shutdown signal -- virt-launcher pod stuck and not terminating -- VirtualMachineInstance (VMI) deletion blocked by finalizers -- Guest shutdown scripts hanging -- Filesystem sync issues in guest OS - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (might show `Stopping`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**2. Check VMI status and deletion timestamp**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check VMI events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vmi -n | grep -A 10 "Events:" -``` - -**5. Check VMI finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.metadata.finalizers}' -``` - -**6. Check if guest is responsive** (if VMI still exists): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Patterns**: -- VMI shows `deletionTimestamp` but never actually deletes -- virt-launcher pod in `Terminating` state -- VM runStrategy is `Halted` but printableStatus shows `Stopping` - -**Solutions** (Use MCP Tools First): - -1. **Wait for graceful shutdown** (default: 30 seconds): - - Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. - -2. **Force stop by deleting VMI**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - This is the standard way to force-stop a VM. - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n - # Should return: Error from server (NotFound) - ``` - -3. **Force delete VMI with grace period** (if VMI won't delete): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -4. **Force delete virt-launcher pod**: - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find the pod using `pods_list_in_namespace` from diagnostic step 3. - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - ⚠️ **Note**: For force deletion with grace period, use CLI fallback. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') - oc delete pod $POD_NAME -n --force --grace-period=0 - ``` - -5. **Remove VMI finalizers** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VMI using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update VMI using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Patch VM runStrategy directly** (ensure consistency): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` (diagnostic step 1) - 2. Set `.spec.runStrategy` to `"Halted"` - 3. Update VM using `resources_create_or_update` with modified JSON - - **CLI Fallback** (merge patch easier via CLI): - Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Stopped` or `Halted`). - -Verify VMI is gone: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Verify virt-launcher pod is gone: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for pods matching `virt-launcher-`. Should return no results. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Stopped or Halted - -oc get vmi -n -# Should return: Error from server (NotFound) - -oc get pods -n | grep virt-launcher- -# Should return: No resources found -``` - -**Prevention**: -- Ensure guest OS has ACPI support enabled -- Use proper shutdown commands in guest OS -- Avoid forceful stops unless necessary (can corrupt guest filesystem) - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/network-errors.md b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/network-errors.md deleted file mode 100644 index 82fa313c..00000000 --- a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/network-errors.md +++ /dev/null @@ -1,429 +0,0 @@ ---- -title: VM Network Errors -category: kubevirt -sources: - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] -semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] -use_cases: [vm-creation, network-troubleshooting] -related_docs: [INDEX.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Network Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. - -**When to use this document**: -- VM created successfully but secondary networks not attached -- NetworkAttachmentDefinition not found errors -- Multus CNI failures -- SR-IOV device attachment issues - -**Skills that use this**: vm-create - ---- - -### Network Attachment Failures - -**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working - -**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). - -**Possible Causes**: -- NetworkAttachmentDefinition doesn't exist in the namespace -- Multus CNI not installed or not configured on cluster -- Namespace mismatch (NAD in different namespace than VM) -- Interface name conflicts in VM spec -- Bridge/network configuration errors in NAD -- SR-IOV device not available (if using SR-IOV) - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List NetworkAttachmentDefinitions in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "" -} -``` - -Review `.items[].metadata.name` for available NADs. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -n -``` - -**2. List NetworkAttachmentDefinitions in all namespaces**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition" -} -``` - -Omit `namespace` parameter to list across all namespaces. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -A -``` - -**3. Check specific NetworkAttachmentDefinition**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "", - "name": "" -} -``` - -Review `.spec.config` for CNI configuration. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definition -n -o yaml -``` - -**4. Check VM network configuration**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.networks` to see network references. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq -``` - -**5. Check VM domain interfaces**: - -Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq -``` - -**6. Check virt-launcher pod network annotations** (shows actual attachments): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**7. Check Multus is installed**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "openshift-multus" -} -``` - -Should show Multus CNI pods running. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n openshift-multus -``` - -**8. Check for errors in virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n -``` - -**Common Error Messages**: -- `"network-attachment-definition not found"` - NAD doesn't exist in namespace -- `"multus CNI not configured"` - Multus not installed or misconfigured -- `"interface name conflict"` - Duplicate interface names in VM spec -- `"failed to add network"` - CNI plugin error (check NAD config) -- `"no available devices"` - SR-IOV device not available (if using SR-IOV) - -**Solutions** (Use MCP Tools First): - -1. **Verify NetworkAttachmentDefinition exists in correct namespace**: - - Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. - - If NAD is in different namespace, copy it to VM namespace: - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get NAD from source namespace using `resources_get` - 2. Modify `.metadata.namespace` to target namespace - 3. Create NAD in target namespace using `resources_create_or_update` - - **CLI Fallback** (stream processing easier via CLI): - Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" - ```bash - oc get network-attachment-definition -n -o yaml | \ - sed "s/namespace: /namespace: /" | \ - oc apply -f - - ``` - -2. **Create missing NetworkAttachmentDefinition**: - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - Example: Linux bridge network - - **Parameters**: - ```json - { - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "metadata": { - "name": "vlan100", - "namespace": "" - }, - "spec": { - "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" - ```bash - cat < - spec: - config: '{ - "cniVersion": "0.3.1", - "type": "bridge", - "bridge": "br1", - "vlan": 100, - "ipam": { - "type": "host-local", - "subnet": "192.168.100.0/24" - } - }' - EOF - ``` - -3. **Check Multus CNI installation**: - - Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. - - To check cluster network operator: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "config.openshift.io/v1", - "kind": "ClusterOperator" - } - ``` - - Filter for `network` operator. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n openshift-multus - oc get clusteroperators network - ``` - -4. **Fix interface name conflicts** (if VM has duplicate names): - - Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. - - Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. - - **CLI Fallback** (interactive edit easier via CLI): - Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' - oc edit vm -n - ``` - -5. **Validate NAD configuration syntax**: - - Use `resources_get` from diagnostic step 3, extract `.spec.config`. - - Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get network-attachment-definition -n -o jsonpath='{.spec.config}' - ``` - -6. **Check SR-IOV device availability** (if using SR-IOV networks): - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - List SR-IOV network node policies: - - **Parameters**: - ```json - { - "apiVersion": "sriovnetwork.openshift.io/v1", - "kind": "SriovNetworkNodePolicy", - "namespace": "openshift-sriov-network-operator" - } - ``` - - Check SR-IOV device plugin pods: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "openshift-sriov-network-operator" - } - ``` - - Filter for pods with "device-plugin" in name. - - Check available SR-IOV devices on node: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "Node", - "name": "" - } - ``` - - Review `.status.allocatable` for SR-IOV resources. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get sriovnetworknodepolicy -n openshift-sriov-network-operator - oc get pods -n openshift-sriov-network-operator | grep device-plugin - oc describe node | grep -A 10 "Allocatable:" - ``` - -7. **Recreate VM with corrected network configuration** (if needed): - - If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -After remediation, check virt-launcher pod network status: - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -Should show all attached networks with status. Example output: -```json -[ - { - "name": "openshift-sdn", - "interface": "eth0", - "ips": ["10.128.2.10"], - "default": true - }, - { - "name": "vlan100", - "interface": "net1", - "ips": ["192.168.100.5"] - } -] -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**Check from inside VM** (via console): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# In guest OS: -ip addr show -# Should show all network interfaces (eth0, net1, etc.) -``` - -**Common Network Types**: -- **Linux Bridge**: Layer 2 bridge for VLAN networks -- **SR-IOV**: High-performance direct device assignment -- **macvlan**: MAC-based VLAN for container networks -- **OVN-Kubernetes**: OpenShift native overlay network - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/runtime-errors.md b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/runtime-errors.md deleted file mode 100644 index 804d3cbc..00000000 --- a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/runtime-errors.md +++ /dev/null @@ -1,616 +0,0 @@ ---- -title: VM Runtime Errors -category: kubevirt -sources: - - title: Kubernetes Pod Lifecycle - url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ - date_accessed: 2026-02-17 -tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] -semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] -use_cases: [vm-creation, vm-lifecycle, diagnostics] -related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Runtime Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. - -**When to use this document**: -- VM or virt-launcher pod shows `CrashLoopBackOff` status -- virt-launcher pod repeatedly restarting -- Guest OS kernel panics on boot - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### CrashLoopBackOff - -**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting - -**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. - -**Possible Causes**: -- Guest OS kernel panic on boot -- Insufficient resources (memory/CPU) for guest OS -- Corrupted disk image or filesystem -- QEMU/libvirt crashes due to configuration errors -- Missing or incompatible device drivers in guest -- Resource limits too low for virt-launcher pod -- Virtualization features (KVM) not available on node - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check virt-launcher pod restart count**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# Look at RESTARTS column -``` - -**2. View recent crash logs** (previous container instance): - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "previous": true, - "tail": 100 -} -``` - -Look for QEMU errors, kernel panics, or segfaults. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n virt-launcher--xxx --previous -``` - -**3. Check current virt-launcher logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. - -**CLI Fallback** (if MCP unavailable or all containers needed): -```bash -oc logs -n virt-launcher--xxx --all-containers -``` - -**4. Check VMI conditions for crash details**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Extract `.status.conditions` for crash details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**5. Check pod events for crash reasons**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" -``` - -**6. Check pod resource limits**: - -**MCP Tool**: `pods_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -Extract `.spec.containers[0].resources` for resource limits. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq -``` - -**7. Check node kubelet logs for OOM kills**: - -⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -oc adm node-logs -u kubelet | grep -i oom -``` - -**8. Access guest console** (if VM briefly starts): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Crash Patterns**: - -1. **Guest Kernel Panic**: - - Console logs show kernel panic messages - - Guest crashes immediately after boot - - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" - -2. **OOM (Out of Memory)**: - - Pod killed with reason: `OOMKilled` - - Guest runs out of memory during boot or operation - - virt-launcher logs show memory allocation failures - -3. **QEMU Crash**: - - virt-launcher logs show QEMU segmentation fault - - Symptoms: "qemu-system-x86_64: terminated by signal" - - Configuration incompatibility or QEMU bug - -4. **Disk Image Corruption**: - - Guest cannot boot from disk - - Filesystem errors in guest console - - DataVolume import failed - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for kernel panic or boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for: - - Kernel panic messages - - Initramfs errors - - Filesystem mounting failures - - Missing device errors - -2. **Review virt-launcher crash logs**: - - Use `pods_log` with `previous: true` from diagnostic step 2. - - Look for: - - QEMU command line errors - - Device initialization failures - - Memory allocation errors - - Signal termination (SIGSEGV, SIGABRT) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx --previous - ``` - -3. **Check for OOM (Out of Memory) kills**: - - **MCP Tool**: `pods_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - Extract `.status.containerStatuses[0].lastState.terminated.reason`. - - If returns `"OOMKilled"`: - - Option 1: Increase virt-launcher memory limits - - Option 2: Decrease guest memory allocation - - Option 3: Use smaller instance type - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' - ``` - -4. **Increase resources if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check current memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -5. **Verify disk image integrity**: - - **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters for list**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" - } - ``` - - **Parameters for specific DV**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check `.status.phase` (should be `Succeeded`). - - If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get datavolume -n -o jsonpath='{.status.phase}' - ``` - -6. **Check virtualization (KVM) availability**: - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -7. **Simplify VM configuration** (eliminate variables): - - Try creating minimal VM using vm-create skill with: - - Small instance type - - No secondary networks - - Simple container disk (e.g., Fedora) - - No cloud-init - - If minimal VM works, add features back one by one. - -8. **Recreate VM with different workload** (test disk image): - - If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) - -After remediation, check pod restart count stops increasing: - -**Parameters for pods**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). - -Check VM reaches Running state: - -**Parameters for VM**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# RESTARTS should stabilize - -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Running -``` - -Verify guest is responsive: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# Should show login prompt or OS console -``` - -**Advanced Debugging**: - -**MCP Tool**: `pods_exec` (from openshift-virtualization) - -Check libvirt domain XML: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["virsh", "dumpxml", "1"] -} -``` - -Check QEMU process: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["ps", "aux"] -} -``` - -Filter output for "qemu" process. - -**CLI Fallback** (if MCP unavailable): -```bash -oc exec -n virt-launcher--xxx -- virsh dumpxml 1 -oc exec -n virt-launcher--xxx -- ps aux | grep qemu -``` - -**Prevention**: -- Start with minimal VM configuration and add complexity gradually -- Use recommended instance types for your workload -- Test disk images before deploying to production -- Ensure nodes have adequate resources and KVM support -- Monitor resource usage and set appropriate limits - ---- - -## Workaround Patterns for MCP Tool Limitations - -### General Pattern: Diagnose → Propose → Confirm → Execute - -When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): - -**1. Diagnose** the root cause using Kubernetes commands -```bash -oc describe vm -n -oc get events -n --field-selector involvedObject.name= -``` - -**2. Propose** a manual workaround to the user with clear explanation -- Explain why the issue occurred -- Show what will be changed -- List alternative options - -**3. Confirm** with user before executing (Human-in-the-Loop) -- Wait for explicit "yes" or "apply workaround" -- Never auto-execute modifications - -**4. Execute** the workaround using oc/kubectl -```bash -oc patch vm -n --type=merge -p '...' -``` - -**5. Verify** the fix was successful -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**6. Document** the limitation and suggest filing enhancement request -- Note this is temporary until MCP tool is enhanced -- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues - -### Example: Adding Tolerations Workaround - -**Diagnostic Output**: -```markdown -## ⚠️ VM Scheduling Issue Detected - -**Root Cause**: Node taints prevent VM scheduling - -**Details**: -- Found 3 nodes with taint: `virtualization=true:NoSchedule` -- VM spec does not include matching tolerations -- This prevents VM scheduling on virtualization-dedicated nodes -``` - -**Proposed Workaround**: -```bash -oc patch vm web-server -n vms --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' -``` - -**User Confirmation Required**: -``` -How would you like to proceed? -- "apply workaround" - I'll patch the VM with tolerations -- "manual" - I'll provide instructions for you to apply manually -- "cancel" - Delete the VM and abort creation -``` - -**After Execution**: -```markdown -## ✓ Workaround Applied Successfully - -**Action**: Added tolerations for taint `virtualization=true:NoSchedule` -**New Status**: Stopped (VM can now be scheduled) - -**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. -Future VMs in this cluster will need the same fix until the tool is enhanced. -``` - ---- - -## VM Status Reference - -### Status Values - -| Status | Meaning | Action Required | -|--------|---------|-----------------| -| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | -| `Running` | VM is running | Normal | -| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | -| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | -| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | -| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | -| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | -| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | -| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | -| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | - -### Checking VM Status - -```bash -# Get printable status -oc get vm -n -o jsonpath='{.status.printableStatus}' - -# Get detailed status and conditions -oc get vm -n -o jsonpath='{.status}' | jq - -# Watch status changes in real-time -oc get vm -n -w -``` - ---- - -## Best Practices for Agents - -When implementing diagnostic workflows: - -1. **Always verify VM status** after creation (wait 5-10 seconds first) -2. **Consult this document** when encountering error status values -3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) -4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) -5. **Respect human-in-the-loop** for all VM modifications -6. **Document temporary workarounds** and their limitations clearly -7. **Suggest filing issues** for missing MCP tool features - -### Document Consultation Pattern - -```markdown -**Document Consultation** (REQUIRED): -1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes -2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." -``` - ---- - -## Known MCP Tool Limitations - -### vm_create tool - -**Currently Supported**: -- ✓ Namespace, name (required) -- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) -- ✓ Size hints (small, medium, large) -- ✓ Storage size -- ✓ Autostart flag -- ✓ Networks (Multus NetworkAttachmentDefinitions) -- ✓ Performance family (u1, o1, c1, m1) -- ✓ Instance type, preference - -**Not Currently Supported** (requires workarounds): -- ✗ Tolerations (for node taints) -- ✗ Node selectors -- ✗ Affinity/anti-affinity rules -- ✗ Resource requests/limits (beyond instance type) -- ✗ Custom labels/annotations -- ✗ SSH keys injection -- ✗ Cloud-init user data - -**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. - -**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues - ---- - -## Additional Resources - -- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) -- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) -- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) -- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) - ---- - -[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/scheduling-errors.md deleted file mode 100644 index e47c8c4f..00000000 --- a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/scheduling-errors.md +++ /dev/null @@ -1,417 +0,0 @@ ---- -title: VM Scheduling Errors -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 -tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] -semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] -use_cases: [vm-creation, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Scheduling Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**When to use this document**: -- VM shows status `ErrorUnschedulable` after creation or start attempt -- VM events mention scheduling failures, taints, resources, or node selectors - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -## ErrorUnschedulable - -**Symptom**: VM shows status `ErrorUnschedulable` after creation - -**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**Possible Causes**: - -### 1. Node Taints (Most Common) - -Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM events for scheduling failures**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `involvedObject.name` == "" and look for messages like: -- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= -``` - -**2. Check node taints in the cluster**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' -``` - -**Common Taint Patterns**: -- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule -- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes -- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads - -**Solution - Add Tolerations to VM**: - -The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. - -**Workaround (post-creation using MCP Tools)**: - -**Step 1**: Get current VM spec - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Step 2**: Modify the returned JSON to add tolerations - -Add to `.spec.template.spec.tolerations`: -```json -{ - "tolerations": [ - { - "key": "virtualization", - "operator": "Equal", - "value": "true", - "effect": "NoSchedule" - } - ] -} -``` - -**Step 3**: Update VM with modified spec - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters**: -```json -{ - "resource": "" -} -``` - -Pass the complete modified VM resource as YAML or JSON string. - -**Step 4**: Verify tolerations were added - -Use `resources_get` again and check `.spec.template.spec.tolerations` in response. - -**Step 5**: Check if VM status improved - -Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. - -**CLI Fallback** (if MCP patch is too complex): -```bash -# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' - -# Verify tolerations -oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq - -# Check status -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**Example - Multiple Tolerations**: -```bash -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" - - key: "dedicated" - operator: "Equal" - value: "virt-workloads" - effect: "NoSchedule" -' -``` - -**Toleration Operators**: -- `Equal` - Key and value must match exactly -- `Exists` - Only key must exist (ignores value) - -**Toleration Effects**: -- `NoSchedule` - Don't schedule new pods (existing pods continue) -- `PreferNoSchedule` - Avoid scheduling if possible -- `NoExecute` - Don't schedule AND evict existing pods - -**Alternative Solutions**: -1. **Remove node taints** (if you have cluster-admin access): - ```bash - oc adm taint nodes virtualization=true:NoSchedule- - ``` - -2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits - -3. **File enhancement request** - Request tolerations support in openshift-mcp-server: - https://github.com/openshift/openshift-mcp-server/issues - ---- - -### 2. Insufficient Resources - -Not enough CPU, memory, or storage available on any node. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM resource requests**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' -``` - -**2. Check node resource availability**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. - -Alternatively, use `nodes_top` MCP tool for current resource usage. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe nodes | grep -A 5 "Allocated resources" -``` - -**3. Look for VM events mentioning "Insufficient"**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep "Insufficient" -``` - -**Example Event**: -``` -0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. -``` - -**Solutions** (Use MCP Tools First): - -1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) -2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill -3. **Delete unused VMs** - Use vm-delete skill to free up resources -4. **Check resource quotas**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters for quota**: - ```json - { - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" - } - ``` - - **Parameters for limit range**: - ```json - { - "apiVersion": "v1", - "kind": "LimitRange", - "namespace": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - oc describe limitrange -n - ``` - ---- - -### 3. Node Selector Mismatch - -VM requires specific node labels that don't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM node selector requirements**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.nodeSelector` to see required node labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' -``` - -**2. List available node labels**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.metadata.labels` for available labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes --show-labels -``` - -**3. Check if any nodes match the selector**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node", - "labelSelector": "=" -} -``` - -Should return at least one node with matching labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -l = -``` - -**Solutions** (Use MCP Tools First): - -**Option 1: Remove node selector from VM** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get current VM using `resources_get` (diagnostic step 1) -2. Remove `.spec.template.spec.nodeSelector` field -3. Update VM using `resources_create_or_update` with modified JSON - -**CLI Fallback** (JSON patch easier via CLI): -Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" -```bash -oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' -``` - -**Option 2: Add label to nodes** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get node using `resources_get` -2. Add label to `.metadata.labels` -3. Update node using `resources_create_or_update` - -⚠️ **Note**: Node labeling typically requires cluster admin privileges. - -**CLI Fallback** (simpler via CLI): -Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" -```bash -oc label node = -``` - ---- - -[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/storage-errors.md b/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/storage-errors.md deleted file mode 100644 index 16ba584d..00000000 --- a/evaluation/without_skills/rh-virt__vm-create/environment/docs/troubleshooting/storage-errors.md +++ /dev/null @@ -1,1011 +0,0 @@ ---- -title: VM Storage Errors -category: kubevirt -sources: - - title: KubeVirt CDI - DataVolumes - url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ - date_accessed: 2026-02-17 - - title: Kubernetes Persistent Volumes - url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ - date_accessed: 2026-02-17 -tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] -semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] -use_cases: [vm-creation, vm-deletion, vm-cloning] -related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] -last_updated: 2026-02-17 ---- - -# VM Storage Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. - -**When to use this document**: -- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` -- Storage deletion fails after VM deletion -- DataVolume cloning operations fail -- PVC provisioning issues - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### ErrorDataVolumeNotReady - -**Symptom**: VM shows status `ErrorDataVolumeNotReady` - -**Description**: The DataVolume (persistent storage) backing the VM is not ready. - -**Possible Causes**: - -#### 1. DataVolume Still Provisioning - -Storage provisioning takes time, especially for large disks or when importing images. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Get detailed DataVolume information**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` and `.status.conditions` for provisioning details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o json -``` - -**3. Check PVC (PersistentVolumeClaim) bound status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Check `.status.phase` for each PVC (should be `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. - -#### 2. Storage Class Not Found - -The requested storage class doesn't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List available storage classes**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" -} -``` - -Review the list of available storage classes (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -``` - -**2. Check DataVolume's requested storage class**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.spec.pvc.storageClassName` in the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' -``` - -**Solution**: -1. Use a valid storage class from the cluster -2. Recreate VM with correct storage class parameter - -#### 3. Insufficient Storage Quota - -Namespace has insufficient storage quota to provision the PVC. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check resource quotas**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**2. Check storage usage**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase -``` - -**Solution**: -1. Request quota increase from cluster admin -2. Delete unused PVCs to free quota -3. Reduce VM storage size - ---- - -### ErrorPvcNotFound - -**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List PVCs in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review the list of available PVCs (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check VM's PVC references**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' -``` - -**Solution**: -- Wait for DataVolume to create the PVC -- Manually create missing PVC -- Fix VM spec to reference correct PVC name - ---- - - ---- - -### Storage Deletion Failures - -**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace - -**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. - -**Possible Causes**: -- PVC still bound to active PersistentVolume with `Retain` policy -- DataVolume still being referenced by another resource -- CDI (Containerized Data Importer) controller issues -- Storage class retention policy preventing deletion -- Finalizers on PVC/DataVolume blocking cleanup -- PVC still mounted by a pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check PVC status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check specific PVC phase**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Released` or `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o jsonpath='{.status.phase}' -``` - -**3. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**4. Check what's using the PVC**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' -``` - -**5. Check PVC finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. - -**6. Check DataVolume finalizers**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.metadata.finalizers}' -``` - -**7. Check PV reclaim policy**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume" -} -``` - -Filter results for PV where `.spec.claimRef.name` matches ``. - -To get specific PV policy: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume", - "name": "" -} -``` - -Check `.spec.persistentVolumeReclaimPolicy`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pv | grep -oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' -``` - -**Common Finalizer Patterns**: -- `kubernetes.io/pvc-protection` - Protects PVC while in use -- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer - -**Solutions** (Use MCP Tools First): - -1. **Delete DataVolume first, then PVC**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete DataVolume first (often blocks PVC deletion): - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Wait a few seconds, then delete PVC: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete datavolume -n - oc delete pvc -n - ``` - -2. **Check for pods still using PVC**: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "" - } - ``` - - Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. - - Then delete the pods using `pods_delete`: - - **Parameters**: - ```json - { - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' - oc delete pod -n - ``` - -3. **Force delete PVC** (if safe to do so): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete pvc -n --grace-period=0 --force - ``` - -4. **Remove finalizers from PVC** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PVC using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update PVC using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Remove finalizers from DataVolume** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current DataVolume using `resources_get` (diagnostic step 6) - 2. Remove items from `.metadata.finalizers` array - 3. Update DataVolume using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Change PV reclaim policy** (if PV has Retain policy): - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PV policy using `resources_get` (diagnostic step 7) - 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` - 3. Update PV using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" - ```bash - oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' - oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' - ``` - -**Storage Quota Check** (Use MCP Tools First): - -After deletion, verify storage quota is freed: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters for quota check**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.used` to verify storage quota is freed. - -**Parameters for PVC verification**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -oc get pvc -n -``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Confirm PVC is deleted: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Confirm DataVolume is deleted: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -# Should return: Error from server (NotFound) - -oc get datavolume -n -# Should return: Error from server (NotFound) -``` - ---- - -### DataVolume Cloning Failures - -**Symptom**: VM clone created successfully but DataVolume clone operation fails - -**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. - -**Possible Causes**: -- CSI driver doesn't support volume cloning -- Source PVC storage class incompatible with cloning -- Cross-namespace cloning not permitted by storage backend -- Insufficient storage quota in target namespace -- Source PVC not in `Bound` state -- Storage class doesn't have volume cloning enabled -- CDI (Containerized Data Importer) controller issues - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume clone status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and `.metadata.name`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Check DataVolume events for errors**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe datavolume -n -``` - -**3. Check DataVolume phase**: - -Use `resources_get` from step 1, extract `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -``` - -**4. Check if storage class supports cloning**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass", - "name": "" -} -``` - -Review the full YAML output for cloning-related configurations. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -o yaml | grep -A 5 -i clone -``` - -**5. Check CSI driver capabilities**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "CSIDriver" -} -``` - -Review `.items[].metadata.name` for available CSI drivers. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get csidriver -``` - -**6. Check source PVC status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Bound` for cloning to work). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**7. Check target namespace storage quota**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**8. Check CDI controller logs**: - -**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) - -First, list pods in openshift-cnv namespace: - -**Parameters for pods_list_in_namespace**: -```json -{ - "namespace": "openshift-cnv", - "labelSelector": "app.kubernetes.io/component=cdi-deployment" -} -``` - -Then get logs using `pods_log`: - -**Parameters**: -```json -{ - "namespace": "openshift-cnv", - "name": "", - "tail": 100 -} -``` - -**CLI Fallback** (if MCP unavailable or easier via CLI): -```bash -oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') -``` - -**Common Error Messages**: -- `"volume cloning is not supported"` - CSI driver lacks clone capability -- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage -- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace -- `"insufficient quota"` - Target namespace lacks storage quota -- `"source PVC not bound"` - Source PVC must be in Bound state for cloning -- `"StorageClass does not support cloning"` - Storage class configuration issue - -**Solutions** (Use MCP Tools First): - -1. **Check storage class clone support**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" - } - ``` - - For each storage class in `.items[]`, check: - - `.metadata.name` (storage class name) - - `.provisioner` (CSI driver) - - Storage classes using CSI drivers typically support cloning. Look for provisioners like: - - `csi.ovirt.org` (oVirt CSI) - - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) - - `ebs.csi.aws.com` (AWS EBS CSI) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner - ``` - -2. **Verify source PVC is bound**: - - Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pvc -n -o jsonpath='{.status.phase}' - ``` - -3. **Check target namespace quota**: - - Use `resources_list` from diagnostic step 7 to check quota. - - If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - ``` - -4. **Use snapshot-based cloning** (alternative method): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Step 1**: Create VolumeSnapshot of source PVC - - **Parameters**: - ```json - { - "apiVersion": "snapshot.storage.k8s.io/v1", - "kind": "VolumeSnapshot", - "metadata": { - "name": "-snapshot", - "namespace": "" - }, - "spec": { - "source": { - "persistentVolumeClaimName": "" - } - } - } - ``` - - **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) - - **Step 3**: Create new DataVolume from snapshot - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "metadata": { - "name": "-rootdisk", - "namespace": "" - }, - "spec": { - "source": { - "snapshot": { - "name": "-snapshot", - "namespace": "" - } - }, - "storage": { - "resources": { - "requests": { - "storage": "50Gi" - } - }, - "storageClassName": "" - } - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" - ```bash - cat <-snapshot - namespace: - spec: - source: - persistentVolumeClaimName: - EOF - - oc get volumesnapshot -snapshot -n - - cat <-rootdisk - namespace: - spec: - source: - snapshot: - name: -snapshot - namespace: - storage: - resources: - requests: - storage: 50Gi - storageClassName: - EOF - ``` - -5. **Use "new empty storage" option** (vm-clone skill): - - If cloning isn't supported, create VM with empty storage - - Manually copy data if needed - -6. **Cross-namespace cloning workaround**: - - Some storage backends require snapshot for cross-namespace cloning - - Create snapshot in source namespace, restore in target namespace (see solution 4 above) - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Check DataVolume reached Succeeded phase: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should return `Succeeded`). - -Check PVC was created and bound: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "-rootdisk" -} -``` - -Check `.status.phase` (should return `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -# Should return: Succeeded - -oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' -# Should return: Bound -``` - -**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/.ai-index/semantic-index.json b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/.ai-index/semantic-index.json deleted file mode 100644 index 2512c2a3..00000000 --- a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/.ai-index/semantic-index.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "index_version": "1.0.0", - "last_updated": "2026-02-17", - "description": "Semantic index for VM troubleshooting documentation - maps skills and error types to relevant documentation files", - - "documents": [ - { - "file": "INDEX.md", - "title": "VM Troubleshooting Guide - Index", - "category": "navigation", - "keywords": ["index", "navigation", "overview", "MCP tools", "quick reference"], - "use_cases": ["navigation", "tool-reference", "general-guidance"], - "token_estimate": 450, - "skills": ["all"] - }, - { - "file": "scheduling-errors.md", - "title": "VM Scheduling Errors", - "category": "scheduling", - "keywords": ["ErrorUnschedulable", "node taints", "tolerations", "insufficient resources", "node selector", "scheduling failure"], - "use_cases": ["vm-creation", "vm-lifecycle", "scheduling-diagnosis"], - "errors_covered": [ - "ErrorUnschedulable - Node Taints", - "ErrorUnschedulable - Insufficient Resources", - "ErrorUnschedulable - Node Selector Mismatch" - ], - "token_estimate": 820, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "storage-errors.md", - "title": "VM Storage Errors", - "category": "storage", - "keywords": ["ErrorDataVolumeNotReady", "ErrorPvcNotFound", "storage deletion", "PVC", "DataVolume", "cloning", "CDI", "storage class", "storage provisioning"], - "use_cases": ["vm-creation", "vm-deletion", "vm-cloning", "storage-diagnosis"], - "errors_covered": [ - "ErrorDataVolumeNotReady - DataVolume Still Provisioning", - "ErrorDataVolumeNotReady - Storage Class Not Found", - "ErrorDataVolumeNotReady - Insufficient Storage Quota", - "ErrorPvcNotFound", - "Storage Deletion Failures", - "DataVolume Cloning Failures" - ], - "token_estimate": 1350, - "skills": ["vm-creator", "vm-delete", "vm-clone"] - }, - { - "file": "lifecycle-errors.md", - "title": "VM Lifecycle Errors", - "category": "lifecycle", - "keywords": ["terminating", "finalizers", "start failure", "stop failure", "VMI", "virt-launcher", "runStrategy"], - "use_cases": ["vm-deletion", "vm-lifecycle", "start-stop-issues"], - "errors_covered": [ - "VM Stuck in Terminating State", - "VM Won't Start (Non-Scheduling Issues)", - "VM Won't Stop" - ], - "token_estimate": 1150, - "skills": ["vm-delete", "vm-lifecycle-manager"] - }, - { - "file": "runtime-errors.md", - "title": "VM Runtime Errors", - "category": "runtime", - "keywords": ["CrashLoopBackOff", "pod crashes", "kernel panic", "QEMU crash", "OOM", "guest OS failure", "virt-launcher restart"], - "use_cases": ["vm-creation", "vm-lifecycle", "runtime-diagnosis", "crash-diagnosis"], - "errors_covered": [ - "CrashLoopBackOff" - ], - "token_estimate": 850, - "skills": ["vm-creator", "vm-lifecycle-manager"] - }, - { - "file": "network-errors.md", - "title": "VM Network Errors", - "category": "networking", - "keywords": ["network attachment", "Multus", "NetworkAttachmentDefinition", "NAD", "SR-IOV", "secondary networks", "CNI"], - "use_cases": ["vm-creation", "network-troubleshooting"], - "errors_covered": [ - "Network Attachment Failures" - ], - "token_estimate": 580, - "skills": ["vm-creator"] - } - ], - - "skill_to_docs_mapping": { - "vm-creator": { - "primary_docs": ["scheduling-errors.md", "storage-errors.md"], - "secondary_docs": ["runtime-errors.md", "network-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3650, - "description": "VM creation failures - scheduling, storage provisioning, crashes, networks" - }, - "vm-delete": { - "primary_docs": ["lifecycle-errors.md", "storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 2950, - "description": "VM deletion failures - stuck terminating, storage cleanup" - }, - "vm-clone": { - "primary_docs": ["storage-errors.md"], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 1800, - "description": "VM cloning failures - DataVolume cloning, storage provisioning" - }, - "vm-lifecycle-manager": { - "primary_docs": ["lifecycle-errors.md", "scheduling-errors.md"], - "secondary_docs": ["runtime-errors.md"], - "index": "INDEX.md", - "total_token_estimate": 3270, - "description": "VM start/stop failures - lifecycle management, scheduling" - }, - "vm-inventory": { - "primary_docs": [], - "secondary_docs": [], - "index": "INDEX.md", - "total_token_estimate": 450, - "description": "General guidance for status interpretation" - } - }, - - "error_to_docs_mapping": { - "ErrorUnschedulable": "scheduling-errors.md", - "ErrorDataVolumeNotReady": "storage-errors.md", - "ErrorPvcNotFound": "storage-errors.md", - "Terminating": "lifecycle-errors.md", - "CrashLoopBackOff": "runtime-errors.md", - "NetworkAttachmentFailed": "network-errors.md" - }, - - "cross_references": { - "scheduling-errors.md": ["storage-errors.md", "runtime-errors.md"], - "storage-errors.md": ["scheduling-errors.md", "lifecycle-errors.md"], - "lifecycle-errors.md": ["storage-errors.md", "runtime-errors.md"], - "runtime-errors.md": ["lifecycle-errors.md", "scheduling-errors.md"], - "network-errors.md": ["scheduling-errors.md"] - }, - - "progressive_disclosure_strategy": { - "description": "Load only the minimum documentation needed for current task", - "pattern": "error_type → primary_doc → cross_references (if needed)", - "token_savings": "29% average reduction vs single-file (based on rh-sre data)", - "navigation_overhead_reduction": "85% vs single-file navigation" - } -} diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/INDEX.md b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/INDEX.md deleted file mode 100644 index a2e76963..00000000 --- a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/INDEX.md +++ /dev/null @@ -1,332 +0,0 @@ ---- -title: VM Troubleshooting Guide - Index -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 - - title: OpenShift Virtualization - Virtual Machine Status - url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html - date_accessed: 2026-02-06 - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] -semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] -use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] -last_updated: 2026-02-17 ---- - -# VM Troubleshooting Guide - Index - -## Overview - -This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: - -- **Creation**: Scheduling failures, storage provisioning issues -- **Lifecycle**: Start/stop failures, stuck states -- **Deletion**: Resources stuck in Terminating, storage cleanup failures -- **Cloning**: DataVolume cloning errors, cross-namespace issues -- **Networking**: Secondary network attachment failures -- **Runtime**: CrashLoopBackOff, guest OS boot failures - -This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. - ---- - -## 🗂️ Troubleshooting Categories - -The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: - -### 1. [Scheduling Errors](scheduling-errors.md) -**When to use**: VM fails to schedule on any node - -**Errors covered**: -- ErrorUnschedulable - Node Taints -- ErrorUnschedulable - Insufficient Resources -- ErrorUnschedulable - Node Selector Mismatch - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 2. [Storage Errors](storage-errors.md) -**When to use**: VM has storage provisioning, deletion, or cloning issues - -**Errors covered**: -- ErrorDataVolumeNotReady (all 3 subsections) -- ErrorPvcNotFound -- Storage Deletion Failures -- DataVolume Cloning Failures - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### 3. [Lifecycle Errors](lifecycle-errors.md) -**When to use**: VM has start/stop/termination issues - -**Errors covered**: -- VM Stuck in Terminating State -- VM Won't Start (Non-Scheduling Issues) -- VM Won't Stop - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### 4. [Runtime Errors](runtime-errors.md) -**When to use**: VM repeatedly crashes or fails at runtime - -**Errors covered**: -- CrashLoopBackOff - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### 5. [Network Errors](network-errors.md) -**When to use**: VM has secondary network attachment failures - -**Errors covered**: -- Network Attachment Failures (Multus, SR-IOV) - -**Skills that use this**: vm-create - ---- - -## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last - -### Critical Principle: MCP-First Approach - -**All diagnostic procedures in this guide follow the MCP-first pattern**: - -``` -1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) -2. ⚠️ IF FAILS: Ask user permission to use CLI command -3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval -``` - -### Why MCP Tools First? - -- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources -- **Consistency**: Same interface across all operations -- **Better Error Handling**: MCP tools return structured errors -- **Audit Trail**: MCP tool usage is logged and trackable - -### Command Pattern Examples - -Throughout this guide, you'll see diagnostic steps formatted like this: - -**✅ CORRECT Pattern (MCP First)**: - -```markdown -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Extract**: `.metadata.finalizers` from returned JSON - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` -``` - -### Available MCP Tools - -The `openshift-virtualization` MCP server provides these tools: - -**Resource Operations**: -- `resources_get` - Get specific resource (replaces `oc get `) -- `resources_list` - List resources (replaces `oc get `) -- `resources_delete` - Delete resource (replaces `oc delete`) -- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) - -**Pod Operations**: -- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) -- `pods_get` - Get pod details (replaces `oc get pod`) -- `pods_log` - Get pod logs (replaces `oc logs`) -- `pods_exec` - Execute in pod (replaces `oc exec`) -- `pods_delete` - Delete pod (replaces `oc delete pod`) -- `pods_top` - Pod resource usage (replaces `oc top pods`) - -**Events & Monitoring**: -- `events_list` - List events (replaces `oc get events`) -- `nodes_top` - Node resource usage (replaces `oc top nodes`) -- `nodes_log` - Node logs (replaces `oc adm node-logs`) -- `nodes_stats_summary` - Detailed node stats - -**VM Operations** (KubeVirt toolset): -- `vm_create` - Create VMs -- `vm_lifecycle` - Start/stop/restart VMs - -**Namespaces**: -- `namespaces_list` - List namespaces (replaces `oc get namespaces`) -- `projects_list` - List OpenShift projects (replaces `oc get projects`) - -### When CLI Commands Are Required - -Some operations have **NO MCP equivalent** and require CLI: - -- `virtctl` commands (console, VNC access) -- `oc debug node` (node debugging) -- `oc auth can-i` (permission checks) -- `oc adm taint` (node taint management) -- Complex JSON patch operations - -For these, the guide will note: **"CLI Only - No MCP equivalent"** - -### Quick Reference: CLI → MCP Mapping - -| CLI Command | MCP Tool Equivalent | -|-------------|---------------------| -| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | -| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | -| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | -| `oc logs -n ` | `pods_log` with name="", namespace="" | -| `oc get events -n ` | `events_list` with namespace="" | -| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | - -**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. - -### How to Read Diagnostic Sections - -Each error section includes: -1. **Symptom** - What you observe -2. **Description** - What's happening -3. **Possible Causes** - Why it's happening -4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback -5. **Solutions** - **MCP tools first**, then CLI fallback -6. **Verification** - **MCP tools first**, then CLI fallback - -**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. - ---- - -## 🔍 Quick Navigation by Skill - -**vm-create**: -- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics -- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning -- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff -- [Network Errors](network-errors.md) - Network attachment failures - -**vm-delete**: -- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state -- [Storage Errors](storage-errors.md) - Storage deletion failures - -**vm-clone**: -- [Storage Errors](storage-errors.md) - DataVolume cloning failures - -**vm-lifecycle-manager**: -- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop -- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling - -**vm-inventory**: -- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed - ---- - -## 📊 Documentation Coverage & Maintenance - -### Current Coverage - -This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: - -- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) -- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures -- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop -- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) -- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems - -**Total errors documented**: 12 error types across 6 categories - ---- - -### Encountering Undocumented Errors - -#### For AI Agents (Claude Code) - -If you encounter an error **not documented** in the categories above: - -1. **Report to user** with all available details (error message, affected resources, namespace) -2. **Provide best-effort diagnostics** using MCP tools: - - `resources_get` to inspect resource status - - `pods_log` to check virt-launcher or compute container logs - - `events_list` to view Kubernetes events -3. **Suggest documentation update**: - ``` - ⚠️ This error is not yet documented in the troubleshooting guide. - - **Error**: - **Affected resource**: / - - I recommend adding this error to the troubleshooting guide: - - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file - - If it's a new error class → Create new category file in docs/troubleshooting/ - - Would you like me to help document this error for future reference? - ``` - -4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs - ---- - -#### For Human Contributors - -To document a new error: - -1. **Determine the category**: - - Scheduling issue → `scheduling-errors.md` - - Storage problem → `storage-errors.md` - - Lifecycle/start/stop → `lifecycle-errors.md` - - Runtime crash → `runtime-errors.md` - - Network issue → `network-errors.md` - - New error class → Create new category file (e.g., `security-errors.md`) - -2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): - - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback - - **Solutions**: MCP Tool approach → Implementation → CLI Fallback - - **Verification**: MCP Tool checks → Expected results → CLI Fallback - -3. **Update semantic index**: - - Edit `.ai-index/semantic-index.json` - - Add error to `error_to_docs_mapping` - - Update relevant skill's `primary_docs` or `secondary_docs` - - Increment token estimates if needed - -4. **Reference in skill files**: - - Update skill's Reference Documentation section - - Ensure skill's Document Consultation steps point to the right category - -5. **Update this INDEX.md**: - - Add error to the appropriate category section (lines 50-120) - - Update "Current Coverage" list above - ---- - -## 📚 Additional Resources - -- [rh-virt README](../../README.md) - MCP server setup and skill documentation -- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources -- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/lifecycle-errors.md b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/lifecycle-errors.md deleted file mode 100644 index 6167fcc8..00000000 --- a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/lifecycle-errors.md +++ /dev/null @@ -1,869 +0,0 @@ ---- -title: VM Lifecycle Errors -category: kubevirt -sources: - - title: Kubernetes Finalizers - url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ - date_accessed: 2026-02-17 - - title: KubeVirt Virtual Machine Status Conditions - url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ - date_accessed: 2026-02-17 -tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] -semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] -use_cases: [vm-deletion, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Lifecycle Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM lifecycle issues including start/stop failures and stuck termination states. - -**When to use this document**: -- VM stuck in `Terminating` state -- VM won't start (runStrategy is Always but VM never reaches Running) -- VM won't stop (runStrategy is Halted but VM never reaches Stopped) - -**Skills that use this**: vm-delete, vm-lifecycle-manager - ---- - -### VM Stuck in Terminating State - -**Symptom**: VM shows status `Terminating` but deletion never completes - -**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). - -**Possible Causes**: -- Finalizers blocking deletion -- PVC/DataVolume still attached and preventing cleanup -- VirtualMachineInstance (VMI) not terminating properly -- Custom controllers or operators blocking deletion -- Stuck virt-launcher pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check finalizers on the VM**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" -```bash -oc get vm -n -o jsonpath='{.metadata.finalizers}' -``` - -**2. Check if VMI still exists**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name containing "virt-launcher-". - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check events for deletion issues**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results where `involvedObject.name` == "" and sort by timestamp. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' -``` - -**5. Check VM deletion timestamp**: - -Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. - -If present, VM is in deletion process. If null, VM is not being deleted. - -**Common Finalizer Patterns**: -- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer -- `foregroundDeletion` - Waits for dependent objects to be deleted -- Custom finalizers from operators - -**Solutions** (Use MCP Tools First): - -1. **Wait for dependent resources** (safest approach): - - Use `resources_list` to check if PVCs, DataVolumes are still deleting - - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) - -2. **Force delete VMI** (if VMI is stuck): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable - requires explicit user permission): - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -3. **Force delete virt-launcher pod** (if pod is stuck): - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - # Find the virt-launcher pod name first - oc get pods -n | grep virt-launcher- - - # Then delete it - oc delete pod virt-launcher--xxx -n --force --grace-period=0 - ``` - -4. **Remove finalizers** (⚠️ dangerous - use only as last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` - 2. Remove items from `.metadata.finalizers` array - 3. Update VM using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch not easily done via MCP): - ```bash - # This operation is complex for MCP - may need CLI - oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Check for protection labels** (vm-delete skill specific): - - Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. - - If value is "true", the vm-delete skill refuses deletion (this is expected behavior). - -**Verification**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error if deletion successful. - -**CLI Fallback**: -```bash -oc get vm -n -# Should return: Error from server (NotFound) -``` - ---- - - ---- - -### VM Won't Start (Non-Scheduling Issues) - -**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state - -**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. - -**Possible Causes**: -- Guest OS kernel panic or boot failure -- Cloud-init configuration errors -- Missing or corrupted disk image -- Insufficient memory for guest OS to boot -- QEMU/KVM virtualization errors -- VirtualMachineInstance (VMI) creation failures -- virt-launcher pod crashes - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VMI (VirtualMachineInstance) status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and overall VMI state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -``` - -**2. Check VMI conditions for errors**: - -Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. View virt-launcher pod logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -First, get pod name from step 3, then: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -Look for QEMU errors, memory allocation failures, device errors. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**5. Check virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. - -Alternatively, use `pods_get` to get full pod details: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') -``` - -**6. Access VM console to see guest OS boot messages**: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -Look for kernel panic, initramfs errors, filesystem errors. - -**7. Check VM events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep -A 20 "Events:" -``` - -**Common Error Patterns**: - -1. **Guest OS Boot Failure**: - - Console shows kernel panic - - Guest hangs at GRUB or boot loader - - Cloud-init errors during first boot - -2. **Insufficient Memory**: - - Guest OS kills processes due to OOM - - VMI logs show memory allocation errors - -3. **QEMU Crashes**: - - virt-launcher logs show QEMU segfaults - - VMI repeatedly restarts - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for kernel panic, initramfs errors, filesystem errors. - -2. **Check virt-launcher pod logs for QEMU errors**: - - Use `pods_log` from diagnostic step 4 to view logs. - - Look for: - - "qemu-system-x86_64: ..." errors - - Memory allocation failures - - Device errors - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx - ``` - -3. **Increase memory if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check VMI memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -4. **Verify disk image integrity**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check PVC is bound: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - If using container disk, verify image exists and is accessible (check VMI spec). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get pvc -n - ``` - -5. **Check cloud-init configuration** (if applicable): - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. - - Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq - ``` - -6. **Restart VMI** (soft reset): - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete VMI (VM controller will recreate it): - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for new VMI to start (use `resources_get` to check status). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n -w - ``` - -7. **Check virtualization extensions** (KVM): - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - - # In debug shell: - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should eventually return `Running`). - -Check VMI is running: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should show `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should eventually return: Running - -oc get vmi -n -# Should show: Running -``` - ---- - -### VM Won't Stop - -**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state - -**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. - -**Possible Causes**: -- Guest OS not responding to ACPI shutdown signal -- virt-launcher pod stuck and not terminating -- VirtualMachineInstance (VMI) deletion blocked by finalizers -- Guest shutdown scripts hanging -- Filesystem sync issues in guest OS - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (might show `Stopping`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**2. Check VMI status and deletion timestamp**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' -``` - -**3. Check virt-launcher pod status**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -``` - -**4. Check VMI events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vmi -n | grep -A 10 "Events:" -``` - -**5. Check VMI finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.metadata.finalizers}' -``` - -**6. Check if guest is responsive** (if VMI still exists): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Patterns**: -- VMI shows `deletionTimestamp` but never actually deletes -- virt-launcher pod in `Terminating` state -- VM runStrategy is `Halted` but printableStatus shows `Stopping` - -**Solutions** (Use MCP Tools First): - -1. **Wait for graceful shutdown** (default: 30 seconds): - - Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. - -2. **Force stop by deleting VMI**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - This is the standard way to force-stop a VM. - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete vmi -n - oc get vmi -n - # Should return: Error from server (NotFound) - ``` - -3. **Force delete VMI with grace period** (if VMI won't delete): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete vmi -n --grace-period=0 --force - ``` - -4. **Force delete virt-launcher pod**: - - **MCP Tool**: `pods_delete` (from openshift-virtualization) - - First, find the pod using `pods_list_in_namespace` from diagnostic step 3. - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - ⚠️ **Note**: For force deletion with grace period, use CLI fallback. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') - oc delete pod $POD_NAME -n --force --grace-period=0 - ``` - -5. **Remove VMI finalizers** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VMI using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update VMI using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Patch VM runStrategy directly** (ensure consistency): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current VM using `resources_get` (diagnostic step 1) - 2. Set `.spec.runStrategy` to `"Halted"` - 3. Update VM using `resources_create_or_update` with modified JSON - - **CLI Fallback** (merge patch easier via CLI): - Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" - ```bash - oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' - ``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -After remediation, check VM status: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Stopped` or `Halted`). - -Verify VMI is gone: - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Verify virt-launcher pod is gone: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for pods matching `virt-launcher-`. Should return no results. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Stopped or Halted - -oc get vmi -n -# Should return: Error from server (NotFound) - -oc get pods -n | grep virt-launcher- -# Should return: No resources found -``` - -**Prevention**: -- Ensure guest OS has ACPI support enabled -- Use proper shutdown commands in guest OS -- Avoid forceful stops unless necessary (can corrupt guest filesystem) - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/network-errors.md b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/network-errors.md deleted file mode 100644 index 82fa313c..00000000 --- a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/network-errors.md +++ /dev/null @@ -1,429 +0,0 @@ ---- -title: VM Network Errors -category: kubevirt -sources: - - title: Multus CNI - Network Attachment Definitions - url: https://github.com/k8snetworkplumbingwg/multus-cni - date_accessed: 2026-02-17 -tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] -semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] -use_cases: [vm-creation, network-troubleshooting] -related_docs: [INDEX.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Network Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. - -**When to use this document**: -- VM created successfully but secondary networks not attached -- NetworkAttachmentDefinition not found errors -- Multus CNI failures -- SR-IOV device attachment issues - -**Skills that use this**: vm-create - ---- - -### Network Attachment Failures - -**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working - -**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). - -**Possible Causes**: -- NetworkAttachmentDefinition doesn't exist in the namespace -- Multus CNI not installed or not configured on cluster -- Namespace mismatch (NAD in different namespace than VM) -- Interface name conflicts in VM spec -- Bridge/network configuration errors in NAD -- SR-IOV device not available (if using SR-IOV) - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List NetworkAttachmentDefinitions in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "" -} -``` - -Review `.items[].metadata.name` for available NADs. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -n -``` - -**2. List NetworkAttachmentDefinitions in all namespaces**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition" -} -``` - -Omit `namespace` parameter to list across all namespaces. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definitions -A -``` - -**3. Check specific NetworkAttachmentDefinition**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "namespace": "", - "name": "" -} -``` - -Review `.spec.config` for CNI configuration. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get network-attachment-definition -n -o yaml -``` - -**4. Check VM network configuration**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.networks` to see network references. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq -``` - -**5. Check VM domain interfaces**: - -Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq -``` - -**6. Check virt-launcher pod network annotations** (shows actual attachments): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**7. Check Multus is installed**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "openshift-multus" -} -``` - -Should show Multus CNI pods running. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n openshift-multus -``` - -**8. Check for errors in virt-launcher pod events**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n -``` - -**Common Error Messages**: -- `"network-attachment-definition not found"` - NAD doesn't exist in namespace -- `"multus CNI not configured"` - Multus not installed or misconfigured -- `"interface name conflict"` - Duplicate interface names in VM spec -- `"failed to add network"` - CNI plugin error (check NAD config) -- `"no available devices"` - SR-IOV device not available (if using SR-IOV) - -**Solutions** (Use MCP Tools First): - -1. **Verify NetworkAttachmentDefinition exists in correct namespace**: - - Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. - - If NAD is in different namespace, copy it to VM namespace: - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get NAD from source namespace using `resources_get` - 2. Modify `.metadata.namespace` to target namespace - 3. Create NAD in target namespace using `resources_create_or_update` - - **CLI Fallback** (stream processing easier via CLI): - Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" - ```bash - oc get network-attachment-definition -n -o yaml | \ - sed "s/namespace: /namespace: /" | \ - oc apply -f - - ``` - -2. **Create missing NetworkAttachmentDefinition**: - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - Example: Linux bridge network - - **Parameters**: - ```json - { - "apiVersion": "k8s.cni.cncf.io/v1", - "kind": "NetworkAttachmentDefinition", - "metadata": { - "name": "vlan100", - "namespace": "" - }, - "spec": { - "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" - ```bash - cat < - spec: - config: '{ - "cniVersion": "0.3.1", - "type": "bridge", - "bridge": "br1", - "vlan": 100, - "ipam": { - "type": "host-local", - "subnet": "192.168.100.0/24" - } - }' - EOF - ``` - -3. **Check Multus CNI installation**: - - Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. - - To check cluster network operator: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "config.openshift.io/v1", - "kind": "ClusterOperator" - } - ``` - - Filter for `network` operator. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n openshift-multus - oc get clusteroperators network - ``` - -4. **Fix interface name conflicts** (if VM has duplicate names): - - Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. - - Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. - - **CLI Fallback** (interactive edit easier via CLI): - Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" - ```bash - oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' - oc edit vm -n - ``` - -5. **Validate NAD configuration syntax**: - - Use `resources_get` from diagnostic step 3, extract `.spec.config`. - - Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get network-attachment-definition -n -o jsonpath='{.spec.config}' - ``` - -6. **Check SR-IOV device availability** (if using SR-IOV networks): - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - List SR-IOV network node policies: - - **Parameters**: - ```json - { - "apiVersion": "sriovnetwork.openshift.io/v1", - "kind": "SriovNetworkNodePolicy", - "namespace": "openshift-sriov-network-operator" - } - ``` - - Check SR-IOV device plugin pods: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "openshift-sriov-network-operator" - } - ``` - - Filter for pods with "device-plugin" in name. - - Check available SR-IOV devices on node: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "Node", - "name": "" - } - ``` - - Review `.status.allocatable` for SR-IOV resources. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get sriovnetworknodepolicy -n openshift-sriov-network-operator - oc get pods -n openshift-sriov-network-operator | grep device-plugin - oc describe node | grep -A 10 "Allocatable:" - ``` - -7. **Recreate VM with corrected network configuration** (if needed): - - If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -After remediation, check virt-launcher pod network status: - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. - -Should show all attached networks with status. Example output: -```json -[ - { - "name": "openshift-sdn", - "interface": "eth0", - "ips": ["10.128.2.10"], - "default": true - }, - { - "name": "vlan100", - "interface": "net1", - "ips": ["192.168.100.5"] - } -] -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq -``` - -**Check from inside VM** (via console): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# In guest OS: -ip addr show -# Should show all network interfaces (eth0, net1, etc.) -``` - -**Common Network Types**: -- **Linux Bridge**: Layer 2 bridge for VLAN networks -- **SR-IOV**: High-performance direct device assignment -- **macvlan**: MAC-based VLAN for container networks -- **OVN-Kubernetes**: OpenShift native overlay network - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/runtime-errors.md b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/runtime-errors.md deleted file mode 100644 index 804d3cbc..00000000 --- a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/runtime-errors.md +++ /dev/null @@ -1,616 +0,0 @@ ---- -title: VM Runtime Errors -category: kubevirt -sources: - - title: Kubernetes Pod Lifecycle - url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ - date_accessed: 2026-02-17 -tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] -semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] -use_cases: [vm-creation, vm-lifecycle, diagnostics] -related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] -last_updated: 2026-02-17 ---- - -# VM Runtime Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. - -**When to use this document**: -- VM or virt-launcher pod shows `CrashLoopBackOff` status -- virt-launcher pod repeatedly restarting -- Guest OS kernel panics on boot - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -### CrashLoopBackOff - -**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting - -**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. - -**Possible Causes**: -- Guest OS kernel panic on boot -- Insufficient resources (memory/CPU) for guest OS -- Corrupted disk image or filesystem -- QEMU/libvirt crashes due to configuration errors -- Missing or incompatible device drivers in guest -- Resource limits too low for virt-launcher pod -- Virtualization features (KVM) not available on node - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check virt-launcher pod restart count**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# Look at RESTARTS column -``` - -**2. View recent crash logs** (previous container instance): - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "previous": true, - "tail": 100 -} -``` - -Look for QEMU errors, kernel panics, or segfaults. - -**CLI Fallback** (if MCP unavailable): -```bash -oc logs -n virt-launcher--xxx --previous -``` - -**3. Check current virt-launcher logs**: - -**MCP Tool**: `pods_log` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "tail": 100 -} -``` - -⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. - -**CLI Fallback** (if MCP unavailable or all containers needed): -```bash -oc logs -n virt-launcher--xxx --all-containers -``` - -**4. Check VMI conditions for crash details**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" -} -``` - -Extract `.status.conditions` for crash details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vmi -n -o jsonpath='{.status.conditions}' | jq -``` - -**5. Check pod events for crash reasons**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches the virt-launcher pod name. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" -``` - -**6. Check pod resource limits**: - -**MCP Tool**: `pods_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx" -} -``` - -Extract `.spec.containers[0].resources` for resource limits. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq -``` - -**7. Check node kubelet logs for OOM kills**: - -⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -oc adm node-logs -u kubelet | grep -i oom -``` - -**8. Access guest console** (if VM briefly starts): - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -``` - -**Common Crash Patterns**: - -1. **Guest Kernel Panic**: - - Console logs show kernel panic messages - - Guest crashes immediately after boot - - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" - -2. **OOM (Out of Memory)**: - - Pod killed with reason: `OOMKilled` - - Guest runs out of memory during boot or operation - - virt-launcher logs show memory allocation failures - -3. **QEMU Crash**: - - virt-launcher logs show QEMU segmentation fault - - Symptoms: "qemu-system-x86_64: terminated by signal" - - Configuration incompatibility or QEMU bug - -4. **Disk Image Corruption**: - - Guest cannot boot from disk - - Filesystem errors in guest console - - DataVolume import failed - -**Solutions** (Use MCP Tools First): - -1. **Check guest console for kernel panic or boot errors**: - - ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - virtctl console -n - ``` - - Look for: - - Kernel panic messages - - Initramfs errors - - Filesystem mounting failures - - Missing device errors - -2. **Review virt-launcher crash logs**: - - Use `pods_log` with `previous: true` from diagnostic step 2. - - Look for: - - QEMU command line errors - - Device initialization failures - - Memory allocation errors - - Signal termination (SIGSEGV, SIGABRT) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc logs -n virt-launcher--xxx --previous - ``` - -3. **Check for OOM (Out of Memory) kills**: - - **MCP Tool**: `pods_get` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "", - "name": "virt-launcher--xxx" - } - ``` - - Extract `.status.containerStatuses[0].lastState.terminated.reason`. - - If returns `"OOMKilled"`: - - Option 1: Increase virt-launcher memory limits - - Option 2: Decrease guest memory allocation - - Option 3: Use smaller instance type - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' - ``` - -4. **Increase resources if OOM detected**: - - **MCP Tool**: `resources_get` (from openshift-virtualization) - - Check current memory allocation: - - **Parameters**: - ```json - { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachineInstance", - "namespace": "", - "name": "" - } - ``` - - Extract `.spec.domain.resources.requests.memory`. - - If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' - ``` - -5. **Verify disk image integrity**: - - **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) - - Check DataVolume status: - - **Parameters for list**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" - } - ``` - - **Parameters for specific DV**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Check `.status.phase` (should be `Succeeded`). - - If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get datavolume -n - oc get datavolume -n -o jsonpath='{.status.phase}' - ``` - -6. **Check virtualization (KVM) availability**: - - ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). - - **CLI Required** (no MCP alternative): - ```bash - oc debug node/ - chroot /host - lsmod | grep kvm - # Should show kvm_intel or kvm_amd - ``` - -7. **Simplify VM configuration** (eliminate variables): - - Try creating minimal VM using vm-create skill with: - - Small instance type - - No secondary networks - - Simple container disk (e.g., Fedora) - - No cloud-init - - If minimal VM works, add features back one by one. - -8. **Recreate VM with different workload** (test disk image): - - If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) - -After remediation, check pod restart count stops increasing: - -**Parameters for pods**: -```json -{ - "namespace": "" -} -``` - -Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). - -Check VM reaches Running state: - -**Parameters for VM**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Check `.status.printableStatus` (should return `Running`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n | grep virt-launcher- -# RESTARTS should stabilize - -oc get vm -n -o jsonpath='{.status.printableStatus}' -# Should return: Running -``` - -Verify guest is responsive: - -⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). - -**CLI Required** (no MCP alternative): -```bash -virtctl console -n -# Should show login prompt or OS console -``` - -**Advanced Debugging**: - -**MCP Tool**: `pods_exec` (from openshift-virtualization) - -Check libvirt domain XML: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["virsh", "dumpxml", "1"] -} -``` - -Check QEMU process: - -**Parameters**: -```json -{ - "namespace": "", - "name": "virt-launcher--xxx", - "command": ["ps", "aux"] -} -``` - -Filter output for "qemu" process. - -**CLI Fallback** (if MCP unavailable): -```bash -oc exec -n virt-launcher--xxx -- virsh dumpxml 1 -oc exec -n virt-launcher--xxx -- ps aux | grep qemu -``` - -**Prevention**: -- Start with minimal VM configuration and add complexity gradually -- Use recommended instance types for your workload -- Test disk images before deploying to production -- Ensure nodes have adequate resources and KVM support -- Monitor resource usage and set appropriate limits - ---- - -## Workaround Patterns for MCP Tool Limitations - -### General Pattern: Diagnose → Propose → Confirm → Execute - -When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): - -**1. Diagnose** the root cause using Kubernetes commands -```bash -oc describe vm -n -oc get events -n --field-selector involvedObject.name= -``` - -**2. Propose** a manual workaround to the user with clear explanation -- Explain why the issue occurred -- Show what will be changed -- List alternative options - -**3. Confirm** with user before executing (Human-in-the-Loop) -- Wait for explicit "yes" or "apply workaround" -- Never auto-execute modifications - -**4. Execute** the workaround using oc/kubectl -```bash -oc patch vm -n --type=merge -p '...' -``` - -**5. Verify** the fix was successful -```bash -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**6. Document** the limitation and suggest filing enhancement request -- Note this is temporary until MCP tool is enhanced -- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues - -### Example: Adding Tolerations Workaround - -**Diagnostic Output**: -```markdown -## ⚠️ VM Scheduling Issue Detected - -**Root Cause**: Node taints prevent VM scheduling - -**Details**: -- Found 3 nodes with taint: `virtualization=true:NoSchedule` -- VM spec does not include matching tolerations -- This prevents VM scheduling on virtualization-dedicated nodes -``` - -**Proposed Workaround**: -```bash -oc patch vm web-server -n vms --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' -``` - -**User Confirmation Required**: -``` -How would you like to proceed? -- "apply workaround" - I'll patch the VM with tolerations -- "manual" - I'll provide instructions for you to apply manually -- "cancel" - Delete the VM and abort creation -``` - -**After Execution**: -```markdown -## ✓ Workaround Applied Successfully - -**Action**: Added tolerations for taint `virtualization=true:NoSchedule` -**New Status**: Stopped (VM can now be scheduled) - -**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. -Future VMs in this cluster will need the same fix until the tool is enhanced. -``` - ---- - -## VM Status Reference - -### Status Values - -| Status | Meaning | Action Required | -|--------|---------|-----------------| -| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | -| `Running` | VM is running | Normal | -| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | -| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | -| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | -| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | -| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | -| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | -| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | -| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | - -### Checking VM Status - -```bash -# Get printable status -oc get vm -n -o jsonpath='{.status.printableStatus}' - -# Get detailed status and conditions -oc get vm -n -o jsonpath='{.status}' | jq - -# Watch status changes in real-time -oc get vm -n -w -``` - ---- - -## Best Practices for Agents - -When implementing diagnostic workflows: - -1. **Always verify VM status** after creation (wait 5-10 seconds first) -2. **Consult this document** when encountering error status values -3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) -4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) -5. **Respect human-in-the-loop** for all VM modifications -6. **Document temporary workarounds** and their limitations clearly -7. **Suggest filing issues** for missing MCP tool features - -### Document Consultation Pattern - -```markdown -**Document Consultation** (REQUIRED): -1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes -2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." -``` - ---- - -## Known MCP Tool Limitations - -### vm_create tool - -**Currently Supported**: -- ✓ Namespace, name (required) -- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) -- ✓ Size hints (small, medium, large) -- ✓ Storage size -- ✓ Autostart flag -- ✓ Networks (Multus NetworkAttachmentDefinitions) -- ✓ Performance family (u1, o1, c1, m1) -- ✓ Instance type, preference - -**Not Currently Supported** (requires workarounds): -- ✗ Tolerations (for node taints) -- ✗ Node selectors -- ✗ Affinity/anti-affinity rules -- ✗ Resource requests/limits (beyond instance type) -- ✗ Custom labels/annotations -- ✗ SSH keys injection -- ✗ Cloud-init user data - -**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. - -**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues - ---- - -## Additional Resources - -- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) -- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) -- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) -- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) - ---- - -[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/scheduling-errors.md b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/scheduling-errors.md deleted file mode 100644 index e47c8c4f..00000000 --- a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/scheduling-errors.md +++ /dev/null @@ -1,417 +0,0 @@ ---- -title: VM Scheduling Errors -category: kubevirt -sources: - - title: KubeVirt User Guide - Node Placement - url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ - date_accessed: 2026-02-06 - - title: Kubernetes Taints and Tolerations - url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - date_accessed: 2026-02-06 -tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] -semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] -use_cases: [vm-creation, vm-lifecycle] -related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] -last_updated: 2026-02-17 ---- - -# VM Scheduling Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**When to use this document**: -- VM shows status `ErrorUnschedulable` after creation or start attempt -- VM events mention scheduling failures, taints, resources, or node selectors - -**Skills that use this**: vm-create, vm-lifecycle-manager - ---- - -## ErrorUnschedulable - -**Symptom**: VM shows status `ErrorUnschedulable` after creation - -**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. - -**Possible Causes**: - -### 1. Node Taints (Most Common) - -Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM events for scheduling failures**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events where `involvedObject.name` == "" and look for messages like: -- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" - -**CLI Fallback** (if MCP unavailable): -```bash -oc get events -n --field-selector involvedObject.name= -``` - -**2. Check node taints in the cluster**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' -``` - -**Common Taint Patterns**: -- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule -- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes -- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads - -**Solution - Add Tolerations to VM**: - -The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. - -**Workaround (post-creation using MCP Tools)**: - -**Step 1**: Get current VM spec - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -**Step 2**: Modify the returned JSON to add tolerations - -Add to `.spec.template.spec.tolerations`: -```json -{ - "tolerations": [ - { - "key": "virtualization", - "operator": "Equal", - "value": "true", - "effect": "NoSchedule" - } - ] -} -``` - -**Step 3**: Update VM with modified spec - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Parameters**: -```json -{ - "resource": "" -} -``` - -Pass the complete modified VM resource as YAML or JSON string. - -**Step 4**: Verify tolerations were added - -Use `resources_get` again and check `.spec.template.spec.tolerations` in response. - -**Step 5**: Check if VM status improved - -Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. - -**CLI Fallback** (if MCP patch is too complex): -```bash -# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" -' - -# Verify tolerations -oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq - -# Check status -oc get vm -n -o jsonpath='{.status.printableStatus}' -``` - -**Example - Multiple Tolerations**: -```bash -oc patch vm -n --type=merge -p ' -spec: - template: - spec: - tolerations: - - key: "virtualization" - operator: "Equal" - value: "true" - effect: "NoSchedule" - - key: "dedicated" - operator: "Equal" - value: "virt-workloads" - effect: "NoSchedule" -' -``` - -**Toleration Operators**: -- `Equal` - Key and value must match exactly -- `Exists` - Only key must exist (ignores value) - -**Toleration Effects**: -- `NoSchedule` - Don't schedule new pods (existing pods continue) -- `PreferNoSchedule` - Avoid scheduling if possible -- `NoExecute` - Don't schedule AND evict existing pods - -**Alternative Solutions**: -1. **Remove node taints** (if you have cluster-admin access): - ```bash - oc adm taint nodes virtualization=true:NoSchedule- - ``` - -2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits - -3. **File enhancement request** - Request tolerations support in openshift-mcp-server: - https://github.com/openshift/openshift-mcp-server/issues - ---- - -### 2. Insufficient Resources - -Not enough CPU, memory, or storage available on any node. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM resource requests**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' -``` - -**2. Check node resource availability**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. - -Alternatively, use `nodes_top` MCP tool for current resource usage. - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe nodes | grep -A 5 "Allocated resources" -``` - -**3. Look for VM events mentioning "Insufficient"**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe vm -n | grep "Insufficient" -``` - -**Example Event**: -``` -0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. -``` - -**Solutions** (Use MCP Tools First): - -1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) -2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill -3. **Delete unused VMs** - Use vm-delete skill to free up resources -4. **Check resource quotas**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters for quota**: - ```json - { - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" - } - ``` - - **Parameters for limit range**: - ```json - { - "apiVersion": "v1", - "kind": "LimitRange", - "namespace": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - oc describe limitrange -n - ``` - ---- - -### 3. Node Selector Mismatch - -VM requires specific node labels that don't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check VM node selector requirements**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.nodeSelector` to see required node labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' -``` - -**2. List available node labels**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node" -} -``` - -For each node in `.items[]`, review `.metadata.labels` for available labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes --show-labels -``` - -**3. Check if any nodes match the selector**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "Node", - "labelSelector": "=" -} -``` - -Should return at least one node with matching labels. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get nodes -l = -``` - -**Solutions** (Use MCP Tools First): - -**Option 1: Remove node selector from VM** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get current VM using `resources_get` (diagnostic step 1) -2. Remove `.spec.template.spec.nodeSelector` field -3. Update VM using `resources_create_or_update` with modified JSON - -**CLI Fallback** (JSON patch easier via CLI): -Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" -```bash -oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' -``` - -**Option 2: Add label to nodes** - -**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - -**Process**: -1. Get node using `resources_get` -2. Add label to `.metadata.labels` -3. Update node using `resources_create_or_update` - -⚠️ **Note**: Node labeling typically requires cluster admin privileges. - -**CLI Fallback** (simpler via CLI): -Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" -```bash -oc label node = -``` - ---- - -[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/storage-errors.md b/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/storage-errors.md deleted file mode 100644 index 16ba584d..00000000 --- a/evaluation/without_skills/rh-virt__vm-snapshot-create/environment/docs/troubleshooting/storage-errors.md +++ /dev/null @@ -1,1011 +0,0 @@ ---- -title: VM Storage Errors -category: kubevirt -sources: - - title: KubeVirt CDI - DataVolumes - url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ - date_accessed: 2026-02-17 - - title: Kubernetes Persistent Volumes - url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ - date_accessed: 2026-02-17 -tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] -semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] -use_cases: [vm-creation, vm-deletion, vm-cloning] -related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] -last_updated: 2026-02-17 ---- - -# VM Storage Errors - -[← Back to Index](INDEX.md) - -## Overview - -This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. - -**When to use this document**: -- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` -- Storage deletion fails after VM deletion -- DataVolume cloning operations fail -- PVC provisioning issues - -**Skills that use this**: vm-create, vm-delete, vm-clone - ---- - -### ErrorDataVolumeNotReady - -**Symptom**: VM shows status `ErrorDataVolumeNotReady` - -**Description**: The DataVolume (persistent storage) backing the VM is not ready. - -**Possible Causes**: - -#### 1. DataVolume Still Provisioning - -Storage provisioning takes time, especially for large disks or when importing images. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Get detailed DataVolume information**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` and `.status.conditions` for provisioning details. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o json -``` - -**3. Check PVC (PersistentVolumeClaim) bound status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Check `.status.phase` for each PVC (should be `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. - -#### 2. Storage Class Not Found - -The requested storage class doesn't exist in the cluster. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List available storage classes**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" -} -``` - -Review the list of available storage classes (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -``` - -**2. Check DataVolume's requested storage class**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.spec.pvc.storageClassName` in the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' -``` - -**Solution**: -1. Use a valid storage class from the cluster -2. Recreate VM with correct storage class parameter - -#### 3. Insufficient Storage Quota - -Namespace has insufficient storage quota to provision the PVC. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check resource quotas**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**2. Check storage usage**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase -``` - -**Solution**: -1. Request quota increase from cluster admin -2. Delete unused PVCs to free quota -3. Reduce VM storage size - ---- - -### ErrorPvcNotFound - -**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. - -**Diagnostic Steps** (Use MCP Tools First): - -**1. List PVCs in namespace**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review the list of available PVCs (check `.items[].metadata.name`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check VM's PVC references**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", - "namespace": "", - "name": "" -} -``` - -Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' -``` - -**Solution**: -- Wait for DataVolume to create the PVC -- Manually create missing PVC -- Fix VM spec to reference correct PVC name - ---- - - ---- - -### Storage Deletion Failures - -**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace - -**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. - -**Possible Causes**: -- PVC still bound to active PersistentVolume with `Retain` policy -- DataVolume still being referenced by another resource -- CDI (Containerized Data Importer) controller issues -- Storage class retention policy preventing deletion -- Finalizers on PVC/DataVolume blocking cleanup -- PVC still mounted by a pod - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check PVC status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**2. Check specific PVC phase**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Released` or `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -o jsonpath='{.status.phase}' -``` - -**3. Check DataVolume status**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "" -} -``` - -Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**4. Check what's using the PVC**: - -**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' -``` - -**5. Check PVC finalizers**: - -Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. - -**6. Check DataVolume finalizers**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Extract `.metadata.finalizers` from the returned JSON. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.metadata.finalizers}' -``` - -**7. Check PV reclaim policy**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume" -} -``` - -Filter results for PV where `.spec.claimRef.name` matches ``. - -To get specific PV policy: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolume", - "name": "" -} -``` - -Check `.spec.persistentVolumeReclaimPolicy`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pv | grep -oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' -``` - -**Common Finalizer Patterns**: -- `kubernetes.io/pvc-protection` - Protects PVC while in use -- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer - -**Solutions** (Use MCP Tools First): - -1. **Delete DataVolume first, then PVC**: - - **MCP Tool**: `resources_delete` (from openshift-virtualization) - - Delete DataVolume first (often blocks PVC deletion): - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" - } - ``` - - Wait a few seconds, then delete PVC: - - **Parameters**: - ```json - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc delete datavolume -n - oc delete pvc -n - ``` - -2. **Check for pods still using PVC**: - - **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) - - **Parameters**: - ```json - { - "namespace": "" - } - ``` - - Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. - - Then delete the pods using `pods_delete`: - - **Parameters**: - ```json - { - "namespace": "", - "name": "" - } - ``` - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' - oc delete pod -n - ``` - -3. **Force delete PVC** (if safe to do so): - - ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. - - **CLI Fallback** (required for force delete): - Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" - ```bash - oc delete pvc -n --grace-period=0 --force - ``` - -4. **Remove finalizers from PVC** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PVC using `resources_get` (diagnostic step 2) - 2. Remove items from `.metadata.finalizers` array - 3. Update PVC using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -5. **Remove finalizers from DataVolume** (⚠️ last resort): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current DataVolume using `resources_get` (diagnostic step 6) - 2. Remove items from `.metadata.finalizers` array - 3. Update DataVolume using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" - ```bash - oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' - ``` - -6. **Change PV reclaim policy** (if PV has Retain policy): - - **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) - - **Process**: - 1. Get current PV policy using `resources_get` (diagnostic step 7) - 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` - 3. Update PV using `resources_create_or_update` with modified JSON - - ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. - - **CLI Fallback** (JSON patch easier via CLI): - Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" - ```bash - oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' - oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' - ``` - -**Storage Quota Check** (Use MCP Tools First): - -After deletion, verify storage quota is freed: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters for quota check**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.used` to verify storage quota is freed. - -**Parameters for PVC verification**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "" -} -``` - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -oc get pvc -n -``` - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Confirm PVC is deleted: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -Confirm DataVolume is deleted: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Should return "Not Found" error. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -# Should return: Error from server (NotFound) - -oc get datavolume -n -# Should return: Error from server (NotFound) -``` - ---- - -### DataVolume Cloning Failures - -**Symptom**: VM clone created successfully but DataVolume clone operation fails - -**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. - -**Possible Causes**: -- CSI driver doesn't support volume cloning -- Source PVC storage class incompatible with cloning -- Cross-namespace cloning not permitted by storage backend -- Insufficient storage quota in target namespace -- Source PVC not in `Bound` state -- Storage class doesn't have volume cloning enabled -- CDI (Containerized Data Importer) controller issues - -**Diagnostic Steps** (Use MCP Tools First): - -**1. Check DataVolume clone status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Review `.status.phase`, `.status.conditions`, and `.metadata.name`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -``` - -**2. Check DataVolume events for errors**: - -**MCP Tool**: `events_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "namespace": "" -} -``` - -Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe datavolume -n -``` - -**3. Check DataVolume phase**: - -Use `resources_get` from step 1, extract `.status.phase`. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -``` - -**4. Check if storage class supports cloning**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass", - "name": "" -} -``` - -Review the full YAML output for cloning-related configurations. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get storageclass -o yaml | grep -A 5 -i clone -``` - -**5. Check CSI driver capabilities**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "storage.k8s.io/v1", - "kind": "CSIDriver" -} -``` - -Review `.items[].metadata.name` for available CSI drivers. - -**CLI Fallback** (if MCP unavailable): -```bash -oc get csidriver -``` - -**6. Check source PVC status**: - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should be `Bound` for cloning to work). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get pvc -n -``` - -**7. Check target namespace storage quota**: - -**MCP Tool**: `resources_list` (from openshift-virtualization) - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "namespace": "" -} -``` - -Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). - -**CLI Fallback** (if MCP unavailable): -```bash -oc describe quota -n -``` - -**8. Check CDI controller logs**: - -**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) - -First, list pods in openshift-cnv namespace: - -**Parameters for pods_list_in_namespace**: -```json -{ - "namespace": "openshift-cnv", - "labelSelector": "app.kubernetes.io/component=cdi-deployment" -} -``` - -Then get logs using `pods_log`: - -**Parameters**: -```json -{ - "namespace": "openshift-cnv", - "name": "", - "tail": 100 -} -``` - -**CLI Fallback** (if MCP unavailable or easier via CLI): -```bash -oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') -``` - -**Common Error Messages**: -- `"volume cloning is not supported"` - CSI driver lacks clone capability -- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage -- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace -- `"insufficient quota"` - Target namespace lacks storage quota -- `"source PVC not bound"` - Source PVC must be in Bound state for cloning -- `"StorageClass does not support cloning"` - Storage class configuration issue - -**Solutions** (Use MCP Tools First): - -1. **Check storage class clone support**: - - **MCP Tool**: `resources_list` (from openshift-virtualization) - - **Parameters**: - ```json - { - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass" - } - ``` - - For each storage class in `.items[]`, check: - - `.metadata.name` (storage class name) - - `.provisioner` (CSI driver) - - Storage classes using CSI drivers typically support cloning. Look for provisioners like: - - `csi.ovirt.org` (oVirt CSI) - - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) - - `ebs.csi.aws.com` (AWS EBS CSI) - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner - ``` - -2. **Verify source PVC is bound**: - - Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc get pvc -n -o jsonpath='{.status.phase}' - ``` - -3. **Check target namespace quota**: - - Use `resources_list` from diagnostic step 7 to check quota. - - If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). - - **CLI Fallback** (if MCP unavailable): - ```bash - oc describe quota -n - ``` - -4. **Use snapshot-based cloning** (alternative method): - - **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) - - **Step 1**: Create VolumeSnapshot of source PVC - - **Parameters**: - ```json - { - "apiVersion": "snapshot.storage.k8s.io/v1", - "kind": "VolumeSnapshot", - "metadata": { - "name": "-snapshot", - "namespace": "" - }, - "spec": { - "source": { - "persistentVolumeClaimName": "" - } - } - } - ``` - - **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) - - **Step 3**: Create new DataVolume from snapshot - - **Parameters**: - ```json - { - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "metadata": { - "name": "-rootdisk", - "namespace": "" - }, - "spec": { - "source": { - "snapshot": { - "name": "-snapshot", - "namespace": "" - } - }, - "storage": { - "resources": { - "requests": { - "storage": "50Gi" - } - }, - "storageClassName": "" - } - } - } - ``` - - **CLI Fallback** (YAML easier via CLI): - Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" - ```bash - cat <-snapshot - namespace: - spec: - source: - persistentVolumeClaimName: - EOF - - oc get volumesnapshot -snapshot -n - - cat <-rootdisk - namespace: - spec: - source: - snapshot: - name: -snapshot - namespace: - storage: - resources: - requests: - storage: 50Gi - storageClassName: - EOF - ``` - -5. **Use "new empty storage" option** (vm-clone skill): - - If cloning isn't supported, create VM with empty storage - - Manually copy data if needed - -6. **Cross-namespace cloning workaround**: - - Some storage backends require snapshot for cross-namespace cloning - - Create snapshot in source namespace, restore in target namespace (see solution 4 above) - -**Verification** (Use MCP Tools First): - -**MCP Tool**: `resources_get` (from openshift-virtualization) - -Check DataVolume reached Succeeded phase: - -**Parameters**: -```json -{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "namespace": "", - "name": "" -} -``` - -Check `.status.phase` (should return `Succeeded`). - -Check PVC was created and bound: - -**Parameters**: -```json -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "namespace": "", - "name": "-rootdisk" -} -``` - -Check `.status.phase` (should return `Bound`). - -**CLI Fallback** (if MCP unavailable): -```bash -oc get datavolume -n -o jsonpath='{.status.phase}' -# Should return: Succeeded - -oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' -# Should return: Bound -``` - -**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. - ---- - - ---- - -[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md)